Compare commits
46
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e1e3bbc691 | ||
|
|
cd6fad6c60 | ||
|
|
1dcebd640c | ||
|
|
8bf759d6f8 | ||
|
|
fc3266d455 | ||
|
|
06099579e6 | ||
|
|
f68e92b3b6 | ||
|
|
e34bd70e13 | ||
|
|
50703c960a | ||
|
|
6505d339aa | ||
|
|
b4c7a15045 | ||
|
|
3f11ea6168 | ||
|
|
69de20a427 | ||
|
|
cbc53ee9f1 | ||
|
|
7c25b351f7 | ||
|
|
785b744554 | ||
|
|
966a25d129 | ||
|
|
7511a0fd38 | ||
|
|
add035d940 | ||
|
|
88e5b78207 | ||
|
|
b0a025a326 | ||
|
|
c86739178d | ||
|
|
b50bf862f0 | ||
|
|
4f1c6e236b | ||
|
|
5b2c21bdaf | ||
|
|
25e7ae158b | ||
|
|
08151ca93d | ||
|
|
b2cc489fd4 | ||
|
|
347de6a023 | ||
|
|
fd747a0ffc | ||
|
|
7a4b6eebb2 | ||
|
|
a64e99402b | ||
|
|
3b1d00501f | ||
|
|
5123654b0d | ||
|
|
a8aff3a39d | ||
|
|
9dcad31441 | ||
|
|
e29ff4d4b2 | ||
|
|
195a1d33ac | ||
|
|
2561d3b789 | ||
|
|
1b8fbdbb75 | ||
|
|
0095db8cbc | ||
|
|
b4c0189225 | ||
|
|
b7cf210f0f | ||
|
|
bc14f29db7 | ||
|
|
f95f66d76a | ||
|
|
2cecfcf0ac |
@@ -16,9 +16,19 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
# Timeouts are the measured job time plus at least 50%, and never under 20 minutes for a browser job,
|
||||
# because runner speed varies. Before sharding (main 2cecfcf, one worker): frontend coverage 32m10s,
|
||||
# page workflows firefox 18m59s, webkit 19m26s. First sharded run (PR #274 run 39712909047, 3 workers,
|
||||
# no npm cache, where npm ci stalled for 5m in six of eight jobs): coverage shards 11m19s/10m47s/10m49s
|
||||
# (tests 5m20s/4m42s/4m46s), merge 5m52s (report and check 4s), firefox shards 7m53s/9m27s (tests
|
||||
# 4m41s/3m29s), webkit shards 12m25s+ (cancelled at the old 12-minute limit after its 267 tests passed in
|
||||
# 6m29s) and 4m53s (tests 3m08s). With the npm cache hit (run 39725176320 at cbc8a61, npm ci 0-2s):
|
||||
# coverage shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox 7m57s/5m21s (tests
|
||||
# 7m08s/4m34s), webkit 8m16s/5m03s (tests 7m26s/3m58s); a cache miss adds up to 5m to npm ci, which
|
||||
# the 20-minute limits absorb.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (chromium and touch)
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -26,12 +36,15 @@ jobs:
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
# This job runs every page workflow twice (chromium, then touch) on one worker with V8 coverage on, so it
|
||||
# takes about twice a single-browser job. Measured 2026-10-20: whole job 18m26s-24m57s (main 5ded3cf
|
||||
# 22m15s; PR head 6359999 was cancelled at the old 25-minute limit while still passing), coverage step
|
||||
# alone 20m55s-22m17s+, against ~12-15 minutes for the firefox/webkit jobs. 40 minutes leaves headroom
|
||||
# for suite growth while still stopping a hung run; re-measure if a run passes 30 minutes.
|
||||
timeout-minutes: 40
|
||||
timeout-minutes: 20
|
||||
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
|
||||
# server, leave one CPU for the servers and the runner.
|
||||
env:
|
||||
PM_TEST_WORKERS: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -50,24 +63,107 @@ jobs:
|
||||
"$venv/bin/python" -m pip install --no-cache-dir -r requirements-dev.txt
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
# npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home),
|
||||
# which npm also uses. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- run: npm run --prefix tools/vendor check
|
||||
- run: npm run --prefix tools/vendor test
|
||||
- name: Frontend unit, viewer, workflow, and coverage checks
|
||||
run: npm run --prefix tools/vendor test:coverage
|
||||
- name: Upload frontend coverage and Chromium traces
|
||||
# Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests
|
||||
# run on the lightest shard instead.
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor check
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor test
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
|
||||
- name: Upload raw coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: coverage-raw-${{ matrix.shard }}
|
||||
path: coverage/js/raw/
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
- name: Upload Chromium and touch traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: frontend-traces-shard-${{ matrix.shard }}
|
||||
path: tools/vendor/test-results/
|
||||
retention-days: 14
|
||||
|
||||
frontend-coverage-merge:
|
||||
name: frontend coverage (merge)
|
||||
needs: frontend-coverage
|
||||
runs-on: ubuntu-latest
|
||||
# Same image as the shards: the raw files name sources by the container's workspace path.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- run: node --version && node --version | grep -q '^v24\.'
|
||||
# npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home),
|
||||
# which npm also uses. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
# Each artifact lands in its own coverage-raw-N directory, so the next step can count them.
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: coverage-raw-*
|
||||
path: ${{ runner.temp }}/coverage-raw
|
||||
# coverage:merge only checks that raw files exist; merging two of three shards would report
|
||||
# partial coverage, so a missing or empty shard artifact fails here.
|
||||
- name: Verify every shard's raw coverage arrived
|
||||
shell: bash
|
||||
env:
|
||||
STAGED_RAW: ${{ runner.temp }}/coverage-raw
|
||||
EXPECTED_SHARDS: 3
|
||||
run: |
|
||||
found=0
|
||||
for shard in $(seq 1 "$EXPECTED_SHARDS"); do
|
||||
directory="$STAGED_RAW/coverage-raw-$shard"
|
||||
count=0
|
||||
if [ -d "$directory" ]; then count=$(find "$directory" -maxdepth 1 -type f -name '*.json' | wc -l); fi
|
||||
echo "coverage-raw-$shard: $count raw files"
|
||||
if [ "$count" -gt 0 ]; then found=$((found + 1)); fi
|
||||
done
|
||||
echo "shard artifacts with raw coverage: $found of $EXPECTED_SHARDS"
|
||||
if [ "$found" -ne "$EXPECTED_SHARDS" ]; then
|
||||
echo "::error::Expected raw coverage from $EXPECTED_SHARDS shards, got $found of $EXPECTED_SHARDS"
|
||||
exit 1
|
||||
fi
|
||||
staged=$(find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' | wc -l)
|
||||
mkdir -p coverage/js/raw
|
||||
# cp -n never overwrites; a clash shows up as fewer merged files than staged ones.
|
||||
find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' -exec cp -n -t coverage/js/raw {} + || true
|
||||
merged=$(find coverage/js/raw -maxdepth 1 -type f -name '*.json' | wc -l)
|
||||
if [ "$merged" -ne "$staged" ]; then
|
||||
echo "::error::Shards wrote clashing raw file names: $staged staged, $merged merged"
|
||||
exit 1
|
||||
fi
|
||||
echo "merged $merged raw files"
|
||||
- name: Frontend coverage report and baseline check
|
||||
run: npm run --prefix tools/vendor coverage:merge
|
||||
- name: Upload frontend coverage report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: frontend-test-reports
|
||||
path: |
|
||||
tools/vendor/test-results/
|
||||
coverage/js/
|
||||
!coverage/js/raw/**
|
||||
retention-days: 14
|
||||
|
||||
browser-workflows:
|
||||
name: page workflows (${{ matrix.browser }})
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -75,11 +171,16 @@ jobs:
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 25
|
||||
timeout-minutes: 20
|
||||
# Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test
|
||||
# server, leave one CPU for the servers and the runner.
|
||||
env:
|
||||
PM_TEST_WORKERS: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: [firefox, webkit]
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -98,22 +199,29 @@ jobs:
|
||||
"$venv/bin/python" -m pip install --no-cache-dir -r requirements-dev.txt
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
# npm's download cache, keyed on the lockfile; inside the container ~ is the job's HOME (/github/home),
|
||||
# which npm also uses. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- name: Page workflows in ${{ matrix.browser }}
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }}
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: browser-traces-${{ matrix.browser }}
|
||||
name: browser-traces-${{ matrix.browser }}-shard-${{ matrix.shard }}
|
||||
path: tools/vendor/test-results/
|
||||
retention-days: 14
|
||||
|
||||
# Branch protection requires the exact context "viewerjs". This job keeps that
|
||||
# name and fails unless every frontend job above succeeded, including when one
|
||||
# was cancelled or skipped.
|
||||
# name and fails unless every frontend job above succeeded (every shard, the
|
||||
# coverage merge and every browser shard), including when one was cancelled or skipped.
|
||||
viewerjs:
|
||||
needs: [frontend-coverage, browser-workflows]
|
||||
needs: [frontend-coverage, frontend-coverage-merge, browser-workflows]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
@@ -121,8 +229,10 @@ jobs:
|
||||
- name: Require every frontend job to succeed
|
||||
env:
|
||||
COVERAGE_RESULT: ${{ needs.frontend-coverage.result }}
|
||||
MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }}
|
||||
WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }}
|
||||
run: |
|
||||
echo "frontend-coverage: $COVERAGE_RESULT"
|
||||
echo "frontend-coverage-merge: $MERGE_RESULT"
|
||||
echo "browser-workflows: $WORKFLOWS_RESULT"
|
||||
test "$COVERAGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
|
||||
@@ -211,3 +211,6 @@ coverage/
|
||||
|
||||
# Disposable upstream checkout for real-host CI
|
||||
.ci-comfyui/
|
||||
|
||||
# Old library files a Move library renamed (never deleted; #249 S6b)
|
||||
*.moved-to-*
|
||||
|
||||
@@ -751,9 +751,34 @@ ComfyUI_PromptManager/
|
||||
|
||||
## Configuration
|
||||
|
||||
### Where your library lives
|
||||
|
||||
New installs keep the library (`prompts.db`, `config.json` with the CivitAI key, the
|
||||
database backups and the logs) in ComfyUI's system user folder
|
||||
`ComfyUI/user/__promptmanager/`. An update, a reinstall or ComfyUI-Manager replacing
|
||||
the node folder no longer touches it, and ComfyUI's `/userdata` web API does not
|
||||
serve `__` system folders. Thumbnails stay in `output/thumbnails`.
|
||||
|
||||
- **Upgrading from an earlier version:** your library stays where it is, in the node
|
||||
folder, until you move it; nothing is moved on its own. The workspace shows a notice
|
||||
above every page and System › Overview a "Library location" row. **Move library**
|
||||
copies the database with SQLite's backup API, verifies the copy (integrity check, every
|
||||
workflow document, the same rows in every table), copies `config.json` and the backups,
|
||||
switches to the new folder without a restart and only then renames the old files
|
||||
`prompts.db.moved-to-__promptmanager` and `config.json.moved-to-__promptmanager` (never
|
||||
deleted; the old backups stay too). If anything fails, the library stays in the node
|
||||
folder, unchanged. **Keep it here** stops the notice; the Overview row stays. Take a
|
||||
backup first (System › Backup & export or the dashboard's Backup) as with any change.
|
||||
- **Your own folder:** set `PROMPT_MANAGER_DATA_DIR` to an absolute folder outside
|
||||
ComfyUI's output, input and temp folders and your image sources. If it is empty and a
|
||||
node-folder library exists, that library stays in use until you move it there.
|
||||
- **An explicit `database.default_path`** in `config.json` and `PROMPT_MANAGER_CONFIG_PATH`
|
||||
keep working as before (described below); such a library is not offered a move.
|
||||
|
||||
### Database Settings
|
||||
|
||||
You can customize the database path by creating a `config.json` file in the extension root:
|
||||
You can customize the database path by creating a `config.json` file in the library's
|
||||
folder (the node folder for a library that was not moved):
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -970,6 +995,15 @@ MIT License - see LICENSE file for details.
|
||||
|
||||
## Changelog
|
||||
|
||||
### Unreleased (Library location)
|
||||
|
||||
- **📁 Library outside the node folder**: new installs keep `prompts.db`, `config.json`, the database backups and the logs in `ComfyUI/user/__promptmanager/`, where an update or reinstall of the node cannot delete them and ComfyUI's `/userdata` API does not reach. An existing library stays in the node folder until you press **Move library** (a verified copy, then a live switch; the old files are renamed `*.moved-to-__promptmanager`, never deleted) or **Keep it here**. `PROMPT_MANAGER_DATA_DIR` names your own folder. After moving, restart ComfyUI once so the logs follow. Going back to an older version after a move: an older version does not look in `user/__promptmanager`, so stop ComfyUI and copy `user/__promptmanager/prompts.db` (with its `-wal` file, if one is left) and `config.json` into the node folder; the `*.moved-to-__promptmanager` files are the library as it was at the move, without later changes.
|
||||
|
||||
### Unreleased (Workflows stored once)
|
||||
|
||||
- **🗜️ Smaller database**: each distinct workflow and prompt graph is stored once, compressed, instead of again for every image. Images recorded by this version use it from the first start; the maintenance job "Store each workflow once" rewrites the existing records and then reclaims the space. A 20,000-image test library went from 1.4 GB to 60 MB. The job always takes a verified backup first: the first time, `prompts.db.pre-workflow-storage_<date time>` beside the database, kept until you delete it; on later runs, a normal pooled safety backup. No image or video file is opened or changed.
|
||||
- **⚠️ Downgrading**: images recorded by this version (and, after the job, all images) are stored in the new form. An older Prompt Manager version still opens and even restores such a database, but shows those images without workflows or prompt graphs until you update again. To go back to an older version, stop ComfyUI, delete `prompts.db-wal` and `prompts.db-shm`, and copy the **earliest** `prompts.db.pre-workflow-storage_<date time>` over `prompts.db`, or, if the job never ran, a backup taken before this update (records added since that backup are lost). Otherwise stay on this version.
|
||||
|
||||
### v3.3.1 (Proxy Routing Fix)
|
||||
|
||||
- **🌐 Proxy support**: the dashboard, gallery, tag page and metadata viewer now work when ComfyUI is reached through a proxy mount such as SwarmUI's `/ComfyBackendDirect/`. Scripts, images, API calls, progress streams and gallery folder filters stay under the mount, and the canvas node's buttons use ComfyUI's `fileURL()`. Direct ComfyUI URLs are unchanged. Thanks to @ghosttigerllc-bit ([#164](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/164))
|
||||
|
||||
+2262
-41
File diff suppressed because it is too large
Load Diff
+26
@@ -90,6 +90,10 @@ try:
|
||||
host_app = getattr(config.server_instance, "app", None)
|
||||
if host_app is not None and hasattr(host_app, "on_cleanup"):
|
||||
host_app.on_cleanup.append(api.cleanup_history_reads)
|
||||
host_app.on_cleanup.append(api.cleanup_annotation_captures)
|
||||
if host_app is not None and hasattr(host_app, "on_shutdown"):
|
||||
# Open Needs review change streams end before the server waits on them.
|
||||
host_app.on_shutdown.append(api.close_review_events)
|
||||
|
||||
except Exception as e:
|
||||
# Log to internal logging system without console spam
|
||||
@@ -121,6 +125,28 @@ try:
|
||||
_global_prompt_tracker = get_prompt_tracker(_global_db)
|
||||
_global_image_monitor = get_image_monitor(_global_db, _global_prompt_tracker)
|
||||
|
||||
# The Needs review pass for ambiguous new outputs runs through the API's
|
||||
# capture store (the one the scan and the Tag dialog use): a linked output
|
||||
# is never queued and its pending items leave the queue. It stops before
|
||||
# that store is closed at shutdown.
|
||||
_review_api = globals().get("api")
|
||||
if _review_api is not None:
|
||||
_global_image_monitor.set_review_recorder(_review_api.record_review_file)
|
||||
# Every settled output's preview is queued for the API's background
|
||||
# worker (#249): the workspace never asks to generate thumbnails.
|
||||
_global_image_monitor.set_thumbnail_queue(_review_api.queue_capture_thumbnail)
|
||||
_review_host = getattr(config.server_instance, "app", None)
|
||||
if _review_host is not None and hasattr(_review_host, "on_cleanup"):
|
||||
|
||||
async def _stop_review_recording(
|
||||
app, monitor=_global_image_monitor, api=_review_api
|
||||
):
|
||||
monitor.set_review_recorder(None)
|
||||
monitor.set_thumbnail_queue(None)
|
||||
api.close_auto_thumbnails(final=True)
|
||||
|
||||
_review_host.on_cleanup.insert(0, _stop_review_recording)
|
||||
|
||||
# Start monitoring
|
||||
_global_image_monitor.start_monitoring()
|
||||
_init_logger.info("Global image monitoring system started successfully")
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Image annotation schema group (#249): file locators, byte records, image tags.
|
||||
|
||||
The group installs like ``image_favorites`` and ``saved_views``: optional in old
|
||||
backups, required once a database is current. It owns
|
||||
|
||||
- ``media_annotation_metadata``: one row, the group's schema version and the
|
||||
image index revision;
|
||||
- ``media_annotation_files``: one private canonical locator per physical file
|
||||
(``normcase(join(realpath(root), relative))``, server-side only);
|
||||
- ``media_annotation_bytes``: one record per observed whole-file SHA-256 of a
|
||||
file, with the size and ``mtime_ns`` verified when those bytes were last seen
|
||||
and the tag set's revision;
|
||||
- ``media_image_tags``: the ordered image tag set of one byte record, joined
|
||||
into the global ``tags`` registry so vocabulary maintenance carries image
|
||||
links like prompt and pair links.
|
||||
|
||||
Installation never backfills anything from prompt or pair tags and never
|
||||
touches the filesystem. Validation reads only: the catalog objects are compared
|
||||
after whitespace compaction, columns, foreign keys and indexes through the
|
||||
PRAGMAs, and persisted values in single bounded SQL passes.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
from .image_favorites import MAX_KEY_LENGTH
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
METADATA_TABLE = "media_annotation_metadata"
|
||||
FILES_TABLE = "media_annotation_files"
|
||||
BYTES_TABLE = "media_annotation_bytes"
|
||||
IMAGE_TAGS_TABLE = "media_image_tags"
|
||||
TABLES = (METADATA_TABLE, FILES_TABLE, BYTES_TABLE, IMAGE_TAGS_TABLE)
|
||||
MAX_LOCATOR_LENGTH = MAX_KEY_LENGTH
|
||||
_TIMESTAMP_BYTES = 64
|
||||
|
||||
_TABLE_DDL = {
|
||||
METADATA_TABLE: (
|
||||
f"CREATE TABLE {METADATA_TABLE} (singleton_id INTEGER PRIMARY KEY "
|
||||
"CHECK(singleton_id=1), schema_version INTEGER NOT NULL "
|
||||
f"CHECK(schema_version={SCHEMA_VERSION}), "
|
||||
"image_index_revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(image_index_revision>=0))"
|
||||
),
|
||||
FILES_TABLE: (
|
||||
f"CREATE TABLE {FILES_TABLE} (id TEXT PRIMARY KEY CHECK(length(id)=36), "
|
||||
"locator TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL)"
|
||||
),
|
||||
BYTES_TABLE: (
|
||||
f"CREATE TABLE {BYTES_TABLE} (id TEXT PRIMARY KEY CHECK(length(id)=36), "
|
||||
f"file_id TEXT NOT NULL REFERENCES {FILES_TABLE}(id) ON DELETE CASCADE, "
|
||||
"sha256 TEXT NOT NULL CHECK(length(sha256)=64 AND sha256 NOT GLOB "
|
||||
"'*[^0-9a-f]*'), size INTEGER NOT NULL CHECK(size>=0), "
|
||||
"mtime_ns INTEGER NOT NULL, verified_at TEXT NOT NULL, "
|
||||
"first_seen_at TEXT NOT NULL, tag_revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(tag_revision>=0), UNIQUE(file_id, sha256))"
|
||||
),
|
||||
IMAGE_TAGS_TABLE: (
|
||||
f"CREATE TABLE {IMAGE_TAGS_TABLE} (byte_id TEXT NOT NULL REFERENCES "
|
||||
f"{BYTES_TABLE}(id) ON DELETE CASCADE, tag_id INTEGER NOT NULL "
|
||||
"REFERENCES tags(id) ON DELETE CASCADE, position INTEGER NOT NULL "
|
||||
"CHECK(position>=0), PRIMARY KEY(byte_id, tag_id))"
|
||||
),
|
||||
}
|
||||
_INDEX_DDL = {
|
||||
"idx_media_annotation_bytes_fingerprint": (
|
||||
"CREATE INDEX idx_media_annotation_bytes_fingerprint ON "
|
||||
f"{BYTES_TABLE}(file_id, size, mtime_ns)"
|
||||
),
|
||||
"idx_media_image_tags_tag": (
|
||||
f"CREATE INDEX idx_media_image_tags_tag ON {IMAGE_TAGS_TABLE}(tag_id)"
|
||||
),
|
||||
"idx_media_image_tags_position": (
|
||||
"CREATE UNIQUE INDEX idx_media_image_tags_position ON "
|
||||
f"{IMAGE_TAGS_TABLE}(byte_id, position)"
|
||||
),
|
||||
}
|
||||
# SQLite names the indexes behind TEXT primary keys and UNIQUE constraints
|
||||
# ``sqlite_autoindex_<table>_<n>``; their catalog rows carry no SQL.
|
||||
_AUTOINDEXES = {FILES_TABLE: 2, BYTES_TABLE: 2, IMAGE_TAGS_TABLE: 1}
|
||||
# (name, declared type, not null, default, primary key position)
|
||||
_COLUMNS = {
|
||||
METADATA_TABLE: (
|
||||
("singleton_id", "INTEGER", 0, None, 1),
|
||||
("schema_version", "INTEGER", 1, None, 0),
|
||||
("image_index_revision", "INTEGER", 1, "0", 0),
|
||||
),
|
||||
FILES_TABLE: (
|
||||
("id", "TEXT", 0, None, 1),
|
||||
("locator", "TEXT", 1, None, 0),
|
||||
("created_at", "TEXT", 1, None, 0),
|
||||
),
|
||||
BYTES_TABLE: (
|
||||
("id", "TEXT", 0, None, 1),
|
||||
("file_id", "TEXT", 1, None, 0),
|
||||
("sha256", "TEXT", 1, None, 0),
|
||||
("size", "INTEGER", 1, None, 0),
|
||||
("mtime_ns", "INTEGER", 1, None, 0),
|
||||
("verified_at", "TEXT", 1, None, 0),
|
||||
("first_seen_at", "TEXT", 1, None, 0),
|
||||
("tag_revision", "INTEGER", 1, "0", 0),
|
||||
),
|
||||
IMAGE_TAGS_TABLE: (
|
||||
("byte_id", "TEXT", 1, None, 1),
|
||||
("tag_id", "INTEGER", 1, None, 2),
|
||||
("position", "INTEGER", 1, None, 0),
|
||||
),
|
||||
}
|
||||
# (column, parent table, parent column, on delete)
|
||||
_FOREIGN_KEYS = {
|
||||
METADATA_TABLE: set(),
|
||||
FILES_TABLE: set(),
|
||||
BYTES_TABLE: {("file_id", FILES_TABLE, "id", "CASCADE")},
|
||||
IMAGE_TAGS_TABLE: {
|
||||
("byte_id", BYTES_TABLE, "id", "CASCADE"),
|
||||
("tag_id", "tags", "id", "CASCADE"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql).lower()
|
||||
|
||||
|
||||
def _uuid(column: str) -> str:
|
||||
"""SQL: *column* is a lowercase hyphenated 36-character UUID string."""
|
||||
return (
|
||||
f"typeof({column}) = 'text' AND length({column}) = 36 "
|
||||
f"AND substr({column}, 9, 1) = '-' AND substr({column}, 14, 1) = '-' "
|
||||
f"AND substr({column}, 19, 1) = '-' AND substr({column}, 24, 1) = '-' "
|
||||
f"AND length(replace({column}, '-', '')) = 32 "
|
||||
f"AND replace({column}, '-', '') NOT GLOB '*[^0-9a-f]*'"
|
||||
)
|
||||
|
||||
|
||||
def _timestamp(column: str) -> str:
|
||||
return (
|
||||
f"typeof({column}) = 'text' "
|
||||
f"AND length(CAST({column} AS BLOB)) <= {_TIMESTAMP_BYTES}"
|
||||
)
|
||||
|
||||
|
||||
def _non_negative(column: str) -> str:
|
||||
return f"typeof({column}) = 'integer' AND {column} >= 0"
|
||||
|
||||
|
||||
# Every row of a table must satisfy its predicate; one pass each, LIMIT 1.
|
||||
_ROW_PREDICATES = {
|
||||
FILES_TABLE: " AND ".join(
|
||||
(
|
||||
_uuid("id"),
|
||||
"typeof(locator) = 'text'",
|
||||
# UTF-8 is at most 4 bytes per character; the decoded length of
|
||||
# the few candidates above the byte bound is checked in Python.
|
||||
f"length(CAST(locator AS BLOB)) <= {MAX_LOCATOR_LENGTH * 4}",
|
||||
_timestamp("created_at"),
|
||||
)
|
||||
),
|
||||
BYTES_TABLE: " AND ".join(
|
||||
(
|
||||
_uuid("id"),
|
||||
_uuid("file_id"),
|
||||
"typeof(sha256) = 'text' AND length(sha256) = 64 "
|
||||
"AND sha256 NOT GLOB '*[^0-9a-f]*'",
|
||||
_non_negative("size"),
|
||||
"typeof(mtime_ns) = 'integer'",
|
||||
_timestamp("verified_at"),
|
||||
_timestamp("first_seen_at"),
|
||||
_non_negative("tag_revision"),
|
||||
)
|
||||
),
|
||||
IMAGE_TAGS_TABLE: " AND ".join(
|
||||
(_uuid("byte_id"), "typeof(tag_id) = 'integer'", _non_negative("position"))
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _expected_catalog() -> set:
|
||||
objects = {("table", name, _compact(sql)) for name, sql in _TABLE_DDL.items()}
|
||||
objects |= {("index", name, _compact(sql)) for name, sql in _INDEX_DDL.items()}
|
||||
objects |= {
|
||||
("index", f"sqlite_autoindex_{table}_{n}", "")
|
||||
for table, count in _AUTOINDEXES.items()
|
||||
for n in range(1, count + 1)
|
||||
}
|
||||
return objects
|
||||
|
||||
|
||||
def _catalog(conn: sqlite3.Connection) -> list:
|
||||
names = TABLES + tuple(_INDEX_DDL)
|
||||
marks = ",".join("?" * len(names))
|
||||
table_marks = ",".join("?" * len(TABLES))
|
||||
return conn.execute( # only "?" placeholders are interpolated
|
||||
"SELECT type,name,sql FROM sqlite_master "
|
||||
f"WHERE name IN ({marks}) OR tbl_name IN ({table_marks})", # nosec B608
|
||||
names + TABLES,
|
||||
).fetchall()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
for table, expected in _COLUMNS.items():
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[4], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({table})")
|
||||
)
|
||||
if columns != expected:
|
||||
return f"Image annotations table {table} has invalid columns"
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({table})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS[table]:
|
||||
return f"Image annotations table {table} has invalid foreign keys"
|
||||
if conn.execute(f"PRAGMA foreign_key_check({table})").fetchone():
|
||||
return f"Image annotations table {table} contains foreign key violations"
|
||||
return None
|
||||
|
||||
|
||||
def _value_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
# Only this module's constants are interpolated into the SQL below.
|
||||
rows = conn.execute(
|
||||
"SELECT COUNT(*), MIN(typeof(singleton_id) = 'integer' " # nosec B608
|
||||
"AND singleton_id = 1 AND typeof(schema_version) = 'integer' "
|
||||
f"AND schema_version = {SCHEMA_VERSION} "
|
||||
f"AND {_non_negative('image_index_revision')}) FROM {METADATA_TABLE}"
|
||||
).fetchone()
|
||||
if rows[0] != 1 or rows[1] != 1:
|
||||
return "Image annotations metadata is invalid"
|
||||
for table, predicate in _ROW_PREDICATES.items():
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {table} WHERE NOT ({predicate}) LIMIT 1" # nosec B608
|
||||
).fetchone():
|
||||
return f"Image annotations table {table} contains invalid values"
|
||||
# length() stops at an embedded NUL, so the SQL pass bounds bytes and the
|
||||
# few candidates over the character bound are measured decoded here.
|
||||
for (raw,) in conn.execute(
|
||||
f"SELECT CAST(locator AS BLOB) FROM {FILES_TABLE} " # nosec B608
|
||||
f"WHERE length(CAST(locator AS BLOB)) > {MAX_LOCATOR_LENGTH}"
|
||||
):
|
||||
if len(bytes(raw).decode("utf-8", "replace")) > MAX_LOCATOR_LENGTH:
|
||||
return f"Image annotations table {FILES_TABLE} contains invalid values"
|
||||
return None
|
||||
|
||||
|
||||
def annotation_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> Optional[str]:
|
||||
"""Reject partial or unsupported annotation schemas and invalid rows.
|
||||
|
||||
Reads only: nothing is ever repaired. A database without any of the
|
||||
group's objects is acceptable for backups that predate it unless
|
||||
``allow_missing`` is false.
|
||||
"""
|
||||
objects = _catalog(conn)
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing image annotation schema"
|
||||
present = {name for kind, name, _sql in objects if kind == "table"}
|
||||
if present != set(TABLES):
|
||||
return "Incomplete image annotation schema"
|
||||
actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
|
||||
if actual != _expected_catalog():
|
||||
return "Image annotations has an invalid schema definition"
|
||||
return _structure_error(conn) or _value_error(conn)
|
||||
|
||||
|
||||
def migrate_annotation_state(conn: sqlite3.Connection) -> None:
|
||||
"""Install the group atomically; validate before and after creating it."""
|
||||
conn.execute("SAVEPOINT media_annotation_schema")
|
||||
try:
|
||||
error = annotation_schema_error(conn)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
for sql in _TABLE_DDL.values():
|
||||
conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
for sql in _INDEX_DDL.values():
|
||||
conn.execute(sql.replace("INDEX", "INDEX IF NOT EXISTS", 1))
|
||||
conn.execute(
|
||||
f"INSERT OR IGNORE INTO {METADATA_TABLE} "
|
||||
"(singleton_id, schema_version, image_index_revision) VALUES (1, ?, 0)",
|
||||
(SCHEMA_VERSION,),
|
||||
)
|
||||
error = annotation_schema_error(conn, allow_missing=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE media_annotation_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO media_annotation_schema")
|
||||
conn.execute("RELEASE media_annotation_schema")
|
||||
raise
|
||||
@@ -0,0 +1,461 @@
|
||||
"""Per-image tag sets (#249): byte-exact storage in the shared vocabulary.
|
||||
|
||||
A tag set belongs to one byte record (``media_annotation_bytes``): the
|
||||
whole-file SHA-256 of one physical file, owned by the file's private locator
|
||||
(``media_annotation_files``). Callers pass verified facts (digest, size,
|
||||
``mtime_ns``); nothing here reads the filesystem or resolves an owner through
|
||||
prompt links, so tagging one image never changes another image, a prompt or a
|
||||
pair.
|
||||
|
||||
Commands run in one ``_pair_transaction`` fenced by the storage epoch, like the
|
||||
native Library and vocabulary commands. A save is a full replacement guarded
|
||||
by the set's ``tag_revision``; the identical set sent again (a lost-response
|
||||
retry, or the current set at its own revision) answers ``replayed: true`` and
|
||||
writes nothing. ``add_image_tags`` unions into the existing set, keeping the
|
||||
stored spelling. Names enter the global ``tags`` registry exactly as spelled,
|
||||
so vocabulary rename, merge and delete carry image links with prompt and pair
|
||||
links (``PromptDatabase._move_tag_links``).
|
||||
|
||||
Cheap reads (spec amendment S1) treat a byte record as *current* only while
|
||||
the file's present ``(size, mtime_ns)`` equals the record's verified
|
||||
fingerprint; a file without a current record shows no image tags.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from typing import Iterator
|
||||
from uuid import uuid4
|
||||
|
||||
from .annotation_schema import (
|
||||
BYTES_TABLE,
|
||||
FILES_TABLE,
|
||||
IMAGE_TAGS_TABLE,
|
||||
MAX_LOCATOR_LENGTH,
|
||||
METADATA_TABLE,
|
||||
)
|
||||
from .library_commands import _UUID, LibraryCommandError, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .vocabulary import validate_vocabulary_tag_name
|
||||
|
||||
MAX_IMAGE_TAGS = 100
|
||||
_LOOKUP_CHUNK = 500
|
||||
_DIGEST = re.compile(r"[0-9a-f]{64}")
|
||||
_MAX_INT = 2**63 - 1
|
||||
_MIN_INT = -(2**63)
|
||||
# Only this module's table constants are interpolated into the SQL below.
|
||||
_OWNER = f"SELECT id FROM {FILES_TABLE} WHERE locator=?" # nosec B608
|
||||
_INSERT_OWNER = (
|
||||
f"INSERT INTO {FILES_TABLE}(id,locator,created_at) VALUES(?,?,?)" # nosec B608
|
||||
)
|
||||
_BYTE = (
|
||||
f"SELECT id,tag_revision,size,mtime_ns FROM {BYTES_TABLE} " # nosec B608
|
||||
"WHERE file_id=? AND sha256=?"
|
||||
)
|
||||
_INSERT_BYTE = (
|
||||
f"INSERT INTO {BYTES_TABLE}(id,file_id,sha256,size,mtime_ns," # nosec B608
|
||||
"verified_at,first_seen_at,tag_revision) VALUES(?,?,?,?,?,?,?,0)"
|
||||
)
|
||||
_REFRESH_BYTE = (
|
||||
f"UPDATE {BYTES_TABLE} SET size=?,mtime_ns=?,verified_at=? WHERE id=?" # nosec B608
|
||||
)
|
||||
_REFRESH_BY_LOCATOR = (
|
||||
f"UPDATE {BYTES_TABLE} SET size=?,mtime_ns=?,verified_at=? " # nosec B608
|
||||
f"WHERE sha256=? AND file_id=({_OWNER})"
|
||||
)
|
||||
_BYTE_TAGS = (
|
||||
f"SELECT t.name FROM {IMAGE_TAGS_TABLE} mit " # nosec B608
|
||||
"JOIN tags t ON t.id=mit.tag_id WHERE mit.byte_id=? ORDER BY mit.position"
|
||||
)
|
||||
_CLEAR_TAGS = f"DELETE FROM {IMAGE_TAGS_TABLE} WHERE byte_id=?" # nosec B608
|
||||
_INSERT_TAG = (
|
||||
f"INSERT INTO {IMAGE_TAGS_TABLE}(byte_id,tag_id,position) " # nosec B608
|
||||
"VALUES(?,?,?)"
|
||||
)
|
||||
_BUMP_REVISION = (
|
||||
f"UPDATE {BYTES_TABLE} SET tag_revision=tag_revision+1 WHERE id=?" # nosec B608
|
||||
)
|
||||
_REVISION = f"SELECT tag_revision FROM {BYTES_TABLE} WHERE id=?" # nosec B608
|
||||
_BUMP_INDEX = (
|
||||
f"UPDATE {METADATA_TABLE} SET " # nosec B608
|
||||
"image_index_revision=image_index_revision+1 WHERE singleton_id=1"
|
||||
)
|
||||
_READ = (
|
||||
"SELECT b.id,b.tag_revision," # nosec B608
|
||||
f"(SELECT COUNT(*) FROM {BYTES_TABLE} o WHERE o.file_id=f.id AND o.sha256!=?) "
|
||||
f"FROM {FILES_TABLE} f LEFT JOIN {BYTES_TABLE} b "
|
||||
"ON b.file_id=f.id AND b.sha256=? WHERE f.locator=?"
|
||||
)
|
||||
_FINGERPRINTS = (
|
||||
"SELECT f.locator,b.id,b.size,b.mtime_ns,b.verified_at " # nosec B608
|
||||
f"FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
"WHERE f.locator IN ({marks})"
|
||||
)
|
||||
_TAGS_OF = (
|
||||
f"SELECT mit.byte_id,t.name FROM {IMAGE_TAGS_TABLE} mit " # nosec B608
|
||||
"JOIN tags t ON t.id=mit.tag_id WHERE mit.byte_id IN ({marks}) "
|
||||
"ORDER BY mit.byte_id,mit.position"
|
||||
)
|
||||
_RECORDS = (
|
||||
"SELECT f.locator,b.id,b.size,b.mtime_ns,t.name " # nosec B608
|
||||
f"FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"LEFT JOIN {IMAGE_TAGS_TABLE} mit ON mit.byte_id=b.id "
|
||||
"LEFT JOIN tags t ON t.id=mit.tag_id "
|
||||
"ORDER BY f.locator,b.first_seen_at,b.id,mit.position"
|
||||
)
|
||||
|
||||
|
||||
class AnnotationCommandError(LibraryCommandError):
|
||||
"""An image tag command lost a race or hit the set's bound.
|
||||
|
||||
``code`` is ``tag-conflict`` (the set changed since it was read) or
|
||||
``tag-limit`` (the union would exceed ``MAX_IMAGE_TAGS``); ``current`` is
|
||||
the stored set ``{sha256, tags, tag_revision}`` read inside the
|
||||
transaction, so the caller can show it.
|
||||
"""
|
||||
|
||||
def __init__(self, code: str, epoch: str, current: dict | None = None):
|
||||
super().__init__(code, epoch)
|
||||
self.current = current
|
||||
|
||||
|
||||
class ImageTagLimitError(ValueError):
|
||||
"""More than ``MAX_IMAGE_TAGS`` distinct names in one set."""
|
||||
|
||||
|
||||
def normalize_image_tags(names: object) -> list[str]:
|
||||
"""Trim and validate each name, fold case-insensitive repeats, cap at 100.
|
||||
|
||||
The first occurrence keeps its spelling and position. Raises ``ValueError``
|
||||
for anything but a list of valid vocabulary names, and its subclass
|
||||
``ImageTagLimitError`` for more than ``MAX_IMAGE_TAGS`` distinct names.
|
||||
"""
|
||||
if not isinstance(names, list):
|
||||
raise ValueError("tags must be a list of strings")
|
||||
result: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for value in names:
|
||||
name = validate_vocabulary_tag_name(value)
|
||||
key = name.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(name)
|
||||
if len(result) > MAX_IMAGE_TAGS:
|
||||
raise ImageTagLimitError(f"An image can carry at most {MAX_IMAGE_TAGS} tags")
|
||||
return result
|
||||
|
||||
|
||||
def _validate_locator(locator: object) -> None:
|
||||
if not isinstance(locator, str) or not 0 < len(locator) <= MAX_LOCATOR_LENGTH:
|
||||
raise ValueError(
|
||||
f"locator must be a nonblank path of at most {MAX_LOCATOR_LENGTH} chars"
|
||||
)
|
||||
|
||||
|
||||
def _validate_digest(sha256: object) -> None:
|
||||
if not isinstance(sha256, str) or not _DIGEST.fullmatch(sha256):
|
||||
raise ValueError("sha256 must be 64 lowercase hexadecimal characters")
|
||||
|
||||
|
||||
def _validate_int(value: object, name: str, minimum: int) -> None:
|
||||
if type(value) is not int or not minimum <= value <= _MAX_INT:
|
||||
raise ValueError(f"{name} must be an integer not below {minimum}")
|
||||
|
||||
|
||||
def _validate_target(locator: object, sha256: object, size: object, mtime_ns: object):
|
||||
_validate_locator(locator)
|
||||
_validate_digest(sha256)
|
||||
_validate_int(size, "size", 0)
|
||||
_validate_int(mtime_ns, "mtime_ns", _MIN_INT)
|
||||
|
||||
|
||||
def _validate_fingerprint(locator: object, fingerprint: object) -> None:
|
||||
_validate_locator(locator)
|
||||
if (
|
||||
not isinstance(fingerprint, tuple)
|
||||
or len(fingerprint) != 2
|
||||
or any(type(value) is not int for value in fingerprint)
|
||||
):
|
||||
raise ValueError("stats must map locators to (size, mtime_ns) integer pairs")
|
||||
|
||||
|
||||
def _new_id() -> str:
|
||||
"""A canonical UUID; the schema's CHECK lets NULL through, this does not."""
|
||||
value = str(uuid4())
|
||||
if not _UUID.fullmatch(value):
|
||||
raise ValueError("Annotation ids must be canonical UUIDs")
|
||||
return value
|
||||
|
||||
|
||||
def _receipt(epoch: str, sha256: str, tags: list, revision: int, replayed: bool):
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"sha256": sha256,
|
||||
"tags": list(tags),
|
||||
"tag_revision": revision,
|
||||
"replayed": replayed,
|
||||
}
|
||||
|
||||
|
||||
class AnnotationOperations:
|
||||
"""PromptDatabase mixin: image tag sets, fingerprints and cheap reads.
|
||||
|
||||
``_`` helpers take the open ``_pair_transaction`` connection.
|
||||
"""
|
||||
|
||||
@contextmanager
|
||||
def _annotation_read(self) -> Iterator[sqlite3.Connection]:
|
||||
"""One deferred read transaction, so paired SELECTs see one snapshot."""
|
||||
conn = self.model.get_connection()
|
||||
if conn.in_transaction:
|
||||
raise sqlite3.ProgrammingError(
|
||||
"Annotation reads require their own snapshot"
|
||||
)
|
||||
with conn:
|
||||
conn.execute("BEGIN")
|
||||
yield conn
|
||||
|
||||
@staticmethod
|
||||
def _annotation_owner(conn: sqlite3.Connection, locator: str, now: str) -> str:
|
||||
row = conn.execute(_OWNER, (locator,)).fetchone()
|
||||
if row is not None:
|
||||
return row[0]
|
||||
file_id = _new_id()
|
||||
conn.execute(_INSERT_OWNER, (file_id, locator, now))
|
||||
return file_id
|
||||
|
||||
@staticmethod
|
||||
def _annotation_byte(
|
||||
conn: sqlite3.Connection,
|
||||
file_id: str,
|
||||
sha256: str,
|
||||
size: int,
|
||||
mtime_ns: int,
|
||||
now: str,
|
||||
) -> tuple[str, int, bool, bool]:
|
||||
"""The byte record for these bytes, refreshed or created.
|
||||
|
||||
Returns ``(id, tag_revision, created, refingerprinted)``; the last is
|
||||
True when an existing record's verified ``(size, mtime_ns)`` changed,
|
||||
which moves its currency for cheap reads (the caller bumps the index
|
||||
revision when nothing else does).
|
||||
"""
|
||||
row = conn.execute(_BYTE, (file_id, sha256)).fetchone()
|
||||
if row is None:
|
||||
byte_id = _new_id()
|
||||
conn.execute(
|
||||
_INSERT_BYTE, (byte_id, file_id, sha256, size, mtime_ns, now, now)
|
||||
)
|
||||
return byte_id, 0, True, False
|
||||
conn.execute(_REFRESH_BYTE, (size, mtime_ns, now, row[0]))
|
||||
return row[0], row[1], False, (row[2], row[3]) != (size, mtime_ns)
|
||||
|
||||
@staticmethod
|
||||
def _byte_tags(conn: sqlite3.Connection, byte_id: str) -> list[str]:
|
||||
return [row[0] for row in conn.execute(_BYTE_TAGS, (byte_id,))]
|
||||
|
||||
def _write_image_tags(
|
||||
self, conn: sqlite3.Connection, byte_id: str, tags: list[str]
|
||||
) -> int:
|
||||
"""Rewrite the junction in order, bump both revisions; return the new one."""
|
||||
tag_ids = self._ensure_tags(conn, tags)
|
||||
conn.execute(_CLEAR_TAGS, (byte_id,))
|
||||
conn.executemany(
|
||||
_INSERT_TAG,
|
||||
[(byte_id, tag_ids[name], position) for position, name in enumerate(tags)],
|
||||
)
|
||||
conn.execute(_BUMP_REVISION, (byte_id,))
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return conn.execute(_REVISION, (byte_id,)).fetchone()[0]
|
||||
|
||||
def _current_set(self, conn, sha256, byte_id, revision) -> dict:
|
||||
return {
|
||||
"sha256": sha256,
|
||||
"tags": self._byte_tags(conn, byte_id),
|
||||
"tag_revision": revision,
|
||||
}
|
||||
|
||||
def read_image_tags(self, locator: str, sha256: str) -> dict:
|
||||
"""The stored set of these exact bytes, or an empty set at revision 0.
|
||||
|
||||
``prior_count`` is the number of other byte records at the locator
|
||||
(earlier or later bytes of the same file).
|
||||
"""
|
||||
_validate_locator(locator)
|
||||
_validate_digest(sha256)
|
||||
with self._annotation_read() as conn:
|
||||
row = conn.execute(_READ, (sha256, sha256, locator)).fetchone()
|
||||
if row is None:
|
||||
return {"tags": [], "tag_revision": 0, "prior_count": 0}
|
||||
tags = self._byte_tags(conn, row[0]) if row[0] is not None else []
|
||||
return {"tags": tags, "tag_revision": row[1] or 0, "prior_count": row[2]}
|
||||
|
||||
def replace_image_tags(
|
||||
self,
|
||||
*,
|
||||
locator: str,
|
||||
sha256: str,
|
||||
size: int,
|
||||
mtime_ns: int,
|
||||
expected_epoch: str,
|
||||
expected_tag_revision: int,
|
||||
tags: list,
|
||||
) -> dict:
|
||||
"""Replace the set of these exact bytes, guarded by epoch and revision.
|
||||
|
||||
Creates the owner and byte record when the bytes are new (an explicit
|
||||
empty set is a real save). The identical set at the current revision,
|
||||
or one behind it (a lost-response retry), is ``replayed`` and writes
|
||||
nothing; any other revision mismatch is ``tag-conflict`` with the
|
||||
current set and no writes.
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
_validate_int(expected_tag_revision, "expected_tag_revision", 0)
|
||||
tags = normalize_image_tags(tags)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
now = utc_now_iso()
|
||||
file_id = self._annotation_owner(conn, locator, now)
|
||||
byte_id, revision, created, refingerprinted = self._annotation_byte(
|
||||
conn, file_id, sha256, size, mtime_ns, now
|
||||
)
|
||||
current = self._byte_tags(conn, byte_id)
|
||||
if (
|
||||
not created
|
||||
and current == tags
|
||||
and expected_tag_revision in (revision, revision - 1)
|
||||
):
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return _receipt(epoch, sha256, current, revision, True)
|
||||
if expected_tag_revision != revision:
|
||||
raise AnnotationCommandError(
|
||||
"tag-conflict",
|
||||
epoch,
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
revision = self._write_image_tags(conn, byte_id, tags)
|
||||
return _receipt(epoch, sha256, tags, revision, False)
|
||||
|
||||
def add_image_tags(
|
||||
self,
|
||||
*,
|
||||
locator: str,
|
||||
sha256: str,
|
||||
size: int,
|
||||
mtime_ns: int,
|
||||
expected_epoch: str,
|
||||
tags: list,
|
||||
) -> dict:
|
||||
"""Union ``tags`` into the set of these exact bytes, keeping stored spelling.
|
||||
|
||||
``added`` counts the names that were new (case-insensitively). A union
|
||||
over ``MAX_IMAGE_TAGS`` is ``tag-limit`` with the current set and no
|
||||
writes; a union that adds nothing to an existing record is ``replayed``.
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
tags = normalize_image_tags(tags)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
now = utc_now_iso()
|
||||
file_id = self._annotation_owner(conn, locator, now)
|
||||
byte_id, revision, created, refingerprinted = self._annotation_byte(
|
||||
conn, file_id, sha256, size, mtime_ns, now
|
||||
)
|
||||
current = self._byte_tags(conn, byte_id)
|
||||
present = {name.casefold() for name in current}
|
||||
added = [name for name in tags if name.casefold() not in present]
|
||||
merged = current + added
|
||||
if len(merged) > MAX_IMAGE_TAGS:
|
||||
raise AnnotationCommandError(
|
||||
"tag-limit",
|
||||
epoch,
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
if not created and not added:
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return {**_receipt(epoch, sha256, current, revision, True), "added": 0}
|
||||
revision = self._write_image_tags(conn, byte_id, merged)
|
||||
return {
|
||||
**_receipt(epoch, sha256, merged, revision, False),
|
||||
"added": len(added),
|
||||
}
|
||||
|
||||
def refresh_image_fingerprint(
|
||||
self, locator: str, sha256: str, size: int, mtime_ns: int
|
||||
) -> bool:
|
||||
"""Record that these exact bytes were just seen with this stat.
|
||||
|
||||
True when a byte record was refreshed; nothing is created and no tag
|
||||
or tag revision changes. The image index revision moves because the
|
||||
record's currency for cheap reads may have changed.
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute(
|
||||
_REFRESH_BY_LOCATOR, (size, mtime_ns, utc_now_iso(), sha256, locator)
|
||||
)
|
||||
if cursor.rowcount > 0:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
def current_image_tags(self, stats: dict) -> dict:
|
||||
"""Image tags of the files whose current ``(size, mtime_ns)`` is verified.
|
||||
|
||||
``stats`` maps locators to their present fingerprint. A locator is
|
||||
answered (possibly with an empty set) only when a byte record carries
|
||||
exactly that verified fingerprint; the most recently verified record
|
||||
wins when several do. Lookups go in chunks of 500 locators.
|
||||
"""
|
||||
if not isinstance(stats, dict):
|
||||
raise ValueError(
|
||||
"stats must map locators to (size, mtime_ns) integer pairs"
|
||||
)
|
||||
for locator, fingerprint in stats.items():
|
||||
_validate_fingerprint(locator, fingerprint)
|
||||
locators = list(stats)
|
||||
chosen: dict[str, tuple[tuple[str, str], str]] = {}
|
||||
result: dict[str, list[str]] = {}
|
||||
with self._annotation_read() as conn:
|
||||
for start in range(0, len(locators), _LOOKUP_CHUNK):
|
||||
chunk = locators[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
|
||||
_FINGERPRINTS.format(marks=marks), chunk
|
||||
):
|
||||
if (size, mtime_ns) != stats[locator]:
|
||||
continue
|
||||
key = (verified_at, byte_id)
|
||||
if locator not in chosen or key > chosen[locator][0]:
|
||||
chosen[locator] = (key, byte_id)
|
||||
owners = {byte_id: locator for locator, (_key, byte_id) in chosen.items()}
|
||||
result = {locator: [] for locator in chosen}
|
||||
byte_ids = list(owners)
|
||||
for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
|
||||
chunk = byte_ids[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for byte_id, name in conn.execute(_TAGS_OF.format(marks=marks), chunk):
|
||||
result[owners[byte_id]].append(name)
|
||||
return result
|
||||
|
||||
def iter_image_tag_records(self) -> Iterator[tuple[str, int, int, list[str]]]:
|
||||
"""Stream every byte record as ``(locator, size, mtime_ns, tags)``.
|
||||
|
||||
One SELECT ordered by locator, then by the bytes' first sighting, with
|
||||
each set's names in position order.
|
||||
"""
|
||||
conn = self.model.get_connection()
|
||||
current: tuple | None = None
|
||||
for locator, byte_id, size, mtime_ns, name in conn.execute(_RECORDS):
|
||||
if current is None or current[0] != byte_id:
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
current = (byte_id, (locator, size, mtime_ns, []))
|
||||
if name is not None:
|
||||
current[1][3].append(name)
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Records of duplicate image files: count them and move them (#249 S6, T17).
|
||||
|
||||
A removed copy's records move to its verified kept copy before the copy is
|
||||
unlinked (``py/api/duplicate_files.py``), one file per short ``BEGIN
|
||||
IMMEDIATE`` transaction, so no row ever names a deleted file:
|
||||
|
||||
- its image records (``generated_images``), found under every spelling the
|
||||
file is linked by. A record moves to the kept copy, unless its prompt
|
||||
already links the kept copy: then its saved-pair links join that record
|
||||
and it is deleted;
|
||||
- its favorite;
|
||||
- its image tags (``media_image_tags`` of its byte record), joined onto the
|
||||
kept copy's byte record as their union, in order;
|
||||
- its pending Needs review items, settled: only an item still ``pending`` at
|
||||
revision 0 with no decision row is removed (``review._SETTLE_ITEM``), so a
|
||||
decided item is never touched.
|
||||
|
||||
Every statement is an index search (see ``FILE_LINK_SQL``).
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from .annotation_schema import BYTES_TABLE, FILES_TABLE
|
||||
from .annotations import _BUMP_INDEX
|
||||
from .models import normalize_image_path, utc_now_iso
|
||||
from .review import _BUMP_REVIEW_INDEX, _PENDING_OF_LOCATOR, _SETTLE_ITEM
|
||||
from .review_changes import notify_review_change
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
LINKS_BY_FILE_SQL = "SELECT id, prompt_id FROM generated_images WHERE file_path = ?"
|
||||
LEGACY_LINKS_SQL = (
|
||||
"SELECT id, prompt_id FROM generated_images "
|
||||
"WHERE image_path = ? AND file_path IS NULL"
|
||||
)
|
||||
KEEPER_LINK_SQL = (
|
||||
"SELECT id FROM generated_images WHERE prompt_id = ? AND file_path = ?"
|
||||
)
|
||||
MOVE_PAIR_LINKS_SQL = (
|
||||
"UPDATE OR IGNORE image_prompt_pairs SET image_id = ? WHERE image_id = ?"
|
||||
)
|
||||
MOVE_LINK_SQL = (
|
||||
"UPDATE generated_images SET image_path = ?, filename = ?, file_path = ? "
|
||||
"WHERE id = ?"
|
||||
)
|
||||
DELETE_LINK_SQL = "DELETE FROM generated_images WHERE id = ?"
|
||||
FAVORITE_SQL = (
|
||||
"SELECT created_at FROM image_favorites WHERE root_key = ? AND relative_path = ?"
|
||||
)
|
||||
INSERT_FAVORITE_SQL = "INSERT OR IGNORE INTO image_favorites VALUES (?, ?, ?)"
|
||||
DELETE_FAVORITE_SQL = (
|
||||
"DELETE FROM image_favorites WHERE root_key = ? AND relative_path = ?"
|
||||
)
|
||||
COPY_BYTE_SQL = (
|
||||
f"SELECT b.id FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b " # nosec B608
|
||||
"ON b.file_id=f.id WHERE f.locator=? AND b.sha256=?"
|
||||
)
|
||||
COPY_TAG_COUNT_SQL = (
|
||||
"SELECT COUNT(*) FROM media_image_tags WHERE byte_id=?" # nosec B608
|
||||
)
|
||||
PENDING_COUNT_SQL = (
|
||||
f"SELECT COUNT(*) FROM {FILES_TABLE} f " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
"JOIN media_review_items i ON i.byte_id=b.id "
|
||||
"WHERE f.locator=? AND +i.state='pending' AND i.revision=0 "
|
||||
"AND NOT EXISTS (SELECT 1 FROM media_review_decisions d WHERE d.item_id=i.id)"
|
||||
)
|
||||
FILE_LINK_SQL = (
|
||||
LINKS_BY_FILE_SQL,
|
||||
LEGACY_LINKS_SQL,
|
||||
KEEPER_LINK_SQL,
|
||||
MOVE_PAIR_LINKS_SQL,
|
||||
MOVE_LINK_SQL,
|
||||
DELETE_LINK_SQL,
|
||||
FAVORITE_SQL,
|
||||
DELETE_FAVORITE_SQL,
|
||||
COPY_BYTE_SQL,
|
||||
COPY_TAG_COUNT_SQL,
|
||||
PENDING_COUNT_SQL,
|
||||
)
|
||||
|
||||
|
||||
def _file_link_rows(conn, spelled):
|
||||
"""``(id, prompt_id)`` of the image records linking one spelling."""
|
||||
rows = list(conn.execute(LINKS_BY_FILE_SQL, (normalize_image_path(spelled),)))
|
||||
rows += list(conn.execute(LEGACY_LINKS_SQL, (spelled,)))
|
||||
return [tuple(row) for row in rows]
|
||||
|
||||
|
||||
def _copy_byte(conn, locator, sha256):
|
||||
row = conn.execute(COPY_BYTE_SQL, (locator, sha256)).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
class DuplicateCopyOperations:
|
||||
"""PromptDatabase mixin: the records of duplicate copies."""
|
||||
|
||||
def count_copy_records(self, copies) -> dict:
|
||||
"""``{path: {links, tags, review}}`` for each copy (one read snapshot).
|
||||
|
||||
``copies`` maps each path to ``{spellings, locator, sha256}``:
|
||||
``links`` counts the image records under any spelling, ``tags`` the
|
||||
image tags of its bytes and ``review`` its pending, undecided Needs
|
||||
review items.
|
||||
"""
|
||||
from .maintenance import _read_snapshot
|
||||
|
||||
def read(conn):
|
||||
found = {}
|
||||
for path, copy in copies.items():
|
||||
ids = {
|
||||
row[0]
|
||||
for spelled in copy["spellings"]
|
||||
for row in _file_link_rows(conn, spelled)
|
||||
}
|
||||
byte_id = _copy_byte(conn, copy["locator"], copy["sha256"])
|
||||
tags = (
|
||||
conn.execute(COPY_TAG_COUNT_SQL, (byte_id,)).fetchone()[0]
|
||||
if byte_id
|
||||
else 0
|
||||
)
|
||||
review = conn.execute(PENDING_COUNT_SQL, (copy["locator"],))
|
||||
found[path] = {
|
||||
"links": len(ids),
|
||||
"tags": tags,
|
||||
"review": review.fetchone()[0],
|
||||
}
|
||||
return found
|
||||
|
||||
return _read_snapshot(self.model.db_path, read)
|
||||
|
||||
def move_copy_records(self, move) -> dict:
|
||||
"""Move one removed copy's records to its kept copy (one transaction).
|
||||
|
||||
``move``: ``removed`` and ``kept`` (the spellings each file is linked
|
||||
by), ``targets`` (``{removed spelling: kept spelling}``, the spelling
|
||||
a moved record takes; else ``kept_path``), ``favorites`` (``(root key,
|
||||
removed key, kept root key, kept key)`` per listed root holding the
|
||||
file), ``removed_locator``,
|
||||
``kept_locator``, ``sha256`` and the kept copy's ``kept_size`` and
|
||||
``kept_mtime_ns``. Returns ``{moved, merged, favorites, tags,
|
||||
review}``.
|
||||
"""
|
||||
counts = dict.fromkeys(("moved", "merged", "favorites", "tags", "review"), 0)
|
||||
kept_keys = [normalize_image_path(path) for path in move["kept"]]
|
||||
with self._pair_transaction() as conn:
|
||||
for spelled in dict.fromkeys(move["removed"]):
|
||||
target = move["targets"].get(spelled, move["kept_path"])
|
||||
keys = list(dict.fromkeys(kept_keys + [normalize_image_path(target)]))
|
||||
for image_id, prompt_id in _file_link_rows(conn, spelled):
|
||||
existing = None
|
||||
for key in keys:
|
||||
existing = conn.execute(
|
||||
KEEPER_LINK_SQL, (prompt_id, key)
|
||||
).fetchone()
|
||||
if existing is not None:
|
||||
break
|
||||
if existing is not None:
|
||||
conn.execute(MOVE_PAIR_LINKS_SQL, (existing[0], image_id))
|
||||
documents = documents_of_rows(conn, "id", [image_id])
|
||||
conn.execute(DELETE_LINK_SQL, (image_id,))
|
||||
release_documents(conn, documents)
|
||||
counts["merged"] += 1
|
||||
continue
|
||||
conn.execute(
|
||||
MOVE_LINK_SQL,
|
||||
(
|
||||
target,
|
||||
os.path.basename(target),
|
||||
normalize_image_path(target),
|
||||
image_id,
|
||||
),
|
||||
)
|
||||
counts["moved"] += 1
|
||||
for favorite in move.get("favorites", ()):
|
||||
counts["favorites"] += _move_favorite(conn, *favorite)
|
||||
counts["tags"] = self._move_copy_tags(conn, move)
|
||||
for (item_id,) in conn.execute(
|
||||
_PENDING_OF_LOCATOR, (move["removed_locator"],)
|
||||
).fetchall():
|
||||
counts["review"] += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
||||
if counts["review"]:
|
||||
conn.execute(_BUMP_REVIEW_INDEX)
|
||||
if counts["review"]:
|
||||
notify_review_change(self.model.db_path)
|
||||
return counts
|
||||
|
||||
def _move_copy_tags(self, conn, move):
|
||||
"""Join the removed copy's image tags onto the kept copy's; the count added."""
|
||||
removed = _copy_byte(conn, move["removed_locator"], move["sha256"])
|
||||
if removed is None:
|
||||
return 0
|
||||
tags = self._byte_tags(conn, removed)
|
||||
if not tags:
|
||||
return 0
|
||||
now = utc_now_iso()
|
||||
file_id = self._annotation_owner(conn, move["kept_locator"], now)
|
||||
byte_id, _revision, created, moved = self._annotation_byte(
|
||||
conn, file_id, move["sha256"], move["kept_size"], move["kept_mtime_ns"], now
|
||||
)
|
||||
current = self._byte_tags(conn, byte_id)
|
||||
union = current + [tag for tag in tags if tag not in current]
|
||||
if union != current:
|
||||
self._write_image_tags(conn, byte_id, union)
|
||||
elif created or moved:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return len(union) - len(current)
|
||||
|
||||
|
||||
def _move_favorite(conn, root_key, removed_key, kept_root, kept_key):
|
||||
row = conn.execute(FAVORITE_SQL, (root_key, removed_key)).fetchone()
|
||||
if row is None:
|
||||
return 0
|
||||
conn.execute(INSERT_FAVORITE_SQL, (kept_root, kept_key, row[0]))
|
||||
conn.execute(DELETE_FAVORITE_SQL, (root_key, removed_key))
|
||||
return 1
|
||||
@@ -196,3 +196,14 @@ class ExecutionHistoryOperations:
|
||||
.fetchone()
|
||||
)
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
def rekey_history_restore(old_path: str, new_path: str) -> None:
|
||||
"""Move the observers of ``old_path`` to ``new_path`` (Move library)."""
|
||||
old, new = _database_identity(old_path), _database_identity(new_path)
|
||||
if old == new:
|
||||
return
|
||||
with _restore_lock:
|
||||
observers = _restore_observers.pop(old, None)
|
||||
if observers:
|
||||
_restore_observers.setdefault(new, {}).update(observers)
|
||||
|
||||
@@ -0,0 +1,380 @@
|
||||
"""The read-only health check behind System › Overview and Maintenance.
|
||||
|
||||
``run_health_checks`` opens its own ``mode=ro`` connection, reads one
|
||||
snapshot and closes the connection in ``finally``; it never writes. Each
|
||||
check reports ``passed``, ``issues`` (with an exact ``count``; the
|
||||
integrity check stops at ``INTEGRITY_LIMIT`` problems and then says
|
||||
``capped``),
|
||||
``unavailable`` (it could not be checked here, so ``count`` is None, or
|
||||
``missing_files`` checked only some sources and found nothing) or ``failed``
|
||||
(it raised; the other checks still run). ``missing_files`` also reports each
|
||||
source and whether the check was ``partial``; it reads the image rows inside
|
||||
the snapshot and checks their files after the snapshot ended, so a slow source
|
||||
never holds the WAL. At most ``MAX_DETAIL`` detail strings are returned per
|
||||
check, and none of them carries a server path.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import ntpath
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import PurePosixPath, PureWindowsPath
|
||||
from urllib.parse import quote
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
from .models import normalize_image_path
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from database.models import normalize_image_path
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.health")
|
||||
|
||||
MAX_DETAIL = 10 # detail strings returned per check
|
||||
INTEGRITY_LIMIT = 100 # PRAGMA integrity_check(N): problems SQLite reports
|
||||
BATCH_ROWS = 100 # image rows read per fetch
|
||||
REVIEW_UNAVAILABLE = "Needs review is not available yet."
|
||||
CHECK_FAILED = "The check could not finish."
|
||||
# Check ids and the labels of the concept's health report, in its order;
|
||||
# pair uniqueness and consistency are the added rows (D63, now D95).
|
||||
CHECKS = (
|
||||
("integrity", "Integrity check"),
|
||||
("foreign_keys", "Foreign keys & pair references"),
|
||||
("canonical_hashes", "Canonical hashes"),
|
||||
("pair_uniqueness", "Pair uniqueness"),
|
||||
("missing_files", "Missing files"),
|
||||
("ambiguous_metadata", "Ambiguous metadata"),
|
||||
("consistency", "Consistency"),
|
||||
)
|
||||
_DATABASE_CHECKS = (
|
||||
"integrity",
|
||||
"foreign_keys",
|
||||
"canonical_hashes",
|
||||
"pair_uniqueness",
|
||||
"missing_files",
|
||||
"consistency",
|
||||
)
|
||||
|
||||
|
||||
def iter_consistency_issues(conn):
|
||||
"""The legacy consistency issues: tag and image rows of a missing prompt."""
|
||||
for ref in conn.execute(
|
||||
"SELECT pt.prompt_id FROM prompt_tags pt "
|
||||
"LEFT JOIN prompts p ON pt.prompt_id = p.id WHERE p.id IS NULL"
|
||||
):
|
||||
yield f"prompt_tags entry references non-existent prompt {ref[0]}"
|
||||
for ref in conn.execute(
|
||||
"SELECT gi.id, gi.prompt_id FROM generated_images gi "
|
||||
"LEFT JOIN prompts p ON gi.prompt_id = p.id WHERE p.id IS NULL"
|
||||
):
|
||||
yield f"Image {ref[0]} references non-existent prompt {ref[1]}"
|
||||
|
||||
|
||||
def _result(count, detail, *, status=None):
|
||||
return {
|
||||
"status": status or ("issues" if count else "passed"),
|
||||
"count": count,
|
||||
"detail": detail[:MAX_DETAIL],
|
||||
}
|
||||
|
||||
|
||||
def _counted(lines):
|
||||
"""(count, first MAX_DETAIL lines) of an iterable, consumed once."""
|
||||
count, detail = 0, []
|
||||
for line in lines:
|
||||
count += 1
|
||||
if len(detail) < MAX_DETAIL:
|
||||
detail.append(line)
|
||||
return _result(count, detail)
|
||||
|
||||
|
||||
def readonly_uri(db_path, *, windows=None):
|
||||
"""The ``mode=ro`` SQLite URI of *db_path*, on this platform or the given one.
|
||||
|
||||
pathlib spells a UNC path ``\\\\server\\share\\x.db`` as
|
||||
``file://server/share/x.db``, a remote authority SQLite refuses to open.
|
||||
With an empty authority (``file:////server/share/x.db``) SQLite hands
|
||||
Windows the ``//server/share`` path, which it opens as the UNC path.
|
||||
"""
|
||||
if windows is None:
|
||||
windows = os.name == "nt"
|
||||
db_path = os.path.abspath(db_path)
|
||||
if not windows:
|
||||
return PurePosixPath(db_path).as_uri() + "?mode=ro"
|
||||
path = PureWindowsPath(db_path)
|
||||
if ntpath.splitdrive(str(path))[0].startswith("\\\\"):
|
||||
return "file:" + quote("//" + path.as_posix()) + "?mode=ro"
|
||||
return path.as_uri() + "?mode=ro"
|
||||
|
||||
|
||||
def _integrity(conn, _roots):
|
||||
rows = [
|
||||
row[0] for row in conn.execute(f"PRAGMA integrity_check({INTEGRITY_LIMIT})")
|
||||
]
|
||||
if rows == ["ok"]:
|
||||
return {**_result(0, []), "capped": False}
|
||||
# SQLite stops listing at the limit: there may be more problems than that.
|
||||
return {**_result(len(rows), rows), "capped": len(rows) >= INTEGRITY_LIMIT}
|
||||
|
||||
|
||||
def _foreign_keys(conn, _roots):
|
||||
return _counted(
|
||||
f"{table} row {rowid} references a missing {parent} row"
|
||||
for table, rowid, parent, _fkid in conn.execute("PRAGMA foreign_key_check")
|
||||
)
|
||||
|
||||
|
||||
def _prompt_hash(text):
|
||||
return generate_prompt_hash(text) if isinstance(text, str) else None
|
||||
|
||||
|
||||
def _canonical_hashes(conn, _roots):
|
||||
conn.create_function("pm_prompt_hash", 1, _prompt_hash, deterministic=True)
|
||||
return _counted(
|
||||
f"Prompt {row[0]}: the stored hash does not match its text"
|
||||
for row in conn.execute(
|
||||
"SELECT id FROM prompts WHERE hash IS NOT pm_prompt_hash(text) ORDER BY id"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _pair_uniqueness(conn, _roots):
|
||||
groups = conn.execute(
|
||||
"SELECT positive_prompt_id,negative_prompt_id,COUNT(*) FROM prompt_pairs "
|
||||
"GROUP BY positive_prompt_id,negative_prompt_id HAVING COUNT(*)>1 "
|
||||
"ORDER BY positive_prompt_id,negative_prompt_id"
|
||||
).fetchall()
|
||||
detail = [
|
||||
f"Positive prompt {positive} with "
|
||||
+ ("no negative" if negative is None else f"negative {negative}")
|
||||
+ f": {rows} pair rows"
|
||||
for positive, negative, rows in groups
|
||||
]
|
||||
# The surplus rows: what keeping one row per pair would leave out.
|
||||
return _result(sum(rows - 1 for _p, _n, rows in groups), detail)
|
||||
|
||||
|
||||
def _consistency(conn, _roots):
|
||||
return _counted(iter_consistency_issues(conn))
|
||||
|
||||
|
||||
def _readable(root):
|
||||
return os.path.isdir(root) and os.access(root, os.R_OK | os.X_OK)
|
||||
|
||||
|
||||
def _root_prefix(root):
|
||||
return normalize_image_path(root).rstrip("\\/") + os.sep
|
||||
|
||||
|
||||
def _source_name(root, index):
|
||||
"""A source's folder name for the report (never its server path)."""
|
||||
return os.path.basename(str(root).rstrip("\\/")) or f"Source {index + 1}"
|
||||
|
||||
|
||||
def _owner(path, prefixes):
|
||||
"""Index of the longest source prefix *path* lies under, or None."""
|
||||
owner, longest = None, -1
|
||||
for index, prefix in enumerate(prefixes):
|
||||
if path.startswith(prefix) and len(prefix) > longest:
|
||||
owner, longest = index, len(prefix)
|
||||
return owner
|
||||
|
||||
|
||||
def _image_owner(image_path, file_path, prefixes):
|
||||
spelled = file_path or normalize_image_path(image_path)
|
||||
owner = _owner(spelled, prefixes)
|
||||
if owner is None:
|
||||
owner = _owner(normalize_image_path(os.path.realpath(image_path)), prefixes)
|
||||
return owner
|
||||
|
||||
|
||||
def readable_roots(roots):
|
||||
"""Whether each source can be read now (an unmounted drive cannot)."""
|
||||
return [_readable(root) for root in roots]
|
||||
|
||||
|
||||
def link_states(rows, roots, readable):
|
||||
"""``(row, owner, state)`` for each image row ``(id, image_path, filename,
|
||||
file_path)``: ``owner`` is the index of the source it lies under (None
|
||||
outside every source) and ``state`` is ``missing``, ``present``,
|
||||
``unavailable`` (its source cannot be read, so the file is not looked at)
|
||||
or ``outside``. ``readable`` is ``readable_roots(roots)``. The health
|
||||
check's ``missing_files`` and the maintenance missing-link plan both
|
||||
judge links this way, so their counts agree.
|
||||
"""
|
||||
prefixes = [_root_prefix(root) for root in roots]
|
||||
for row in rows:
|
||||
owner = _image_owner(row[1], row[3], prefixes)
|
||||
if owner is None:
|
||||
yield row, None, "outside"
|
||||
elif not readable[owner]:
|
||||
yield row, owner, "unavailable"
|
||||
elif os.path.exists(row[1]):
|
||||
yield row, owner, "present"
|
||||
else:
|
||||
yield row, owner, "missing"
|
||||
|
||||
|
||||
def _unavailable_line(unavailable, *, partial):
|
||||
noun = "source is" if unavailable == 1 else "sources are"
|
||||
effect = "its files were" if unavailable == 1 else "their files were"
|
||||
if not partial:
|
||||
effect = "missing files were"
|
||||
return f"{unavailable} image {noun} unavailable, so {effect} not counted."
|
||||
|
||||
|
||||
def _missing_files(conn, roots):
|
||||
"""Links whose file is gone, counted per source (only readable ones).
|
||||
|
||||
Each source reports ``checked`` with its own count, or ``unavailable``
|
||||
(an unmounted drive) with no count: its links are never turned into a
|
||||
count, while the readable sources still are. The check's ``count`` is the
|
||||
readable sources' total (None when none is readable) and ``partial`` says
|
||||
some sources were not checked. Links outside every source are named in the
|
||||
detail, not counted.
|
||||
|
||||
Inside the snapshot this only reads the image rows; it returns the step
|
||||
that checks their files, which runs after the snapshot ended.
|
||||
"""
|
||||
if not roots:
|
||||
result = _result(None, ["No image source is available."], status="unavailable")
|
||||
return {**result, "partial": False, "sources": []}
|
||||
rows = []
|
||||
cursor = conn.execute(
|
||||
"SELECT id,image_path,filename,file_path FROM generated_images ORDER BY id"
|
||||
)
|
||||
while batch := cursor.fetchmany(BATCH_ROWS):
|
||||
rows.extend(batch)
|
||||
return lambda: _check_files(rows, roots)
|
||||
|
||||
|
||||
def _check_files(rows, roots):
|
||||
"""The ``missing_files`` result of the image *rows*, from the file system."""
|
||||
readable = readable_roots(roots)
|
||||
sources = [
|
||||
{
|
||||
"name": _source_name(root, index),
|
||||
"status": "checked" if ok else "unavailable",
|
||||
"count": 0 if ok else None,
|
||||
}
|
||||
for index, (root, ok) in enumerate(zip(roots, readable))
|
||||
]
|
||||
unavailable = readable.count(False)
|
||||
if not any(readable):
|
||||
result = _result(
|
||||
None, [_unavailable_line(unavailable, partial=False)], status="unavailable"
|
||||
)
|
||||
return {**result, "partial": False, "sources": sources}
|
||||
outside, missing = 0, []
|
||||
for row, owner, state in link_states(rows, roots, readable):
|
||||
if state == "outside":
|
||||
outside += 1
|
||||
elif state == "missing":
|
||||
image_id, image_path, filename, _file_path = row
|
||||
sources[owner]["count"] += 1
|
||||
if len(missing) < MAX_DETAIL:
|
||||
name = filename or os.path.basename(image_path)
|
||||
missing.append(f"Image {image_id}: {name} is missing")
|
||||
count = sum(source["count"] or 0 for source in sources)
|
||||
detail = [_unavailable_line(unavailable, partial=True)] if unavailable else []
|
||||
detail += missing
|
||||
if outside:
|
||||
links = "link" if outside == 1 else "links"
|
||||
verb = "was" if outside == 1 else "were"
|
||||
detail.append(
|
||||
f"{outside} {links} outside the image sources {verb} not checked."
|
||||
)
|
||||
status = None if count or not unavailable else "unavailable"
|
||||
result = _result(count, detail, status=status)
|
||||
return {**result, "partial": bool(unavailable), "sources": sources}
|
||||
|
||||
|
||||
def _ambiguous(review_count):
|
||||
if review_count is None:
|
||||
return _result(None, [REVIEW_UNAVAILABLE], status="unavailable")
|
||||
return _result(review_count, [])
|
||||
|
||||
|
||||
def overall_status(checks) -> str:
|
||||
"""``passed`` only when every check passed; any check not run is incomplete."""
|
||||
statuses = {check["status"] for check in checks}
|
||||
if statuses & {"unavailable", "failed"} or any(
|
||||
check.get("partial") for check in checks
|
||||
):
|
||||
return "incomplete"
|
||||
return "issues" if "issues" in statuses else "passed"
|
||||
|
||||
|
||||
def _failed():
|
||||
return _result(None, [CHECK_FAILED], status="failed")
|
||||
|
||||
|
||||
def _run_check(check_id, step, *args):
|
||||
"""One check's result, or None (reported as failed) when it raised."""
|
||||
try:
|
||||
return step(*args)
|
||||
except Exception: # any error fails this check only; the others still run
|
||||
logger.exception(f"Health check {check_id} failed")
|
||||
return None
|
||||
|
||||
|
||||
def _run_database_checks(db_path, roots):
|
||||
"""Each database check's result (None when it raised), from one snapshot.
|
||||
|
||||
A check may return a step instead of its result: the step runs after the
|
||||
snapshot ended and the connection closed (``missing_files``' file checks).
|
||||
"""
|
||||
results = dict.fromkeys(_DATABASE_CHECKS)
|
||||
try:
|
||||
conn = sqlite3.connect(readonly_uri(db_path), uri=True)
|
||||
except sqlite3.Error:
|
||||
logger.exception("Health check could not open the database")
|
||||
return results
|
||||
try:
|
||||
conn.execute("BEGIN") # one read snapshot for every check
|
||||
for check_id in _DATABASE_CHECKS:
|
||||
runner = globals()["_" + check_id] # looked up per run (tests patch it)
|
||||
results[check_id] = _run_check(check_id, runner, conn, roots)
|
||||
except sqlite3.Error:
|
||||
logger.exception("Health check could not read the database")
|
||||
finally:
|
||||
conn.close()
|
||||
for check_id, result in results.items():
|
||||
if callable(result):
|
||||
results[check_id] = _run_check(check_id, result)
|
||||
return results
|
||||
|
||||
|
||||
class HealthOperations:
|
||||
"""Mixin for PromptDatabase: the read-only System health check."""
|
||||
|
||||
def run_health_checks(self, roots, *, review_count=None) -> dict:
|
||||
"""Run every check and return a ``HealthReport``.
|
||||
|
||||
Args:
|
||||
roots: The image source directories to judge missing files under:
|
||||
the effective sources plus any configured source that is not
|
||||
available (reported as an unavailable source of ``missing_files``).
|
||||
review_count: The Needs-review pending count, or None while no
|
||||
review queue exists (``ambiguous_metadata`` is then unavailable).
|
||||
"""
|
||||
if review_count is not None and (
|
||||
type(review_count) is not int or review_count < 0
|
||||
):
|
||||
raise ValueError("review_count must be a non-negative integer or None")
|
||||
roots = [str(root) for root in roots]
|
||||
database = _run_database_checks(self.model.db_path, roots)
|
||||
checks = []
|
||||
for check_id, label in CHECKS:
|
||||
if check_id == "ambiguous_metadata":
|
||||
result = _ambiguous(review_count)
|
||||
else:
|
||||
result = database[check_id] or _failed()
|
||||
checks.append({"id": check_id, "label": label, **result})
|
||||
return {
|
||||
"success": True,
|
||||
"checked_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"status": overall_status(checks),
|
||||
"checks": checks,
|
||||
}
|
||||
@@ -628,3 +628,57 @@ class LibraryOperations:
|
||||
"kind": "prompt-manager-library",
|
||||
"positives": [owners[key] for key in order],
|
||||
}
|
||||
|
||||
def library_counts(self) -> dict:
|
||||
"""System › Overview counts from one snapshot, writing nothing.
|
||||
|
||||
``positive_prompts`` and ``negative_variants`` are the unfiltered
|
||||
Library's ``total`` and ``variant_total``: eligible parents and active
|
||||
pairs (every pair's positive is eligible). ``images`` counts indexed
|
||||
``generated_images`` rows, not files on disk. An image has no rating
|
||||
of its own (it shows its linked prompt's), so ``rated_images`` counts
|
||||
the indexed images whose linked prompt is rated. ``review_pending`` is
|
||||
the Needs review queue's pending total.
|
||||
"""
|
||||
with self.library_snapshot() as (conn, _epoch, _ceiling):
|
||||
row = conn.execute(
|
||||
"SELECT (SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE + "),"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs WHERE archived=0),"
|
||||
"(SELECT COUNT(*) FROM generated_images),"
|
||||
"(SELECT COUNT(*) FROM generated_images gi JOIN prompts r "
|
||||
"ON r.id=gi.prompt_id WHERE r.rating IS NOT NULL),"
|
||||
"(SELECT COUNT(*) FROM media_review_items WHERE state='pending')"
|
||||
).fetchone()
|
||||
return {
|
||||
"positive_prompts": row[0],
|
||||
"negative_variants": row[1],
|
||||
"images": row[2],
|
||||
"rated_images": row[3],
|
||||
"review_pending": row[4],
|
||||
}
|
||||
|
||||
def usage_by_category(self) -> list[dict]:
|
||||
"""Runs and average rating of the Library's parents, per category.
|
||||
|
||||
Only eligible parents count, so a negative-only prompt's runs (which
|
||||
repeat its pairs' runs) are never added twice. A blank category groups
|
||||
with no category as ``None``. Unrated parents are left out of the
|
||||
average, which is ``None`` when no parent is rated. Most runs first.
|
||||
"""
|
||||
with self.library_snapshot() as (conn, _epoch, _ceiling):
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM (SELECT NULLIF(TRIM(p.category),'') AS category,"
|
||||
"COALESCE(SUM(p.run_count),0) AS runs,AVG(p.rating) AS rating "
|
||||
"FROM prompts p WHERE " + _ELIGIBLE + " GROUP BY 1) "
|
||||
"ORDER BY runs DESC,category IS NULL,category"
|
||||
).fetchall()
|
||||
return [
|
||||
{
|
||||
"category": row["category"],
|
||||
"runs": row["runs"],
|
||||
"average_rating": (
|
||||
None if row["rating"] is None else round(row["rating"], 2)
|
||||
),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
@@ -0,0 +1,805 @@
|
||||
"""Move library (#249 S6b): copy the node-folder library, verify it, switch.
|
||||
|
||||
``plan_move`` checks a ``utils.library_location`` location can move and
|
||||
sizes the copy; ``move_library`` does it, in this order:
|
||||
|
||||
1. **Disk check** on the target's volume (the copy, the backups and
|
||||
config.json, plus a margin). The source is never renamed across folders,
|
||||
so another drive is simply a copy.
|
||||
2. **Copy** with SQLite's backup API from a private connection holding one
|
||||
read transaction: the per-table counts, the workflow documents' digest
|
||||
and ``PRAGMA data_version`` describe exactly the copied snapshot. The copy
|
||||
is a temp file in the target folder, made self-contained
|
||||
(``journal_mode=DELETE``), fsynced and owner-only.
|
||||
3. **Verify**: ``verify_database_file`` (integrity check, schema, every
|
||||
workflow document decompressed against its hash, #315), the same row
|
||||
count in every table and the same documents.
|
||||
4. **Copy files**: every pooled safety backup and pre-migration backup,
|
||||
checked by size and SHA-256 after the copy (their times are kept, so the
|
||||
pool prunes in the same order).
|
||||
5. **Publish** under the old database's write lock (``BEGIN IMMEDIATE``) and
|
||||
the config.json lock: if any other connection wrote since the snapshot
|
||||
(``data_version``), steps 2–4 run again (``attempts``, then
|
||||
``library-busy``). Else config.json is copied (owner-only, checked), a
|
||||
``pending`` marker is written, the temp files take their final names
|
||||
(``os.replace`` inside the target folder), the marker turns ``moved``
|
||||
and ``switch`` points the process at the new library while the old file
|
||||
is still locked, so no write can land in it.
|
||||
6. **Retire**: the old file's WAL is checkpointed and the old ``prompts.db``
|
||||
and ``config.json`` are renamed ``*.moved-to-<folder>`` (never deleted;
|
||||
the old backups stay where they are). A file still open elsewhere is left
|
||||
in place and reported (``old-in-use``): the marker already makes a restart
|
||||
use the new library.
|
||||
|
||||
Any failure before the marker says ``moved`` removes everything the move
|
||||
wrote in the target folder and leaves the old library in use, untouched.
|
||||
A crash at any point leaves either the old library in use (the marker is
|
||||
absent or ``pending``; the next move replaces its own leftovers) or the new
|
||||
one (``moved``). ``MoveFailed`` carries a code and a message without paths.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import time
|
||||
import types
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from .models import BUSY_TIMEOUT_SECONDS, PromptModel
|
||||
|
||||
try:
|
||||
from ..utils import library_location as ll
|
||||
from ..utils.logging_config import get_logger
|
||||
except ImportError:
|
||||
from utils import library_location as ll
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("prompt_manager.database.library_move")
|
||||
|
||||
TEMP_PREFIX = ".pm-move-"
|
||||
MOVED_INFIX = ".moved-to-"
|
||||
DISK_MARGIN_BYTES = 16 * 2**20
|
||||
DEFAULT_ATTEMPTS = 3
|
||||
_COPY_CHUNK = 4 * 2**20
|
||||
# Backups that live beside the database and move with it.
|
||||
BACKUP_INFIXES = (".backup_", ".pre-workflow-storage_")
|
||||
|
||||
MESSAGES = {
|
||||
"not-movable": (
|
||||
"This library is not in the node folder, so there is nothing to move."
|
||||
),
|
||||
"same-folder": "The new folder is the node folder itself.",
|
||||
"target-invalid": "The new folder cannot hold the library.",
|
||||
"target-has-library": (
|
||||
"The new folder already holds a Prompt Manager library. Nothing was moved."
|
||||
),
|
||||
"target-conflict": (
|
||||
"The new folder already holds files with the same names. Nothing was moved."
|
||||
),
|
||||
"insufficient-space": (
|
||||
"There is not enough free space for the copy. Nothing was moved."
|
||||
),
|
||||
"copy-failed": "The copy could not be written or did not match. Nothing was moved.",
|
||||
"verify-failed": "The copy did not pass verification. Nothing was moved.",
|
||||
"library-busy": (
|
||||
"Prompt Manager kept writing to the library while it was copied. "
|
||||
"Nothing was moved; try again when no workflow is running."
|
||||
),
|
||||
"switch-failed": (
|
||||
"Prompt Manager could not switch to the verified copy, so the copy was "
|
||||
"removed. Nothing was moved."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class MoveFailed(Exception):
|
||||
"""A move that did not happen (or did not finish), with a coded reason."""
|
||||
|
||||
def __init__(self, code: str, message: Optional[str] = None):
|
||||
self.code = code
|
||||
self.message = message or MESSAGES.get(code, "The library was not moved.")
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MovePlan:
|
||||
location: "ll.LibraryLocation"
|
||||
db_path: str
|
||||
source_dir: str
|
||||
target_dir: str
|
||||
config_path: Optional[str]
|
||||
backups: Tuple[str, ...]
|
||||
db_bytes: int
|
||||
files_bytes: int
|
||||
|
||||
@property
|
||||
def required_bytes(self) -> int:
|
||||
return self.db_bytes + self.files_bytes + DISK_MARGIN_BYTES
|
||||
|
||||
@property
|
||||
def target_location(self) -> "ll.LibraryLocation":
|
||||
"""Where the library is once moved (an explicit config.json stays)."""
|
||||
source = self.location.target_source or ll.SOURCE_DEFAULT
|
||||
config = (
|
||||
self.location.config_path
|
||||
if self.location.config_explicit
|
||||
else os.path.join(self.target_dir, ll.CONFIG_NAME)
|
||||
)
|
||||
return ll.LibraryLocation(
|
||||
source,
|
||||
os.path.join(self.target_dir, ll.DATABASE_NAME),
|
||||
config,
|
||||
self.target_dir,
|
||||
self.target_dir,
|
||||
None,
|
||||
source,
|
||||
self.location.config_explicit,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MoveResult:
|
||||
location: "ll.LibraryLocation"
|
||||
counts: Dict[str, int]
|
||||
files: List[str]
|
||||
renamed: List[str] = field(default_factory=list)
|
||||
retire_problem: Optional[str] = None
|
||||
|
||||
|
||||
def _size(path: str) -> int:
|
||||
try:
|
||||
return os.path.getsize(path)
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def _free_bytes(path: str) -> int:
|
||||
return shutil.disk_usage(path).free
|
||||
|
||||
|
||||
def _existing_ancestor(path: str) -> str:
|
||||
current = os.path.abspath(path)
|
||||
while not os.path.isdir(current):
|
||||
parent = os.path.dirname(current)
|
||||
if parent == current:
|
||||
break
|
||||
current = parent
|
||||
return current
|
||||
|
||||
|
||||
def backup_names(db_path: str) -> Tuple[str, ...]:
|
||||
"""The pooled and pre-migration backups beside ``db_path`` (names)."""
|
||||
folder = os.path.dirname(os.path.abspath(db_path))
|
||||
base = os.path.basename(db_path)
|
||||
prefixes = tuple(base + infix for infix in BACKUP_INFIXES)
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
return ()
|
||||
return tuple(
|
||||
sorted(
|
||||
name
|
||||
for name in names
|
||||
if name.startswith(prefixes) and os.path.isfile(os.path.join(folder, name))
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def plan_move(location) -> MovePlan:
|
||||
"""Check ``location`` can move to its target; raises MoveFailed."""
|
||||
if not location.movable:
|
||||
raise MoveFailed("not-movable")
|
||||
target = os.path.abspath(location.target_dir)
|
||||
source_dir = os.path.dirname(os.path.abspath(location.db_path))
|
||||
if ll.same_folder(target, source_dir) or os.path.normcase(
|
||||
os.path.realpath(target)
|
||||
) == os.path.normcase(os.path.realpath(source_dir)):
|
||||
raise MoveFailed("same-folder")
|
||||
if os.path.exists(target) and not os.path.isdir(target):
|
||||
raise MoveFailed("target-invalid")
|
||||
backups = backup_names(location.db_path)
|
||||
# config.json moves with a node-folder library; one named by
|
||||
# PROMPT_MANAGER_CONFIG_PATH is the user's and stays where it is.
|
||||
config = None if location.config_explicit else location.config_path
|
||||
config = config if config and os.path.isfile(config) else None
|
||||
if ll.read_marker(target).get("state") != ll.MARKER_PENDING:
|
||||
if os.path.exists(os.path.join(target, ll.DATABASE_NAME)):
|
||||
raise MoveFailed("target-has-library")
|
||||
names = [ll.CONFIG_NAME] if config else []
|
||||
if any(os.path.exists(os.path.join(target, n)) for n in names + list(backups)):
|
||||
raise MoveFailed("target-conflict")
|
||||
db_bytes = _size(location.db_path) + _size(location.db_path + "-wal")
|
||||
files_bytes = sum(_size(os.path.join(source_dir, n)) for n in backups)
|
||||
files_bytes += _size(config) if config else 0
|
||||
return MovePlan(
|
||||
location,
|
||||
os.path.abspath(location.db_path),
|
||||
source_dir,
|
||||
target,
|
||||
config,
|
||||
backups,
|
||||
db_bytes,
|
||||
files_bytes,
|
||||
)
|
||||
|
||||
|
||||
def space_problem(plan: MovePlan) -> Optional[Tuple[int, int]]:
|
||||
"""``(required, free)`` when the target's volume is short, else None."""
|
||||
free = _free_bytes(_existing_ancestor(plan.target_dir))
|
||||
if free < plan.required_bytes:
|
||||
return plan.required_bytes, free
|
||||
return None
|
||||
|
||||
|
||||
# ── copy and verify ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _tables(conn) -> List[str]:
|
||||
return [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' "
|
||||
"AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _describe(conn) -> Tuple[Dict[str, int], str]:
|
||||
"""Row counts per table and a digest of the workflow documents."""
|
||||
counts = {}
|
||||
for table in _tables(conn):
|
||||
# A name read from sqlite_master, quoted as an identifier.
|
||||
quoted = '"' + table.replace('"', '""') + '"'
|
||||
sql = f"SELECT COUNT(*) FROM {quoted}" # nosec B608
|
||||
counts[table] = conn.execute(sql).fetchone()[0]
|
||||
digest = hashlib.sha256()
|
||||
if "workflow_documents" in counts:
|
||||
for row in conn.execute(
|
||||
"SELECT id, sha256, byte_length FROM workflow_documents ORDER BY id"
|
||||
):
|
||||
digest.update(repr((row[0], bytes(row[1]), row[2])).encode())
|
||||
return counts, digest.hexdigest()
|
||||
|
||||
|
||||
def _fsync_file(path: str) -> None:
|
||||
with open(path, "rb+") as handle:
|
||||
os.fsync(handle.fileno())
|
||||
|
||||
|
||||
def _fsync_dir(path: str) -> None:
|
||||
if os.name != "posix":
|
||||
return
|
||||
fd = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _private(path: str) -> None:
|
||||
if os.name == "posix":
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
|
||||
def _copy_snapshot(source: sqlite3.Connection, temp: str):
|
||||
"""Back up ``source``'s current snapshot to ``temp``; return what it holds."""
|
||||
source.execute("BEGIN")
|
||||
try:
|
||||
version = source.execute("PRAGMA data_version").fetchone()[0]
|
||||
counts, documents = _describe(source)
|
||||
destination = sqlite3.connect(temp)
|
||||
try:
|
||||
source.backup(destination)
|
||||
finally:
|
||||
destination.close()
|
||||
finally:
|
||||
source.execute("COMMIT")
|
||||
with contextlib.closing(sqlite3.connect(temp)) as conn:
|
||||
conn.execute("PRAGMA journal_mode=DELETE").fetchone()
|
||||
_fsync_file(temp)
|
||||
_private(temp)
|
||||
return version, counts, documents
|
||||
|
||||
|
||||
def _verify_copy(temp: str, counts, documents) -> None:
|
||||
ok, reason = PromptModel.verify_database_file(temp)
|
||||
if not ok:
|
||||
logger.error(f"Library move: the copy did not verify: {reason}")
|
||||
detail = reason if reason.startswith("Workflow document") else None
|
||||
raise MoveFailed(
|
||||
"verify-failed",
|
||||
f"{MESSAGES['verify-failed']} {detail}" if detail else None,
|
||||
)
|
||||
uri = Path(os.path.abspath(temp)).as_uri() + "?mode=ro"
|
||||
try:
|
||||
with contextlib.closing(sqlite3.connect(uri, uri=True)) as conn:
|
||||
copied = _describe(conn)
|
||||
except sqlite3.Error as error:
|
||||
logger.error(f"Library move: the copy could not be read: {error}")
|
||||
raise MoveFailed("verify-failed") from error
|
||||
if copied != (counts, documents):
|
||||
logger.error("Library move: the copy's rows or documents differ")
|
||||
raise MoveFailed("verify-failed")
|
||||
|
||||
|
||||
def _hash_copy(source: str, destination: str) -> Tuple[int, str]:
|
||||
"""Copy ``source`` to ``destination`` (fsynced, owner-only, times kept)."""
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with open(source, "rb") as reader, open(destination, "wb") as writer:
|
||||
while True:
|
||||
chunk = reader.read(_COPY_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
writer.write(chunk)
|
||||
writer.flush()
|
||||
os.fsync(writer.fileno())
|
||||
stat = os.stat(source)
|
||||
os.utime(destination, ns=(stat.st_atime_ns, stat.st_mtime_ns))
|
||||
_private(destination)
|
||||
return size, digest.hexdigest()
|
||||
|
||||
|
||||
def _file_digest(path: str) -> Tuple[int, str]:
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with open(path, "rb") as reader:
|
||||
while True:
|
||||
chunk = reader.read(_COPY_CHUNK)
|
||||
if not chunk:
|
||||
return size, digest.hexdigest()
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
|
||||
|
||||
def _check_copies(copies) -> None:
|
||||
for source, temp, expected in copies:
|
||||
if _file_digest(temp) != expected or _file_digest(source) != expected:
|
||||
logger.error(
|
||||
f"Library move: {os.path.basename(source)} did not copy unchanged"
|
||||
)
|
||||
raise MoveFailed("copy-failed")
|
||||
|
||||
|
||||
def write_marker(
|
||||
directory: str, state: str, source_db: str, source_config: Optional[str] = None
|
||||
) -> str:
|
||||
"""Atomically write the move marker; return its path.
|
||||
|
||||
``from`` and ``config_from`` name the old files, so a restart after a
|
||||
crash between the switch and the rename can finish the rename.
|
||||
"""
|
||||
path = os.path.join(directory, ll.MARKER_NAME)
|
||||
temp = os.path.join(directory, f"{TEMP_PREFIX}{uuid.uuid4().hex}.marker")
|
||||
payload = json.dumps(
|
||||
{
|
||||
"version": 1,
|
||||
"state": state,
|
||||
"from": source_db,
|
||||
"config_from": source_config,
|
||||
"updated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
},
|
||||
indent=2,
|
||||
).encode("utf-8")
|
||||
fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
_private(temp)
|
||||
os.replace(temp, path)
|
||||
except BaseException:
|
||||
_remove(temp)
|
||||
raise
|
||||
_fsync_dir(directory)
|
||||
return path
|
||||
|
||||
|
||||
def _remove(*paths) -> None:
|
||||
for path in paths:
|
||||
try:
|
||||
if path and os.path.lexists(path):
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
logger.warning(f"Library move: could not remove {os.path.basename(path)}")
|
||||
|
||||
|
||||
def sweep_move_temps(directory: str) -> List[str]:
|
||||
"""Remove the temp files an interrupted move left in ``directory``."""
|
||||
removed = []
|
||||
try:
|
||||
names = os.listdir(directory)
|
||||
except OSError:
|
||||
return removed
|
||||
for name in names:
|
||||
if name.startswith(TEMP_PREFIX):
|
||||
_remove(os.path.join(directory, name))
|
||||
removed.append(name)
|
||||
return removed
|
||||
|
||||
|
||||
# ── the move ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _noop(*_args, **_kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def switch_process(old_db: str, location, detached: list) -> None:
|
||||
"""Point this process at ``location``: resolver, every model, observers.
|
||||
|
||||
Adds the old file's connections, detached and still open, to
|
||||
``detached`` as soon as they are detached, so the mover retires them
|
||||
(``PromptModel.retire_detached``) whether or not the rest succeeds.
|
||||
"""
|
||||
from .execution_history import rekey_history_restore
|
||||
from .review_changes import rekey_review_changes
|
||||
|
||||
ll.switch_to(location)
|
||||
detached.extend(PromptModel.repoint_instances(old_db, location.db_path))
|
||||
rekey_review_changes(old_db, location.db_path)
|
||||
rekey_history_restore(old_db, location.db_path)
|
||||
|
||||
|
||||
def unswitch_process(old_db: str, new_db: str) -> list:
|
||||
"""Undo ``switch_process`` (any part of it); returns detached connections."""
|
||||
from .execution_history import rekey_history_restore
|
||||
from .review_changes import rekey_review_changes
|
||||
|
||||
ll.clear_override()
|
||||
detached = PromptModel.repoint_instances(new_db, old_db)
|
||||
rekey_review_changes(new_db, old_db)
|
||||
rekey_history_restore(new_db, old_db)
|
||||
return detached
|
||||
|
||||
|
||||
class _Move:
|
||||
def __init__(self, plan, *, switch, config_lock, progress, checkpoint):
|
||||
self.plan = plan
|
||||
self.switch = switch
|
||||
self.config_lock = config_lock
|
||||
self.progress = progress
|
||||
self.checkpoint = checkpoint
|
||||
self.created_dir = False
|
||||
self.temps: List[str] = []
|
||||
self.published: List[str] = []
|
||||
self.marker: Optional[str] = None
|
||||
self.moved = False
|
||||
# Retired connections still in use; awaited once the locks are free.
|
||||
self.retiring: list = []
|
||||
|
||||
def temp(self, suffix: str) -> str:
|
||||
path = os.path.join(
|
||||
self.plan.target_dir, f"{TEMP_PREFIX}{uuid.uuid4().hex}{suffix}"
|
||||
)
|
||||
self.temps.append(path)
|
||||
return path
|
||||
|
||||
def prepare_target(self) -> None:
|
||||
target = self.plan.target_dir
|
||||
self.created_dir = not os.path.isdir(target)
|
||||
ll.ensure_private_dir(target)
|
||||
sweep_move_temps(target)
|
||||
if ll.read_marker(target).get("state") == ll.MARKER_PENDING:
|
||||
# An earlier move stopped before it switched: its files are ours.
|
||||
for name in (ll.DATABASE_NAME, ll.CONFIG_NAME) + self.plan.backups:
|
||||
_remove(os.path.join(target, name))
|
||||
for suffix in ("-wal", "-shm", "-journal"):
|
||||
_remove(os.path.join(target, ll.DATABASE_NAME + suffix))
|
||||
|
||||
def copy_round(self, source):
|
||||
"""Steps 2–4 once: the verified temp copies and the snapshot version."""
|
||||
self.progress("Copying the database")
|
||||
temp_db = self.temp(".db")
|
||||
version, counts, documents = _copy_snapshot(source, temp_db)
|
||||
self.checkpoint("copied")
|
||||
self.progress("Verifying the copy")
|
||||
_verify_copy(temp_db, counts, documents)
|
||||
self.checkpoint("verified")
|
||||
copies = []
|
||||
for name in self.plan.backups:
|
||||
src = os.path.join(self.plan.source_dir, name)
|
||||
temp = self.temp(".bak")
|
||||
try:
|
||||
expected = _hash_copy(src, temp)
|
||||
except OSError as error:
|
||||
logger.error(f"Library move: copying {name} failed: {error}")
|
||||
raise MoveFailed("copy-failed") from error
|
||||
copies.append((src, temp, expected, name))
|
||||
if copies:
|
||||
self.progress(f"Copied {len(copies)} backup file(s)")
|
||||
self.checkpoint("files")
|
||||
_check_copies([(s, t, e) for s, t, e, _ in copies])
|
||||
return version, counts, temp_db, copies
|
||||
|
||||
def discard_round(self, temp_db, copies) -> None:
|
||||
_remove(temp_db, *[temp for _s, temp, _e, _n in copies])
|
||||
|
||||
def publish(self, temp_db, copies) -> List[str]:
|
||||
"""Under the locks: config.json, marker, final names, marker moved."""
|
||||
target = self.plan.target_dir
|
||||
files = []
|
||||
if self.plan.config_path:
|
||||
temp = self.temp(".config")
|
||||
try:
|
||||
expected = _hash_copy(self.plan.config_path, temp)
|
||||
except OSError as error:
|
||||
raise MoveFailed("copy-failed") from error
|
||||
_check_copies([(self.plan.config_path, temp, expected)])
|
||||
copies = copies + [(self.plan.config_path, temp, expected, ll.CONFIG_NAME)]
|
||||
self.marker = write_marker(
|
||||
target, ll.MARKER_PENDING, self.plan.db_path, self.plan.config_path
|
||||
)
|
||||
self.checkpoint("pending")
|
||||
for _src, temp, _expected, name in copies:
|
||||
final = os.path.join(target, name)
|
||||
os.replace(temp, final)
|
||||
self.published.append(final)
|
||||
files.append(name)
|
||||
final_db = os.path.join(target, ll.DATABASE_NAME)
|
||||
os.replace(temp_db, final_db)
|
||||
self.published.append(final_db)
|
||||
files.append(ll.DATABASE_NAME)
|
||||
_fsync_dir(target)
|
||||
self.checkpoint("published")
|
||||
write_marker(target, ll.MARKER_MOVED, self.plan.db_path, self.plan.config_path)
|
||||
self.moved = True
|
||||
self.checkpoint("marked")
|
||||
return files
|
||||
|
||||
def cleanup(self) -> None:
|
||||
if self.moved:
|
||||
return
|
||||
_remove(*self.temps, *self.published, self.marker)
|
||||
if self.created_dir:
|
||||
try:
|
||||
os.rmdir(self.plan.target_dir)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _unique(path: str) -> str:
|
||||
candidate, number = path, 1
|
||||
while os.path.lexists(candidate):
|
||||
candidate = f"{path}_{number}"
|
||||
number += 1
|
||||
return candidate
|
||||
|
||||
|
||||
def retire_old_files(plan: MovePlan) -> Tuple[List[str], Optional[str]]:
|
||||
"""Rename the old database and config.json; ``(renamed, problem code)``.
|
||||
|
||||
Every connection to the old file must be closed. The WAL is checkpointed
|
||||
first; when it cannot be emptied (another process still has the file
|
||||
open) nothing is renamed (``old-in-use``).
|
||||
"""
|
||||
old = plan.db_path
|
||||
folder = os.path.basename(os.path.normpath(plan.target_dir))
|
||||
try:
|
||||
with contextlib.closing(sqlite3.connect(old, timeout=1)) as conn:
|
||||
busy = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()[0]
|
||||
except sqlite3.Error as error:
|
||||
logger.warning(
|
||||
f"Library move: the old database could not be checkpointed: {error}"
|
||||
)
|
||||
return [], "old-in-use"
|
||||
if busy or _size(old + "-wal") > 0:
|
||||
return [], "old-in-use"
|
||||
renamed = []
|
||||
try:
|
||||
for path in (old, plan.config_path):
|
||||
if not path or not os.path.isfile(path):
|
||||
continue
|
||||
new_name = _unique(path + MOVED_INFIX + folder)
|
||||
os.rename(path, new_name)
|
||||
renamed.append(os.path.basename(new_name))
|
||||
for suffix in ("-wal", "-shm"):
|
||||
if os.path.isfile(old + suffix) and _size(old + suffix) == 0:
|
||||
os.remove(old + suffix)
|
||||
except OSError as error:
|
||||
logger.warning(f"Library move: the old files could not be renamed: {error}")
|
||||
return renamed, "old-locked"
|
||||
return renamed, None
|
||||
|
||||
|
||||
def finish_interrupted_move(location) -> List[str]:
|
||||
"""Rename old files a crash left between the switch and the rename.
|
||||
|
||||
The marker of the folder in use says ``moved`` and names the old files;
|
||||
an old ``prompts.db`` (and config.json) still there under its own name is
|
||||
renamed ``*.moved-to-<folder>`` now, never deleted. Returns the new
|
||||
names (nothing when there is nothing to finish or the file is in use).
|
||||
"""
|
||||
if location.source not in (ll.SOURCE_DEFAULT, ll.SOURCE_ENVIRONMENT):
|
||||
return []
|
||||
marker = ll.read_marker(location.data_dir)
|
||||
old = marker.get("from")
|
||||
if marker.get("state") != ll.MARKER_MOVED or not isinstance(old, str):
|
||||
return []
|
||||
if not os.path.isfile(old) or os.path.basename(old) != ll.DATABASE_NAME:
|
||||
return []
|
||||
if ll.same_folder(os.path.dirname(old), location.data_dir):
|
||||
return []
|
||||
config = marker.get("config_from")
|
||||
if not (
|
||||
isinstance(config, str)
|
||||
and os.path.basename(config) == ll.CONFIG_NAME
|
||||
and os.path.isfile(config)
|
||||
and os.path.normcase(os.path.abspath(config))
|
||||
!= os.path.normcase(os.path.abspath(location.config_path))
|
||||
):
|
||||
config = None
|
||||
pending = types.SimpleNamespace(
|
||||
db_path=old, config_path=config, target_dir=location.data_dir
|
||||
)
|
||||
renamed, problem = retire_old_files(pending)
|
||||
if problem:
|
||||
logger.warning(f"Library move: the old files are still in use ({problem})")
|
||||
elif renamed:
|
||||
logger.info(f"Library move finished: renamed {', '.join(renamed)}")
|
||||
return renamed
|
||||
|
||||
|
||||
def move_library(
|
||||
plan: MovePlan,
|
||||
*,
|
||||
switch: Callable = switch_process,
|
||||
unswitch: Callable = unswitch_process,
|
||||
after_switch: Callable = _noop,
|
||||
config_lock=None,
|
||||
progress: Callable[[str], None] = _noop,
|
||||
checkpoint: Callable[[str], None] = _noop,
|
||||
attempts: int = DEFAULT_ATTEMPTS,
|
||||
) -> MoveResult:
|
||||
"""Move the library as planned (blocking; see the module docstring).
|
||||
|
||||
``switch(old_db, location, detached)`` points the process at the copy
|
||||
and adds the old connections it detached to ``detached``; if it raises,
|
||||
``unswitch(old_db, new_db)`` undoes it while both files are still
|
||||
locked, every detached connection is closed, the copy is removed and
|
||||
the move fails (``switch-failed``). ``after_switch()`` runs
|
||||
once the switch is final; it is best effort (logged, never fails the move).
|
||||
"""
|
||||
short = space_problem(plan)
|
||||
if short is not None:
|
||||
raise MoveFailed("insufficient-space")
|
||||
lock = config_lock if config_lock is not None else contextlib.nullcontext()
|
||||
move = _Move(
|
||||
plan,
|
||||
switch=switch,
|
||||
config_lock=lock,
|
||||
progress=progress,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
move.unswitch = unswitch
|
||||
location = plan.target_location
|
||||
try:
|
||||
move.prepare_target()
|
||||
files, counts = _copy_and_switch(move, location, attempts)
|
||||
except Exception:
|
||||
move.cleanup()
|
||||
raise
|
||||
try:
|
||||
after_switch()
|
||||
except Exception:
|
||||
logger.exception("Library move: a follow-up after the switch failed")
|
||||
from .execution_history import notify_history_restore
|
||||
|
||||
notify_history_restore(location.db_path)
|
||||
progress("Renaming the old files")
|
||||
renamed, problem = retire_old_files(plan)
|
||||
checkpoint("retired")
|
||||
return MoveResult(location, counts, files, renamed, problem)
|
||||
|
||||
|
||||
def _switch_guarded(move: _Move, location) -> list:
|
||||
"""Switch while the copy is locked too; undo it all if any part fails.
|
||||
|
||||
The copy's write lock keeps every repointed thread from writing to it
|
||||
until the switch is whole, so undoing a partial switch loses nothing.
|
||||
Returns the old file's detached connections (still open). On failure
|
||||
every detached connection is retired; those still in use are added to
|
||||
``move.retiring`` for the caller to await once the locks are released.
|
||||
"""
|
||||
plan = move.plan
|
||||
guard = sqlite3.connect(
|
||||
location.db_path, timeout=BUSY_TIMEOUT_SECONDS, isolation_level=None
|
||||
)
|
||||
try:
|
||||
guard.execute("BEGIN IMMEDIATE")
|
||||
detached: list = []
|
||||
try:
|
||||
move.switch(plan.db_path, location, detached)
|
||||
except Exception as error:
|
||||
logger.exception("Library move: switching failed; undoing it")
|
||||
move.moved = False # the copy and its marker are removed
|
||||
# Retire, never wait, here: a connection still in use may be
|
||||
# waiting on the very locks this thread holds.
|
||||
try:
|
||||
undone = move.unswitch(plan.db_path, location.db_path)
|
||||
except Exception:
|
||||
logger.exception("Library move: undoing the switch failed")
|
||||
else:
|
||||
_retire(move, undone)
|
||||
# The old file's connections a partial switch detached: the
|
||||
# models reopen it on next use, so close them now, not at GC.
|
||||
_retire(move, detached)
|
||||
raise MoveFailed("switch-failed") from error
|
||||
move.checkpoint("switched")
|
||||
return detached
|
||||
finally:
|
||||
if guard.in_transaction:
|
||||
guard.execute("ROLLBACK")
|
||||
guard.close()
|
||||
|
||||
|
||||
def _copy_and_switch(move: _Move, location, attempts: int):
|
||||
"""Copy, then switch under both locks; retired connections are awaited.
|
||||
|
||||
A connection that was inside a statement when it was retired is closed
|
||||
by its own thread once the statement ends, which may need the locks
|
||||
released first; the old files are renamed (and a failed copy removed)
|
||||
only after that, since Windows refuses to rename or delete an open file.
|
||||
"""
|
||||
try:
|
||||
return _copy_and_switch_locked(move, location, attempts)
|
||||
finally:
|
||||
try:
|
||||
PromptModel.await_detached(move.retiring, logger)
|
||||
except Exception:
|
||||
logger.exception("Library move: waiting for old connections failed")
|
||||
|
||||
|
||||
def _retire(move: _Move, connections: list) -> None:
|
||||
"""Retire ``connections``, adding those still in use to ``move.retiring``.
|
||||
|
||||
Never raises: a switch that completed must not be reported as failed
|
||||
because one of the old connections could not be retired (logged; the
|
||||
rename then reports the old files as still in use).
|
||||
"""
|
||||
try:
|
||||
move.retiring.extend(PromptModel.retire_detached(connections, logger))
|
||||
except Exception:
|
||||
logger.exception("Library move: retiring the old connections failed")
|
||||
|
||||
|
||||
def _copy_and_switch_locked(move: _Move, location, attempts: int):
|
||||
plan = move.plan
|
||||
for attempt in range(1, attempts + 1):
|
||||
source = sqlite3.connect(
|
||||
plan.db_path, timeout=BUSY_TIMEOUT_SECONDS, isolation_level=None
|
||||
)
|
||||
try:
|
||||
source.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}")
|
||||
version, counts, temp_db, copies = move.copy_round(source)
|
||||
source.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
if source.execute("PRAGMA data_version").fetchone()[0] != version:
|
||||
move.progress("The library changed during the copy; copying again")
|
||||
move.discard_round(temp_db, copies)
|
||||
continue
|
||||
move.checkpoint("locked")
|
||||
with move.config_lock:
|
||||
files = move.publish(temp_db, copies)
|
||||
detached = _switch_guarded(move, location)
|
||||
# Still holding the old file's lock: a write that was waiting
|
||||
# on it is interrupted and cannot commit, so it fails (and
|
||||
# runs again on the new library) instead of landing in the
|
||||
# old file. Its own thread closes the connection.
|
||||
_retire(move, detached)
|
||||
finally:
|
||||
if source.in_transaction:
|
||||
source.execute("ROLLBACK")
|
||||
return files, counts
|
||||
finally:
|
||||
source.close()
|
||||
raise MoveFailed("library-busy")
|
||||
@@ -0,0 +1,740 @@
|
||||
"""Reviewed maintenance plans: read-only previews and their exact apply (#249 S6).
|
||||
|
||||
Four operations, each the native owner of a legacy ``/maintenance`` one and
|
||||
built on the same queries, so a preview counts what the legacy cleanup would
|
||||
change:
|
||||
|
||||
- ``duplicate_prompts``: prompts whose text is equal ignoring case and outer
|
||||
spaces (``DUPLICATE_PROMPTS_SQL``, legacy ``cleanup_duplicates``); each
|
||||
group keeps its oldest prompt, which takes over the others' images and
|
||||
metadata. A group with a member in a saved pair is kept whole.
|
||||
- ``optimize``: VACUUM (legacy ``vacuum``).
|
||||
- ``unused_components``: prompts nothing references (``ORPHANED_PROMPTS_SQL``,
|
||||
legacy ``prune_orphaned_prompts``, with the #239 protections).
|
||||
- ``missing_links``: image rows whose file is gone (legacy
|
||||
``cleanup_orphaned_images``), counted only under a source that can be read
|
||||
now, as the health check's ``missing_files`` judges them
|
||||
(``health.link_states``). It deletes database rows only, never a file.
|
||||
|
||||
``preview_maintenance`` reads one snapshot on its own ``mode=ro`` connection
|
||||
(``health.readonly_uri``) and closes it in ``finally``; files are looked at
|
||||
after the snapshot ended. It returns a ``MaintenanceDraft``: the public plan
|
||||
fields, the targets an apply needs and the fingerprint of exactly what was
|
||||
reviewed (ids, text hashes, metadata, tags, Library state and pair refs of
|
||||
every prompt involved, or each link's row and source). The fingerprint never
|
||||
leaves the server.
|
||||
|
||||
``apply_maintenance`` recomputes that fingerprint on the write connection
|
||||
inside ``BEGIN IMMEDIATE`` and raises ``PlanChanged`` before writing anything
|
||||
when it differs; otherwise it applies the reviewed targets in batches of
|
||||
``ID_BATCH`` in that one transaction. VACUUM cannot run in a transaction, so
|
||||
``optimize`` binds to its database only (the caller checks that).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from .execution_contexts import LIVE_HINT_CTES
|
||||
from .health import BATCH_ROWS, link_states, readable_roots, readonly_uri
|
||||
from .pairs import PairDependencyError
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
MAINTENANCE_OPERATIONS = (
|
||||
"duplicate_prompts",
|
||||
"optimize",
|
||||
"unused_components",
|
||||
"missing_links",
|
||||
# Applied as its own batched job (database/workflow_storage.py).
|
||||
"workflow_storage",
|
||||
)
|
||||
MAX_DETAILS = 50 # detail rows a plan lists
|
||||
MAX_RESULT_ROWS = 200 # changed records an apply describes (the job lists 200)
|
||||
EXCERPT_CHARS = 80
|
||||
# Ids bound per statement: the pair-reference read binds each id twice, which
|
||||
# stays under SQLite's historical 999-variable limit.
|
||||
ID_BATCH = 400
|
||||
|
||||
# Why a prompt no image links to is kept (#239): predicates on ``p``, in the
|
||||
# order the plan names them. ORPHANED_PROMPTS_SQL is built from them, so the
|
||||
# reasons a plan lists are exactly the rules the prune applies.
|
||||
PROTECTIONS = (
|
||||
(
|
||||
"EXISTS (SELECT 1 FROM prompt_pairs pp "
|
||||
"WHERE pp.positive_prompt_id = p.id OR pp.negative_prompt_id = p.id)",
|
||||
"in saved pairs",
|
||||
"in a saved pair",
|
||||
),
|
||||
(
|
||||
"EXISTS (SELECT 1 FROM prompt_tags pt JOIN tags t ON pt.tag_id = t.id "
|
||||
"WHERE pt.prompt_id = p.id AND t.name = '__protected__')",
|
||||
"tagged __protected__",
|
||||
"tagged __protected__",
|
||||
),
|
||||
(
|
||||
"EXISTS (SELECT 1 FROM prompt_library_state ls WHERE ls.prompt_id = p.id)",
|
||||
"with Library state (a title or an explicit parent)",
|
||||
"with Library state (a title or an explicit parent)",
|
||||
),
|
||||
(
|
||||
"p.id IN (SELECT prompt_id FROM live_hint_prompts)",
|
||||
"used by recorded runs",
|
||||
"used by a recorded run",
|
||||
),
|
||||
)
|
||||
|
||||
# Exempt references: images, pairs, the __protected__ tag, Library parent
|
||||
# state, and live recorded-run hints (computed once, see LIVE_HINT_CTES).
|
||||
ORPHANED_PROMPTS_SQL = (
|
||||
"WITH "
|
||||
+ LIVE_HINT_CTES
|
||||
+ " SELECT p.id FROM prompts p"
|
||||
+ " LEFT JOIN generated_images gi ON p.id = gi.prompt_id"
|
||||
+ " WHERE gi.prompt_id IS NULL"
|
||||
+ "".join(f" AND NOT ({predicate})" for predicate, _many, _one in PROTECTIONS)
|
||||
)
|
||||
# The prompts no image links to, with which protections keep each one.
|
||||
_UNLINKED_PROTECTIONS_SQL = (
|
||||
"WITH "
|
||||
+ LIVE_HINT_CTES
|
||||
+ " SELECT "
|
||||
+ ", ".join(
|
||||
f"COALESCE(SUM({predicate}), 0)" for predicate, _many, _one in PROTECTIONS
|
||||
)
|
||||
+ ", COALESCE(SUM("
|
||||
+ " OR ".join(f"({predicate})" for predicate, _many, _one in PROTECTIONS)
|
||||
+ "), 0) FROM prompts p WHERE NOT EXISTS "
|
||||
"(SELECT 1 FROM generated_images gi WHERE gi.prompt_id = p.id)"
|
||||
)
|
||||
|
||||
# Duplicate prompt groups, as the legacy cleanup finds them (case-insensitive
|
||||
# text; GROUP_CONCAT order is sorted by created_at afterwards).
|
||||
DUPLICATE_PROMPTS_SQL = """
|
||||
SELECT LOWER(TRIM(text)) as normalized_text, COUNT(*) as count,
|
||||
GROUP_CONCAT(id) as ids,
|
||||
GROUP_CONCAT(created_at) as created_dates
|
||||
FROM prompts
|
||||
GROUP BY LOWER(TRIM(text))
|
||||
HAVING COUNT(*) > 1
|
||||
"""
|
||||
|
||||
_LINK_ROWS_SQL = (
|
||||
"SELECT id, image_path, filename, file_path FROM generated_images ORDER BY id"
|
||||
)
|
||||
|
||||
# What a prompt's fingerprint covers, per batch of ids.
|
||||
_PROMPT_STATE_SQL = (
|
||||
"SELECT id, hash, text, category, rating, notes, created_at, updated_at "
|
||||
"FROM prompts WHERE id IN ({marks}) ORDER BY id",
|
||||
"SELECT prompt_id, tag_id FROM prompt_tags WHERE prompt_id IN ({marks}) "
|
||||
"ORDER BY prompt_id, tag_id",
|
||||
"SELECT * FROM prompt_library_state WHERE prompt_id IN ({marks}) "
|
||||
"ORDER BY prompt_id",
|
||||
"SELECT id, positive_prompt_id, negative_prompt_id FROM prompt_pairs "
|
||||
"WHERE positive_prompt_id IN ({marks}) OR negative_prompt_id IN ({marks}) "
|
||||
"ORDER BY id",
|
||||
)
|
||||
|
||||
NO_FILES_DELETED = "No image file is deleted."
|
||||
LINKS_ONLY = "Database links only; no image file is deleted."
|
||||
UNUSED_DETAIL = "No image, saved pair, Library state or recorded run uses it"
|
||||
|
||||
|
||||
# Why a duplicate group is kept whole (``_duplicate_conflict``).
|
||||
CONFLICT_REASONS = {
|
||||
"collision": ("group links one image twice", "groups link one image twice"),
|
||||
"title": (
|
||||
"group has its own Library title",
|
||||
"groups have their own Library titles",
|
||||
),
|
||||
}
|
||||
CONFLICT_DETAILS = {
|
||||
"collision": "Kept: two copies link the same image",
|
||||
"title": "Kept: a copy has its own Library title",
|
||||
}
|
||||
OPTIMIZE_LOCK_NOTE = (
|
||||
"Runs alone: the database is locked while it runs, and on a large library "
|
||||
"this takes a while. Prompt runs counted and images captured meanwhile "
|
||||
"wait up to 30 seconds each; anything that waits longer is not recorded."
|
||||
)
|
||||
|
||||
|
||||
class PlanChanged(Exception):
|
||||
"""What a plan was reviewed on changed before it was applied."""
|
||||
|
||||
|
||||
class MergeConflict(Exception):
|
||||
"""A strict duplicate merge would drop an image link or a Library title."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class MaintenanceDraft:
|
||||
"""A preview: the public plan fields plus what only the server keeps."""
|
||||
|
||||
operation: str
|
||||
fingerprint: str
|
||||
targets: list
|
||||
scope: dict
|
||||
protected: dict
|
||||
media: dict
|
||||
details: list = field(default_factory=list)
|
||||
|
||||
def public(self):
|
||||
"""The plan fields a response may carry (no fingerprint, no targets)."""
|
||||
return {
|
||||
"operation": self.operation,
|
||||
"scope": dict(self.scope),
|
||||
"protected": {
|
||||
"count": self.protected["count"],
|
||||
"reasons": list(self.protected["reasons"]),
|
||||
},
|
||||
"media": dict(self.media),
|
||||
"details": [dict(row) for row in self.details],
|
||||
}
|
||||
|
||||
|
||||
def _plural(count, one, many=None):
|
||||
return f"{count:,} {one if count == 1 else (many or one + 's')}"
|
||||
|
||||
|
||||
def _excerpt(text):
|
||||
text = " ".join(str(text or "").split())
|
||||
if len(text) <= EXCERPT_CHARS:
|
||||
return text
|
||||
return text[: EXCERPT_CHARS - 1].rstrip() + "…"
|
||||
|
||||
|
||||
def _megabytes(size):
|
||||
return f"{size / 2**20:,.1f} MB"
|
||||
|
||||
|
||||
class _Fingerprint:
|
||||
"""A streaming sha256 over JSON-encoded parts."""
|
||||
|
||||
def __init__(self, operation):
|
||||
self._hash = hashlib.sha256()
|
||||
self.add(operation)
|
||||
|
||||
def add(self, part):
|
||||
encoded = json.dumps(part, separators=(",", ":"), default=str)
|
||||
self._hash.update(encoded.encode("utf-8") + b"\n")
|
||||
|
||||
def hexdigest(self):
|
||||
return "sha256:" + self._hash.hexdigest()
|
||||
|
||||
|
||||
def _batches(ids):
|
||||
ids = list(ids)
|
||||
for start in range(0, len(ids), ID_BATCH):
|
||||
yield ids[start : start + ID_BATCH]
|
||||
|
||||
|
||||
def _marks(batch):
|
||||
return ",".join("?" for _ in batch)
|
||||
|
||||
|
||||
def _add_prompt_state(fingerprint, conn, ids):
|
||||
"""Fold every reviewed fact about the prompts ``ids`` into ``fingerprint``."""
|
||||
for batch in _batches(sorted(ids)):
|
||||
marks = _marks(batch)
|
||||
for sql in _PROMPT_STATE_SQL:
|
||||
params = batch * sql.count("{marks}")
|
||||
for row in conn.execute(sql.format(marks=marks), params):
|
||||
fingerprint.add(list(row))
|
||||
|
||||
|
||||
def _texts(conn, ids, limit=MAX_DETAILS):
|
||||
"""``{id: text}`` of the first ``limit`` prompts, for detail rows."""
|
||||
ids = list(ids)[:limit]
|
||||
if not ids:
|
||||
return {}
|
||||
rows = conn.execute(
|
||||
f"SELECT id, text FROM prompts WHERE id IN ({_marks(ids)})", ids
|
||||
)
|
||||
return {row[0]: row[1] for row in rows}
|
||||
|
||||
|
||||
def _orphaned_ids(conn):
|
||||
return sorted(row[0] for row in conn.execute(ORPHANED_PROMPTS_SQL))
|
||||
|
||||
|
||||
def _delete_ids(conn, table, ids):
|
||||
"""Delete ``table`` rows by id in batches; returns the rows removed.
|
||||
|
||||
The workflow documents only those rows referenced go with them (a
|
||||
prompt's images go by cascade), in the same transaction.
|
||||
"""
|
||||
if table not in ("prompts", "generated_images"):
|
||||
raise ValueError(f"Unsupported table {table!r}")
|
||||
removed = 0
|
||||
for batch in _batches(ids):
|
||||
documents = documents_of_rows(
|
||||
conn, "id" if table == "generated_images" else "prompt_id", batch
|
||||
)
|
||||
cursor = conn.execute(
|
||||
f"DELETE FROM {table} WHERE id IN ({_marks(batch)})", # nosec B608
|
||||
batch,
|
||||
)
|
||||
removed += cursor.rowcount
|
||||
release_documents(conn, documents)
|
||||
return removed
|
||||
|
||||
|
||||
def _read_snapshot(db_path, read):
|
||||
"""``read(conn)`` on one ``mode=ro`` snapshot; the connection always closes."""
|
||||
conn = sqlite3.connect(readonly_uri(db_path), uri=True)
|
||||
try:
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("BEGIN") # one read snapshot for the whole plan
|
||||
return read(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _relative(image_path, root):
|
||||
try:
|
||||
return Path(os.path.relpath(image_path, root)).as_posix()
|
||||
except ValueError: # another drive (Windows)
|
||||
return os.path.basename(image_path)
|
||||
|
||||
|
||||
def _source_name(root, index):
|
||||
return os.path.basename(str(root).rstrip("\\/")) or f"Source {index + 1}"
|
||||
|
||||
|
||||
class MaintenanceOperations:
|
||||
"""Mixin for PromptDatabase: maintenance previews and their apply."""
|
||||
|
||||
# ── shared with the legacy cleanup ───────────────────────────────
|
||||
|
||||
def _duplicate_groups(self, conn):
|
||||
"""``(groups, paired)``: duplicate id groups, oldest first.
|
||||
|
||||
``paired`` are the groups with a member in a saved pair, which the
|
||||
cleanup keeps whole.
|
||||
"""
|
||||
groups, paired = [], []
|
||||
for duplicate in conn.execute(DUPLICATE_PROMPTS_SQL).fetchall():
|
||||
ids = duplicate["ids"].split(",")
|
||||
created_dates = duplicate["created_dates"].split(",")
|
||||
# Sort IDs by created_at date to keep the oldest
|
||||
id_date_pairs = list(zip(ids, created_dates))
|
||||
id_date_pairs.sort(key=lambda x: x[1])
|
||||
sorted_ids = [int(pair[0]) for pair in id_date_pairs]
|
||||
try:
|
||||
self._ensure_prompts_unpaired(conn, sorted_ids)
|
||||
except PairDependencyError:
|
||||
paired.append(sorted_ids)
|
||||
continue
|
||||
groups.append(sorted_ids)
|
||||
return groups, paired
|
||||
|
||||
@staticmethod
|
||||
def _duplicate_conflict(conn, sorted_ids):
|
||||
"""Why merging the group would lose something, or None.
|
||||
|
||||
``collision``: two members link the same file, so moving the images
|
||||
to the oldest prompt would hit ``UNIQUE(prompt_id, file_path)``.
|
||||
``title``: a member that would be deleted has its own Library title.
|
||||
"""
|
||||
marks = _marks(sorted_ids)
|
||||
collision = conn.execute(
|
||||
"SELECT 1 FROM generated_images WHERE prompt_id IN " # nosec B608
|
||||
f"({marks}) AND file_path IS NOT NULL GROUP BY file_path "
|
||||
"HAVING COUNT(DISTINCT prompt_id) > 1 LIMIT 1",
|
||||
sorted_ids,
|
||||
).fetchone()
|
||||
if collision:
|
||||
return "collision"
|
||||
duplicates = sorted_ids[1:]
|
||||
titled = conn.execute(
|
||||
"SELECT 1 FROM prompt_library_state WHERE prompt_id IN " # nosec B608
|
||||
f"({_marks(duplicates)}) AND trim(coalesce(title, '')) != '' LIMIT 1",
|
||||
duplicates,
|
||||
).fetchone()
|
||||
return "title" if titled else None
|
||||
|
||||
def _merge_duplicate_group(self, conn, sorted_ids, strict=False):
|
||||
"""Keep the oldest prompt of a group; returns (removed, images moved).
|
||||
|
||||
``strict`` (the reviewed apply): a merge that would drop an image
|
||||
link or a Library title raises MergeConflict, and every helper error
|
||||
propagates, so the transaction rolls back. The legacy cleanup keeps
|
||||
its old, lenient behaviour.
|
||||
"""
|
||||
primary_id, duplicate_ids = sorted_ids[0], sorted_ids[1:]
|
||||
if strict:
|
||||
conflict = self._duplicate_conflict(conn, sorted_ids)
|
||||
if conflict:
|
||||
raise MergeConflict(conflict)
|
||||
self.logger.debug(
|
||||
f"Merging duplicates: keeping {primary_id}, removing {duplicate_ids}"
|
||||
)
|
||||
primary = conn.execute(
|
||||
"SELECT * FROM prompts WHERE id = ?", (primary_id,)
|
||||
).fetchone()
|
||||
if not primary:
|
||||
return 0, 0
|
||||
merged_metadata = self._merge_duplicate_metadata(
|
||||
conn, primary_id, duplicate_ids, strict=strict
|
||||
)
|
||||
moved = self._transfer_images_to_primary(
|
||||
conn, primary_id, duplicate_ids, strict=strict
|
||||
)
|
||||
if merged_metadata:
|
||||
self._update_primary_with_merged_metadata(
|
||||
conn, primary_id, merged_metadata, strict=strict
|
||||
)
|
||||
removed = 0
|
||||
# Images the lenient merge could not move go with their prompt.
|
||||
documents = documents_of_rows(conn, "prompt_id", duplicate_ids)
|
||||
for duplicate_id in duplicate_ids:
|
||||
conn.execute("DELETE FROM prompts WHERE id = ?", (duplicate_id,))
|
||||
removed += 1
|
||||
self.logger.debug(f"Removed duplicate prompt {duplicate_id}")
|
||||
release_documents(conn, documents)
|
||||
return removed, moved
|
||||
|
||||
@staticmethod
|
||||
def _missing_link_ids(rows, roots):
|
||||
"""Ids of the ``rows`` whose file is gone under a readable source."""
|
||||
readable = readable_roots(roots)
|
||||
return [
|
||||
row[0]
|
||||
for row, _owner, state in link_states(rows, roots, readable)
|
||||
if state == "missing"
|
||||
]
|
||||
|
||||
# ── previews ──────────────────────────────────────────────────────
|
||||
|
||||
def preview_maintenance(self, operation, roots=()) -> MaintenanceDraft:
|
||||
"""The reviewed plan of ``operation``; writes nothing (blocking).
|
||||
|
||||
``roots`` are the sources missing links are judged under (the health
|
||||
check's: the effective sources plus configured ones that are gone).
|
||||
"""
|
||||
if operation not in MAINTENANCE_OPERATIONS:
|
||||
raise ValueError(f"Unknown maintenance operation {operation!r}")
|
||||
roots = [str(root) for root in roots]
|
||||
if operation == "optimize":
|
||||
return self._plan_optimize()
|
||||
if operation == "workflow_storage":
|
||||
return self._plan_workflow_storage()
|
||||
if operation == "missing_links":
|
||||
rows = _read_snapshot(self.model.db_path, self._link_rows)
|
||||
return self._plan_missing_links(rows, roots)
|
||||
reader = (
|
||||
self._plan_duplicate_prompts
|
||||
if operation == "duplicate_prompts"
|
||||
else self._plan_unused_components
|
||||
)
|
||||
return _read_snapshot(self.model.db_path, reader)
|
||||
|
||||
@staticmethod
|
||||
def _link_rows(conn):
|
||||
rows = []
|
||||
cursor = conn.execute(_LINK_ROWS_SQL)
|
||||
while batch := cursor.fetchmany(BATCH_ROWS):
|
||||
rows.extend(tuple(row) for row in batch)
|
||||
return rows
|
||||
|
||||
def _duplicates_fingerprint(self, conn):
|
||||
"""``(groups, paired, conflicted, fingerprint)`` of the duplicate merge.
|
||||
|
||||
``conflicted`` holds ``(ids, reason)`` of the groups a merge would
|
||||
lose an image link or a Library title in (``_duplicate_conflict``);
|
||||
the plan protects them.
|
||||
"""
|
||||
candidates, paired = self._duplicate_groups(conn)
|
||||
groups, conflicted = [], []
|
||||
for ids in candidates:
|
||||
reason = self._duplicate_conflict(conn, ids)
|
||||
if reason:
|
||||
conflicted.append((ids, reason))
|
||||
else:
|
||||
groups.append(ids)
|
||||
fingerprint = _Fingerprint("duplicate_prompts")
|
||||
fingerprint.add({"groups": groups, "paired": paired, "conflicted": conflicted})
|
||||
_add_prompt_state(fingerprint, conn, [i for group in groups for i in group])
|
||||
return groups, paired, conflicted, fingerprint.hexdigest()
|
||||
|
||||
def _plan_duplicate_prompts(self, conn):
|
||||
groups, paired, conflicted, fingerprint = self._duplicates_fingerprint(conn)
|
||||
removed = sum(len(group) - 1 for group in groups)
|
||||
held = [ids for ids, _reason in conflicted]
|
||||
texts = _texts(conn, [group[0] for group in (groups + paired + held)])
|
||||
details = (
|
||||
[
|
||||
{
|
||||
"label": _excerpt(texts.get(group[0])),
|
||||
"detail": f"Keeps the oldest of {len(group)} copies",
|
||||
}
|
||||
for group in groups
|
||||
]
|
||||
+ [
|
||||
{
|
||||
"label": _excerpt(texts.get(group[0])),
|
||||
"detail": "Kept: belongs to a saved pair",
|
||||
}
|
||||
for group in paired
|
||||
]
|
||||
+ [
|
||||
{
|
||||
"label": _excerpt(texts.get(ids[0])),
|
||||
"detail": CONFLICT_DETAILS[reason],
|
||||
}
|
||||
for ids, reason in conflicted
|
||||
]
|
||||
)
|
||||
if groups:
|
||||
label = (
|
||||
f"{_plural(removed, 'duplicate prompt')} in "
|
||||
f"{_plural(len(groups), 'group')}"
|
||||
)
|
||||
else:
|
||||
label = "No duplicate prompt text"
|
||||
reasons = []
|
||||
if len(paired) == 1:
|
||||
reasons.append("1 group belongs to a saved pair")
|
||||
elif paired:
|
||||
reasons.append(f"{len(paired):,} groups belong to saved pairs")
|
||||
for reason, (one, many) in CONFLICT_REASONS.items():
|
||||
count = sum(1 for _ids, why in conflicted if why == reason)
|
||||
if count:
|
||||
reasons.append(f"{count:,} {one if count == 1 else many}")
|
||||
return MaintenanceDraft(
|
||||
operation="duplicate_prompts",
|
||||
fingerprint=fingerprint,
|
||||
targets=groups,
|
||||
scope={"count": removed, "label": label},
|
||||
protected={
|
||||
"count": sum(len(group) for group in paired + held),
|
||||
"reasons": reasons,
|
||||
},
|
||||
media={"deletes_files": False, "label": NO_FILES_DELETED},
|
||||
details=details[:MAX_DETAILS],
|
||||
)
|
||||
|
||||
def _unused_fingerprint(self, conn):
|
||||
ids = _orphaned_ids(conn)
|
||||
fingerprint = _Fingerprint("unused_components")
|
||||
fingerprint.add(ids)
|
||||
_add_prompt_state(fingerprint, conn, ids)
|
||||
return ids, fingerprint.hexdigest()
|
||||
|
||||
def _plan_unused_components(self, conn):
|
||||
ids, fingerprint = self._unused_fingerprint(conn)
|
||||
*by_reason, kept = conn.execute(_UNLINKED_PROTECTIONS_SQL).fetchone()
|
||||
reasons = [
|
||||
f"{count:,} {many if count != 1 else one}"
|
||||
for count, (_predicate, many, one) in zip(by_reason, PROTECTIONS)
|
||||
if count
|
||||
]
|
||||
texts = _texts(conn, ids)
|
||||
count = len(ids)
|
||||
return MaintenanceDraft(
|
||||
operation="unused_components",
|
||||
fingerprint=fingerprint,
|
||||
targets=ids,
|
||||
scope={
|
||||
"count": count,
|
||||
"label": (
|
||||
_plural(count, "unused prompt component")
|
||||
if count
|
||||
else "No unused prompt components"
|
||||
),
|
||||
},
|
||||
protected={"count": kept, "reasons": reasons},
|
||||
media={"deletes_files": False, "label": NO_FILES_DELETED},
|
||||
details=[
|
||||
{"label": _excerpt(texts.get(prompt_id)), "detail": UNUSED_DETAIL}
|
||||
for prompt_id in ids[:MAX_DETAILS]
|
||||
],
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _links_fingerprint(candidates):
|
||||
fingerprint = _Fingerprint("missing_links")
|
||||
for row, owner in candidates:
|
||||
fingerprint.add([row[0], row[1], row[3], owner])
|
||||
return fingerprint.hexdigest()
|
||||
|
||||
def _plan_missing_links(self, rows, roots):
|
||||
readable = readable_roots(roots)
|
||||
candidates, unavailable, outside = [], 0, 0
|
||||
for row, owner, state in link_states(rows, roots, readable):
|
||||
if state == "missing":
|
||||
candidates.append((row, owner))
|
||||
elif state == "unavailable":
|
||||
unavailable += 1
|
||||
elif state == "outside":
|
||||
outside += 1
|
||||
reasons = []
|
||||
if unavailable:
|
||||
reasons.append(
|
||||
_plural(
|
||||
unavailable,
|
||||
"link under an unavailable source",
|
||||
"links under unavailable sources",
|
||||
)
|
||||
)
|
||||
if outside:
|
||||
reasons.append(
|
||||
_plural(
|
||||
outside,
|
||||
"link outside the image sources",
|
||||
"links outside the image sources",
|
||||
)
|
||||
)
|
||||
count = len(candidates)
|
||||
return MaintenanceDraft(
|
||||
operation="missing_links",
|
||||
fingerprint=self._links_fingerprint(candidates),
|
||||
targets=[row[0] for row, _owner in candidates],
|
||||
scope={
|
||||
"count": count,
|
||||
"label": (
|
||||
_plural(count, "link to a missing file", "links to missing files")
|
||||
if count
|
||||
else "No links to missing files"
|
||||
),
|
||||
},
|
||||
protected={"count": unavailable + outside, "reasons": reasons},
|
||||
media={"deletes_files": False, "label": LINKS_ONLY},
|
||||
details=[
|
||||
{
|
||||
"label": _relative(row[1], roots[owner]),
|
||||
"detail": (
|
||||
f"{_source_name(roots[owner], owner)} · the file is missing"
|
||||
),
|
||||
}
|
||||
for row, owner in candidates[:MAX_DETAILS]
|
||||
],
|
||||
)
|
||||
|
||||
def _plan_optimize(self):
|
||||
def read(conn):
|
||||
return [
|
||||
conn.execute(f"PRAGMA {name}").fetchone()[0]
|
||||
for name in ("page_size", "page_count", "freelist_count")
|
||||
]
|
||||
|
||||
page_size, page_count, free_pages = _read_snapshot(self.model.db_path, read)
|
||||
db_path = self.model.db_path
|
||||
wal = db_path + "-wal"
|
||||
size = page_size * page_count
|
||||
wal_size = os.path.getsize(wal) if os.path.exists(wal) else 0
|
||||
free = shutil.disk_usage(os.path.dirname(os.path.abspath(db_path))).free
|
||||
# VACUUM writes the rebuilt database through the WAL beside it.
|
||||
return MaintenanceDraft(
|
||||
operation="optimize",
|
||||
fingerprint=_Fingerprint("optimize").hexdigest(),
|
||||
targets=[],
|
||||
scope={"count": 1, "label": "The whole database"},
|
||||
protected={"count": 0, "reasons": []},
|
||||
media={"deletes_files": False, "label": NO_FILES_DELETED},
|
||||
details=[
|
||||
{
|
||||
"label": "Database file",
|
||||
"detail": f"{_megabytes(size)}, plus {_megabytes(wal_size)} "
|
||||
"not yet checkpointed",
|
||||
},
|
||||
{
|
||||
"label": "Free pages",
|
||||
"detail": f"About {_megabytes(free_pages * page_size)} "
|
||||
"can be reclaimed",
|
||||
},
|
||||
{
|
||||
"label": "Lock",
|
||||
"detail": OPTIMIZE_LOCK_NOTE,
|
||||
},
|
||||
{
|
||||
"label": "Free space needed",
|
||||
"detail": f"About {_megabytes(size)} beside the database and "
|
||||
f"{_megabytes(size)} in SQLite's temporary folder; "
|
||||
f"{_megabytes(free)} available beside the database",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
def apply_maintenance(self, operation, fingerprint, targets, roots=()) -> dict:
|
||||
"""Apply a reviewed plan exactly, or raise PlanChanged (blocking).
|
||||
|
||||
Inside ``BEGIN IMMEDIATE`` the fingerprint is computed again on the
|
||||
write connection; when it differs nothing is written. ``targets`` and
|
||||
``roots`` are the plan's. Returns ``{changed, rows}``: how many
|
||||
records changed and up to ``MAX_RESULT_ROWS`` descriptions of them
|
||||
(``{prompt_id[, kept_id]}`` or ``{image_path}``).
|
||||
"""
|
||||
if operation not in MAINTENANCE_OPERATIONS or operation == "workflow_storage":
|
||||
raise ValueError(f"Unknown maintenance operation {operation!r}")
|
||||
if operation == "optimize":
|
||||
# VACUUM cannot run in a transaction; the plan binds the database.
|
||||
if fingerprint != _Fingerprint("optimize").hexdigest():
|
||||
raise PlanChanged("optimize")
|
||||
self.model.vacuum_database()
|
||||
return {"changed": 1, "rows": []}
|
||||
appliers = {
|
||||
"duplicate_prompts": self._apply_duplicate_prompts,
|
||||
"unused_components": self._apply_unused_components,
|
||||
"missing_links": self._apply_missing_links,
|
||||
}
|
||||
with self._pair_transaction() as conn:
|
||||
return appliers[operation](conn, fingerprint, targets, roots)
|
||||
|
||||
def _apply_duplicate_prompts(self, conn, fingerprint, targets, _roots):
|
||||
groups, _paired, _conflicted, current = self._duplicates_fingerprint(conn)
|
||||
if current != fingerprint or groups != targets:
|
||||
raise PlanChanged("duplicate_prompts")
|
||||
changed, rows = 0, []
|
||||
for sorted_ids in groups:
|
||||
removed, _moved = self._merge_duplicate_group(conn, sorted_ids, strict=True)
|
||||
changed += removed
|
||||
rows.extend(
|
||||
{"prompt_id": duplicate_id, "kept_id": sorted_ids[0]}
|
||||
for duplicate_id in sorted_ids[1:]
|
||||
)
|
||||
return {"changed": changed, "rows": rows[:MAX_RESULT_ROWS]}
|
||||
|
||||
def _apply_unused_components(self, conn, fingerprint, targets, _roots):
|
||||
ids, current = self._unused_fingerprint(conn)
|
||||
if current != fingerprint or ids != targets:
|
||||
raise PlanChanged("unused_components")
|
||||
texts = _texts(conn, ids, limit=MAX_RESULT_ROWS)
|
||||
changed = _delete_ids(conn, "prompts", ids)
|
||||
rows = [
|
||||
{"prompt_id": prompt_id, "text": _excerpt(texts.get(prompt_id))}
|
||||
for prompt_id in ids[:MAX_RESULT_ROWS]
|
||||
]
|
||||
return {"changed": changed, "rows": rows}
|
||||
|
||||
def _apply_missing_links(self, conn, fingerprint, targets, roots):
|
||||
rows = []
|
||||
for batch in _batches(sorted(targets)):
|
||||
rows.extend(
|
||||
tuple(row)
|
||||
for row in conn.execute(
|
||||
"SELECT id, image_path, filename, file_path FROM generated_images "
|
||||
f"WHERE id IN ({_marks(batch)}) ORDER BY id", # nosec B608
|
||||
batch,
|
||||
)
|
||||
)
|
||||
readable = readable_roots(roots)
|
||||
candidates = []
|
||||
for row, owner, state in link_states(rows, roots, readable):
|
||||
if state != "missing": # back, or its source went away
|
||||
raise PlanChanged("missing_links")
|
||||
candidates.append((row, owner))
|
||||
if self._links_fingerprint(candidates) != fingerprint:
|
||||
raise PlanChanged("missing_links")
|
||||
changed = _delete_ids(conn, "generated_images", [row[0] for row in rows])
|
||||
return {
|
||||
"changed": changed,
|
||||
"rows": [{"image_path": row[1]} for row in rows[:MAX_RESULT_ROWS]],
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
"""A SQLite connection that is never closed under the thread using it.
|
||||
|
||||
``PromptModel`` gives each thread its own connection, but Move library and
|
||||
``close_all`` must retire every thread's connection from one thread. Closing
|
||||
a ``sqlite3.Connection`` while its owner is inside a statement (Python runs
|
||||
``sqlite3_step`` without the GIL) frees memory the owner then reads: on
|
||||
Windows the process dies with "access violation", elsewhere the owner sees
|
||||
garbage such as a ``MemoryError``.
|
||||
|
||||
``ManagedConnection`` counts the calls its owner is inside. ``retire()`` is
|
||||
safe from any thread and never raises: an idle connection is closed at once;
|
||||
one in use is closed by its owner as the call ends. A retired connection
|
||||
refuses new work (cursors, statements, commits) with the error a closed
|
||||
connection raises.
|
||||
|
||||
What retiring a connection in use does and does not do:
|
||||
|
||||
- It calls ``sqlite3_interrupt`` (thread-safe). That ends a running
|
||||
statement at its next step, but it does not wake a statement that is
|
||||
waiting for another connection's lock: that one waits until the lock is
|
||||
released (or the busy timeout ends it), and only then is it interrupted.
|
||||
An interrupt that arrives between two statements, for example between the
|
||||
implicit ``BEGIN`` and the DML or between ``executemany`` rows, is lost.
|
||||
- Writes are kept out of a file the process has switched away from because
|
||||
a retired connection never commits: legacy-mode DML runs inside a
|
||||
transaction that only ``commit()`` (refused) or ``close()`` (rolls back)
|
||||
ends. A statement that commits by itself is not covered: DDL or a PRAGMA
|
||||
write outside a transaction, or an explicit ``COMMIT`` passed to
|
||||
``execute``/``executescript`` (``executescript`` also commits first, in C,
|
||||
on Python before 3.12). Managed connections must never be used that way
|
||||
for library writes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
from typing import Iterable, Iterator, List, Optional
|
||||
|
||||
import sqlite3
|
||||
|
||||
CLOSED_MESSAGE = "Cannot operate on a closed database."
|
||||
# Rows fetched per step when a cursor is iterated: the connection is marked
|
||||
# in use once per chunk instead of once per row (iteration costs about what
|
||||
# a plain cursor costs). A loop that stops early leaves up to this many rows
|
||||
# fetched; ``fetchone``/``fetchmany`` after a partial loop skip them.
|
||||
ITER_CHUNK = 128
|
||||
|
||||
|
||||
class ManagedCursor(sqlite3.Cursor):
|
||||
"""A cursor whose calls into SQLite count as its connection being in use."""
|
||||
|
||||
def execute(self, sql, parameters=(), /):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().execute(sql, parameters)
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def executemany(self, sql, seq_of_parameters, /):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().executemany(sql, seq_of_parameters)
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def executescript(self, sql_script, /):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().executescript(sql_script)
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def fetchone(self):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().fetchone()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def fetchmany(self, *args, **kwargs):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().fetchmany(*args, **kwargs)
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def fetchall(self):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().fetchall()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def __iter__(self) -> Iterator:
|
||||
return self._chunks()
|
||||
|
||||
def _chunks(self) -> Iterator:
|
||||
while True:
|
||||
rows = self.fetchmany(ITER_CHUNK)
|
||||
if not rows:
|
||||
return
|
||||
yield from rows
|
||||
|
||||
def __next__(self):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().__next__()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def close(self):
|
||||
conn = self.connection
|
||||
try:
|
||||
conn._enter()
|
||||
except sqlite3.ProgrammingError:
|
||||
return super().close() # the connection is closed or closing
|
||||
try:
|
||||
return super().close()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
|
||||
class ManagedConnection(sqlite3.Connection):
|
||||
"""A connection only its owner closes while a call into it is running."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
# Guards _depth, _retired, _closing and _cursors. Re-entrant so that
|
||||
# code running in the closing thread while it holds the lock (a
|
||||
# cursor() during the cursor-set copy) is refused instead of hanging.
|
||||
self._use_lock = threading.RLock()
|
||||
self._depth = 0
|
||||
self._retired = False
|
||||
self._closing = False
|
||||
self._released = threading.Event()
|
||||
# Python before 3.12 keeps the file open after close() while a cursor
|
||||
# still holds a statement; these are closed first (see _close_now).
|
||||
self._cursors: "weakref.WeakSet[sqlite3.Cursor]" = weakref.WeakSet()
|
||||
self.close_logger: Optional[logging.Logger] = None
|
||||
|
||||
# ── in use ───────────────────────────────────────────────────────
|
||||
|
||||
def _enter(self) -> None:
|
||||
with self._use_lock:
|
||||
if self._retired:
|
||||
raise sqlite3.ProgrammingError(CLOSED_MESSAGE)
|
||||
self._depth += 1
|
||||
|
||||
def _leave(self) -> None:
|
||||
with self._use_lock:
|
||||
self._depth -= 1
|
||||
close_now = self._retired and self._depth == 0
|
||||
if close_now:
|
||||
self._close_now()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def using(self):
|
||||
"""Mark the connection in use for a call this class cannot wrap.
|
||||
|
||||
For example as the *target* of another connection's ``backup()``.
|
||||
"""
|
||||
self._enter()
|
||||
try:
|
||||
yield self
|
||||
finally:
|
||||
self._leave()
|
||||
|
||||
def cursor(self, factory=None):
|
||||
with self._use_lock:
|
||||
if self._retired:
|
||||
raise sqlite3.ProgrammingError(CLOSED_MESSAGE)
|
||||
cursor = super().cursor(factory or ManagedCursor)
|
||||
self._cursors.add(cursor)
|
||||
return cursor
|
||||
|
||||
def execute(self, sql, parameters=(), /):
|
||||
return self.cursor().execute(sql, parameters)
|
||||
|
||||
def executemany(self, sql, seq_of_parameters, /):
|
||||
return self.cursor().executemany(sql, seq_of_parameters)
|
||||
|
||||
def executescript(self, sql_script, /):
|
||||
return self.cursor().executescript(sql_script)
|
||||
|
||||
def commit(self):
|
||||
self._enter()
|
||||
try:
|
||||
return super().commit()
|
||||
finally:
|
||||
self._leave()
|
||||
|
||||
def rollback(self):
|
||||
self._enter() # a closed connection raises, as sqlite3's does
|
||||
try:
|
||||
return super().rollback()
|
||||
finally:
|
||||
self._leave()
|
||||
|
||||
def backup(self, target, **kwargs):
|
||||
self._enter()
|
||||
try:
|
||||
return super().backup(target, **kwargs)
|
||||
finally:
|
||||
self._leave()
|
||||
|
||||
def create_function(self, *args, **kwargs):
|
||||
self._enter()
|
||||
try:
|
||||
return super().create_function(*args, **kwargs)
|
||||
finally:
|
||||
self._leave()
|
||||
|
||||
@property
|
||||
def in_transaction(self) -> bool:
|
||||
self._enter()
|
||||
try:
|
||||
return super().in_transaction
|
||||
finally:
|
||||
self._leave()
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
# The C implementation calls its own commit/rollback, not these.
|
||||
if exc_type is None:
|
||||
try:
|
||||
self.commit()
|
||||
except BaseException:
|
||||
with contextlib.suppress(sqlite3.Error):
|
||||
self.rollback()
|
||||
raise
|
||||
else:
|
||||
self.rollback()
|
||||
return False
|
||||
|
||||
# ── closing ──────────────────────────────────────────────────────
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close now if idle, else as soon as the running call ends.
|
||||
|
||||
Unlike ``sqlite3.Connection.close``, a call from another thread while
|
||||
the owner is inside a call returns before the file is closed. A caller
|
||||
that must know the file is closed (before renaming or deleting it)
|
||||
waits for ``wait_closed`` (``PromptModel.close_detached`` does).
|
||||
"""
|
||||
self.retire()
|
||||
|
||||
def retire(self) -> bool:
|
||||
"""Refuse new work and close; True when the connection is closed already.
|
||||
|
||||
Safe from any thread, and never raises: failures are logged.
|
||||
"""
|
||||
try:
|
||||
with self._use_lock:
|
||||
if not self._retired:
|
||||
self._retired = True
|
||||
if self._depth:
|
||||
# Ends the running statement (see the module docstring
|
||||
# for what it does not end). The owner cannot start
|
||||
# closing the connection while this lock is held.
|
||||
self.interrupt()
|
||||
return False
|
||||
elif self._depth:
|
||||
return False
|
||||
else:
|
||||
return self._released.wait(0)
|
||||
except Exception:
|
||||
self._log_exception("Error retiring database connection")
|
||||
return self._released.wait(0)
|
||||
self._close_now()
|
||||
return True
|
||||
|
||||
def wait_closed(self, timeout: float) -> bool:
|
||||
return self._released.wait(timeout)
|
||||
|
||||
def _close_now(self) -> None:
|
||||
# Only ever called with no call into the connection running, and a
|
||||
# retired connection refuses new calls and cursors: its cursors are
|
||||
# idle, and the set is copied under the lock that guards it.
|
||||
with self._use_lock:
|
||||
if self._closing:
|
||||
return # another thread closes it and sets _released
|
||||
self._closing = True
|
||||
cursors = list(self._cursors)
|
||||
try:
|
||||
for cursor in cursors:
|
||||
with contextlib.suppress(sqlite3.Error):
|
||||
sqlite3.Cursor.close(cursor)
|
||||
super().close()
|
||||
except Exception:
|
||||
self._log_exception("Error closing database connection")
|
||||
finally:
|
||||
self._released.set()
|
||||
|
||||
def _log_exception(self, message: str) -> None:
|
||||
if self.close_logger is not None:
|
||||
self.close_logger.exception(message)
|
||||
|
||||
|
||||
def retire_connections(
|
||||
connections: Iterable[sqlite3.Connection], logger=None
|
||||
) -> List[ManagedConnection]:
|
||||
"""Retire every connection; return those whose owner has yet to close them.
|
||||
|
||||
Safe while holding a lock the owners wait on: nothing here waits for them.
|
||||
One connection failing to retire is logged and never skips the rest.
|
||||
"""
|
||||
pending = []
|
||||
for conn in connections:
|
||||
try:
|
||||
if isinstance(conn, ManagedConnection):
|
||||
if logger is not None:
|
||||
conn.close_logger = logger
|
||||
if not conn.retire():
|
||||
pending.append(conn)
|
||||
else:
|
||||
conn.close()
|
||||
except Exception:
|
||||
if logger is not None:
|
||||
logger.exception("Error closing database connection")
|
||||
return pending
|
||||
|
||||
|
||||
def await_closed(
|
||||
connections: Iterable[ManagedConnection], timeout: float, logger=None
|
||||
) -> bool:
|
||||
"""Wait until the owners closed ``connections`` (retired); False on timeout.
|
||||
|
||||
A connection still open after ``timeout`` (its owner is blocked, for
|
||||
example on another process's lock after a lost interrupt) is logged and
|
||||
left to close when its call ends; a Move library then reports the old
|
||||
files as still in use instead of renaming them.
|
||||
"""
|
||||
deadline = time.monotonic() + timeout
|
||||
still_open = 0
|
||||
for conn in connections:
|
||||
if not conn.wait_closed(max(0.0, deadline - time.monotonic())):
|
||||
still_open += 1
|
||||
if still_open and logger is not None:
|
||||
logger.warning(
|
||||
f"{still_open} database connection(s) still in use after "
|
||||
f"{timeout:g}s; they close when their statement ends"
|
||||
)
|
||||
return not still_open
|
||||
+164
-23
@@ -15,9 +15,27 @@ from contextlib import closing
|
||||
from pathlib import Path
|
||||
from typing import Optional, Set, Tuple
|
||||
|
||||
from .managed_connection import (
|
||||
ManagedConnection,
|
||||
await_closed,
|
||||
retire_connections,
|
||||
)
|
||||
from .library_schema import library_schema_error, migrate_library_state
|
||||
from .image_favorites import image_favorites_schema_error, migrate_image_favorites
|
||||
from .annotation_schema import annotation_schema_error, migrate_annotation_state
|
||||
from .review_schema import (
|
||||
migrate_review_state,
|
||||
review_schema_error,
|
||||
review_without_annotations_error,
|
||||
)
|
||||
from .saved_views import migrate_saved_views, saved_views_schema_error
|
||||
from .workflow_documents import (
|
||||
DOCUMENT_ID_COLUMNS,
|
||||
migrate_workflow_documents,
|
||||
workflow_documents_schema_error,
|
||||
)
|
||||
|
||||
|
||||
from .execution_history_schema import (
|
||||
history_schema_error,
|
||||
migrate_execution_history,
|
||||
@@ -44,6 +62,19 @@ MAX_PAIR_PROVENANCE_LENGTH = 4_000_000
|
||||
MAX_PAIR_PROVENANCE_DEPTH = 64
|
||||
|
||||
|
||||
def _refuse_rebuild_with_documents(columns) -> None:
|
||||
"""The legacy generated_images rebuilds copy a fixed column list.
|
||||
|
||||
A file that needs one (``prompt_id`` TEXT, no ``file_path``) but already
|
||||
has the workflow document ids would lose them, so it is refused.
|
||||
"""
|
||||
if any(column in columns for column in DOCUMENT_ID_COLUMNS):
|
||||
raise sqlite3.DatabaseError(
|
||||
"This database needs an old image table rebuild but already has "
|
||||
"workflow document references; the rebuild would drop them"
|
||||
)
|
||||
|
||||
|
||||
def pair_provenance_error(encoded: object) -> Optional[str]:
|
||||
"""Validate the same bounded JSON object for live writes and restores."""
|
||||
if not isinstance(encoded, str):
|
||||
@@ -302,6 +333,16 @@ class PromptModel:
|
||||
migrate_library_state(conn)
|
||||
migrate_image_favorites(conn)
|
||||
migrate_saved_views(conn)
|
||||
# A review group without its annotation group is refused before the
|
||||
# annotation install could make it look valid (catalog only; the rows
|
||||
# are validated once, in migrate_review_state).
|
||||
review_error = review_without_annotations_error(conn)
|
||||
if review_error:
|
||||
raise sqlite3.DatabaseError(review_error)
|
||||
migrate_annotation_state(conn)
|
||||
migrate_review_state(conn)
|
||||
# Last: the older generated_images rebuilds run only before file_path.
|
||||
migrate_workflow_documents(conn)
|
||||
|
||||
def _migrate_prompt_pairs(self, conn: sqlite3.Connection) -> None:
|
||||
conn.execute("SAVEPOINT prompt_pair_schema")
|
||||
@@ -573,19 +614,42 @@ class PromptModel:
|
||||
if conn is not None and self._local.generation == self._generation:
|
||||
return conn
|
||||
|
||||
# check_same_thread=False only so close_all can clean up other
|
||||
# threads' connections; each connection is still used by one thread.
|
||||
conn = sqlite3.connect(
|
||||
self.db_path, check_same_thread=False, timeout=BUSY_TIMEOUT_SECONDS
|
||||
)
|
||||
conn.row_factory = sqlite3.Row
|
||||
configure_connection(conn)
|
||||
while True:
|
||||
# The path and generation are read together, and the new
|
||||
# connection is kept only if no Move library repointed the model
|
||||
# meanwhile: otherwise it would write to the old file.
|
||||
with self._conn_lock:
|
||||
path, generation = self.db_path, self._generation
|
||||
# check_same_thread=False only so close_all and Move library can
|
||||
# retire other threads' connections; each connection is still
|
||||
# used by one thread, and ManagedConnection never closes it under
|
||||
# a running statement (see managed_connection).
|
||||
conn = sqlite3.connect(
|
||||
path,
|
||||
check_same_thread=False,
|
||||
timeout=BUSY_TIMEOUT_SECONDS,
|
||||
factory=ManagedConnection,
|
||||
)
|
||||
try:
|
||||
conn.row_factory = sqlite3.Row
|
||||
configure_connection(conn)
|
||||
except BaseException:
|
||||
conn.close()
|
||||
raise
|
||||
with self._conn_lock:
|
||||
if generation == self._generation:
|
||||
self._connections.add(conn)
|
||||
break
|
||||
conn.close()
|
||||
self._local.conn = conn
|
||||
self._local.generation = self._generation
|
||||
with self._conn_lock:
|
||||
self._connections.add(conn)
|
||||
self._local.generation = generation
|
||||
return conn
|
||||
|
||||
@property
|
||||
def generation(self) -> int:
|
||||
"""Bumped whenever every connection is closed or the model repointed."""
|
||||
return self._generation
|
||||
|
||||
@classmethod
|
||||
def close_all_instances(cls) -> None:
|
||||
"""Close every connection of every live model in this process."""
|
||||
@@ -594,22 +658,76 @@ class PromptModel:
|
||||
for model in models:
|
||||
model.close_all()
|
||||
|
||||
@classmethod
|
||||
def repoint_instances(cls, old_path: str, new_path: str) -> list:
|
||||
"""Point every model of ``old_path`` at ``new_path`` (Move library).
|
||||
|
||||
Every thread opens the new file on its next ``get_connection``. The
|
||||
old connections are detached, not closed, and returned: the caller
|
||||
retires them (``retire_detached``) while it still holds the old
|
||||
file's write lock. A write waiting on that lock is then interrupted
|
||||
and can never commit, so it fails (and, through
|
||||
``PromptDatabase.survive_library_switch``, runs again on the new file).
|
||||
"""
|
||||
old_key = os.path.normcase(os.path.realpath(old_path))
|
||||
with cls._instances_lock:
|
||||
models = list(cls._instances)
|
||||
with cls._init_lock:
|
||||
cls._initialized_paths.add(os.path.normcase(os.path.realpath(new_path)))
|
||||
detached = []
|
||||
for model in models:
|
||||
if model._schema_key() == old_key:
|
||||
detached.extend(model._detach_connections(new_path))
|
||||
return detached
|
||||
|
||||
@staticmethod
|
||||
def retire_detached(connections, logger=None) -> list:
|
||||
"""Retire detached connections without waiting for their owners.
|
||||
|
||||
Idle ones close now. One whose owner thread is inside a statement is
|
||||
never closed from here (that frees memory the owner is using: an
|
||||
access violation on Windows); it is interrupted, refuses to commit and
|
||||
is closed by its owner as the statement ends. Returns those still
|
||||
open, for ``await_detached`` once no lock the owners wait on is held.
|
||||
"""
|
||||
return retire_connections(connections, logger)
|
||||
|
||||
@staticmethod
|
||||
def await_detached(connections, logger=None) -> bool:
|
||||
"""Wait until retired connections are closed; False if some are not."""
|
||||
return await_closed(connections, BUSY_TIMEOUT_SECONDS, logger)
|
||||
|
||||
@staticmethod
|
||||
def close_detached(connections, logger=None) -> bool:
|
||||
"""Retire connections ``repoint_instances`` detached and await them.
|
||||
|
||||
Do not call it while holding a lock their owners may wait on; retire
|
||||
them there (``retire_detached``) and await them after releasing it.
|
||||
"""
|
||||
pending = retire_connections(connections, logger)
|
||||
return await_closed(pending, BUSY_TIMEOUT_SECONDS, logger)
|
||||
|
||||
def close_all(self) -> None:
|
||||
"""Close every thread's connection; they reopen lazily on next use.
|
||||
|
||||
Needed before deleting the database file (Windows keeps open files
|
||||
locked) and by tests that ran queries through executor threads.
|
||||
"""
|
||||
self.close_detached(self._detach_connections(), self.logger)
|
||||
|
||||
def _detach_connections(self, new_path: Optional[str] = None) -> list:
|
||||
"""Forget every connection (optionally pointing the model elsewhere).
|
||||
|
||||
Returns them for the caller to close; threads reopen on next use.
|
||||
"""
|
||||
with self._conn_lock:
|
||||
if new_path is not None:
|
||||
self.db_path = new_path
|
||||
connections = list(self._connections)
|
||||
self._connections.clear()
|
||||
self._generation += 1
|
||||
self._local.conn = None
|
||||
for conn in connections:
|
||||
try:
|
||||
conn.close()
|
||||
except sqlite3.Error as e:
|
||||
self.logger.warning(f"Error closing database connection: {e}")
|
||||
return connections
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the calling thread's connection, if it has one.
|
||||
@@ -623,10 +741,7 @@ class PromptModel:
|
||||
self._local.conn = None
|
||||
with self._conn_lock:
|
||||
self._connections.discard(conn)
|
||||
try:
|
||||
conn.close()
|
||||
except sqlite3.Error as e:
|
||||
self.logger.warning(f"Error closing database connection: {e}")
|
||||
self.close_detached([conn], self.logger)
|
||||
|
||||
def _migrate_workflow_name_removal(self, conn: sqlite3.Connection) -> None:
|
||||
"""
|
||||
@@ -697,6 +812,7 @@ class PromptModel:
|
||||
columns = {column[1]: column[2] for column in cursor.fetchall()}
|
||||
if columns.get("prompt_id") != "TEXT":
|
||||
return
|
||||
_refuse_rebuild_with_documents(columns)
|
||||
|
||||
self.logger.info("Migrating foreign key types: prompt_id TEXT -> INTEGER")
|
||||
with self._table_rebuild(conn):
|
||||
@@ -862,6 +978,7 @@ class PromptModel:
|
||||
}
|
||||
if "file_path" in columns:
|
||||
return
|
||||
_refuse_rebuild_with_documents(columns)
|
||||
|
||||
self.logger.info("Migrating database: image uniqueness by full path")
|
||||
with self._table_rebuild(conn):
|
||||
@@ -927,6 +1044,10 @@ class PromptModel:
|
||||
|
||||
Reclaims unused space and defragments the database file,
|
||||
improving query performance and reducing file size.
|
||||
|
||||
Raises:
|
||||
Exception: whatever stopped VACUUM (logged first); the database is
|
||||
then unchanged.
|
||||
"""
|
||||
try:
|
||||
with closing(sqlite3.connect(self.db_path)) as conn:
|
||||
@@ -934,6 +1055,7 @@ class PromptModel:
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error vacuuming database: {e}")
|
||||
raise
|
||||
|
||||
def get_database_info(self) -> dict:
|
||||
"""
|
||||
@@ -1021,9 +1143,9 @@ class PromptModel:
|
||||
Args:
|
||||
path: File to inspect
|
||||
require_library_state: Also require the tables every current
|
||||
database has (Library state, image favorites and saved
|
||||
views), as a migrated restore stage must; old backups may
|
||||
lack them.
|
||||
database has (Library state, image favorites, saved views,
|
||||
image annotations and the review queue), as a migrated
|
||||
restore stage must; old backups may lack them.
|
||||
|
||||
Returns:
|
||||
(ok, reason): reason is "ok" when the file is acceptable, otherwise
|
||||
@@ -1088,6 +1210,21 @@ class PromptModel:
|
||||
)
|
||||
if views_error:
|
||||
return False, views_error
|
||||
annotation_error = annotation_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if annotation_error:
|
||||
return False, annotation_error
|
||||
review_error = review_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if review_error:
|
||||
return False, review_error
|
||||
documents_error = workflow_documents_schema_error(
|
||||
conn, allow_missing=not require_library_state, documents=True
|
||||
)
|
||||
if documents_error:
|
||||
return False, documents_error
|
||||
return True, "ok"
|
||||
except sqlite3.Error as e:
|
||||
return False, f"Invalid SQLite database: {e}"
|
||||
@@ -1110,7 +1247,11 @@ class PromptModel:
|
||||
Path(src_path).resolve().as_uri() + "?mode=ro", uri=True
|
||||
)
|
||||
try:
|
||||
source.backup(self.get_connection())
|
||||
target = self.get_connection()
|
||||
# The target is written under backup's step too: mark it in use
|
||||
# so a concurrent close_all retires it instead of closing it.
|
||||
with target.using():
|
||||
source.backup(target)
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
|
||||
+451
-253
File diff suppressed because it is too large
Load Diff
+169
-109
@@ -1,16 +1,26 @@
|
||||
"""Atomic metadata enrichment for a single known generation context."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
|
||||
from .models import normalize_image_path
|
||||
from .operations import _coerce_file_info
|
||||
from .workflow_documents import (
|
||||
TEXT_COLUMNS,
|
||||
DocumentReader,
|
||||
release_documents,
|
||||
resolve_image_row,
|
||||
stored_values,
|
||||
)
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
from ..utils.json_strict import dumps_strict
|
||||
from ..utils.prompt_pair_resolver import RESOLVER_VERSION
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.json_strict import dumps_strict
|
||||
from utils.prompt_pair_resolver import RESOLVER_VERSION
|
||||
|
||||
|
||||
def ingest_resolved_pair(
|
||||
@@ -19,118 +29,168 @@ def ingest_resolved_pair(
|
||||
state = resolution.get("state", "unresolved")
|
||||
if state != "resolved":
|
||||
return {"status": state, "pairs_added": 0, "links_added": 0}
|
||||
with db._pair_transaction() as conn:
|
||||
return _ingest_resolved_pair_in(conn, db, image_path, metadata, resolution)
|
||||
|
||||
|
||||
def _conflict() -> Dict[str, Any]:
|
||||
return {"status": "conflict", "pairs_added": 0, "links_added": 0}
|
||||
|
||||
|
||||
def _ingest_resolved_pair_in(
|
||||
conn: Any,
|
||||
db: Any,
|
||||
image_path: str,
|
||||
metadata: Dict[str, Any],
|
||||
resolution: Dict[str, Any],
|
||||
*,
|
||||
provenance: Dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""``ingest_resolved_pair`` inside the caller's open ``_pair_transaction``.
|
||||
|
||||
The compatible-only rules are the public ingester's: an image with more
|
||||
than one exact row, another positive, or a prior pair with another
|
||||
negative is a ``conflict`` and nothing is written. The caller owns the
|
||||
transaction, so its own later failure rolls the link back too.
|
||||
``provenance`` replaces the link's default resolver provenance (Needs
|
||||
review records the user's selection there).
|
||||
"""
|
||||
positive, negative = resolution["positive"], resolution["negative"]
|
||||
positive_text = positive.get("storage_text", positive["text"])
|
||||
negative_text = negative.get("storage_text", negative["text"])
|
||||
normalized = normalize_image_path(image_path)
|
||||
provenance = {"resolver_version": 1, "positive": positive, "negative": negative}
|
||||
encoded = json.dumps(
|
||||
provenance, sort_keys=True, separators=(",", ":"), allow_nan=False
|
||||
)
|
||||
with db._pair_transaction() as conn:
|
||||
images = conn.execute(
|
||||
"SELECT gi.*,p.hash AS positive_hash FROM generated_images gi "
|
||||
"JOIN prompts p ON p.id=gi.prompt_id "
|
||||
"WHERE gi.file_path=? OR gi.image_path=?",
|
||||
(normalized, image_path),
|
||||
if provenance is None:
|
||||
provenance = {
|
||||
"resolver_version": RESOLVER_VERSION,
|
||||
"positive": positive,
|
||||
"negative": negative,
|
||||
}
|
||||
encoded = dumps_strict(provenance, sort_keys=True, separators=(",", ":"))
|
||||
images = conn.execute(
|
||||
"SELECT gi.*,p.hash AS positive_hash FROM generated_images gi "
|
||||
"JOIN prompts p ON p.id=gi.prompt_id "
|
||||
"WHERE gi.file_path=? OR gi.image_path=?",
|
||||
(normalized, image_path),
|
||||
).fetchall()
|
||||
if len(images) > 1 or (
|
||||
images and images[0]["positive_hash"] != generate_prompt_hash(positive_text)
|
||||
):
|
||||
return _conflict()
|
||||
existing = images[0] if images else None
|
||||
if existing:
|
||||
prior = conn.execute(
|
||||
"SELECT n.hash AS negative_hash FROM image_prompt_pairs ip "
|
||||
"JOIN prompt_pairs pp ON pp.id=ip.pair_id "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id WHERE ip.image_id=?",
|
||||
(existing["id"],),
|
||||
).fetchall()
|
||||
if len(images) > 1 or (
|
||||
images and images[0]["positive_hash"] != generate_prompt_hash(positive_text)
|
||||
):
|
||||
return {"status": "conflict", "pairs_added": 0, "links_added": 0}
|
||||
existing = images[0] if images else None
|
||||
if existing:
|
||||
prior = conn.execute(
|
||||
"SELECT n.hash AS negative_hash FROM image_prompt_pairs ip "
|
||||
"JOIN prompt_pairs pp ON pp.id=ip.pair_id "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id WHERE ip.image_id=?",
|
||||
(existing["id"],),
|
||||
).fetchall()
|
||||
expected = (
|
||||
generate_prompt_hash(negative_text) if negative_text.strip() else None
|
||||
)
|
||||
if any(row["negative_hash"] != expected for row in prior):
|
||||
return {"status": "conflict", "pairs_added": 0, "links_added": 0}
|
||||
prior_pair = conn.execute(
|
||||
"SELECT pp.id FROM prompt_pairs pp "
|
||||
"JOIN prompts p ON p.id=pp.positive_prompt_id "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
|
||||
"WHERE p.hash=? AND n.hash IS ?",
|
||||
(
|
||||
generate_prompt_hash(positive_text),
|
||||
generate_prompt_hash(negative_text) if negative_text.strip() else None,
|
||||
),
|
||||
).fetchone()
|
||||
pair_id = db._save_pair(conn, positive_text, negative_text)
|
||||
pair = conn.execute(
|
||||
"SELECT * FROM prompt_pairs WHERE id=?", (pair_id,)
|
||||
).fetchone()
|
||||
pairs_added = int(prior_pair is None)
|
||||
file_size, width, height, image_format = _coerce_file_info(metadata)
|
||||
snapshots = {
|
||||
column: json.dumps(metadata[key], sort_keys=True, allow_nan=False)
|
||||
for column, key in (
|
||||
("workflow_data", "workflow"),
|
||||
("prompt_metadata", "prompt"),
|
||||
("parameters", "parameters"),
|
||||
)
|
||||
if key in metadata
|
||||
}
|
||||
info = {
|
||||
"file_size": file_size,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"format": image_format,
|
||||
}
|
||||
snapshots.update(
|
||||
{key: value for key, value in info.items() if value is not None}
|
||||
expected = (
|
||||
generate_prompt_hash(negative_text) if negative_text.strip() else None
|
||||
)
|
||||
changed = False
|
||||
if existing:
|
||||
image_id = existing["id"]
|
||||
updates = {
|
||||
key: value for key, value in snapshots.items() if existing[key] != value
|
||||
}
|
||||
if updates:
|
||||
conn.execute(
|
||||
"UPDATE generated_images SET "
|
||||
+ ",".join(f"{key}=?" for key in updates)
|
||||
+ " WHERE id=?",
|
||||
[*updates.values(), image_id],
|
||||
)
|
||||
changed = True
|
||||
else:
|
||||
columns = ["prompt_id", "image_path", "filename", "file_path", *snapshots]
|
||||
values = [
|
||||
pair["positive_prompt_id"],
|
||||
image_path,
|
||||
os.path.basename(image_path),
|
||||
normalized,
|
||||
*snapshots.values(),
|
||||
if any(row["negative_hash"] != expected for row in prior):
|
||||
return _conflict()
|
||||
prior_pair = conn.execute(
|
||||
"SELECT pp.id FROM prompt_pairs pp "
|
||||
"JOIN prompts p ON p.id=pp.positive_prompt_id "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
|
||||
"WHERE p.hash=? AND n.hash IS ?",
|
||||
(
|
||||
generate_prompt_hash(positive_text),
|
||||
generate_prompt_hash(negative_text) if negative_text.strip() else None,
|
||||
),
|
||||
).fetchone()
|
||||
pair_id = db._save_pair(conn, positive_text, negative_text)
|
||||
pair = conn.execute("SELECT * FROM prompt_pairs WHERE id=?", (pair_id,)).fetchone()
|
||||
pairs_added = int(prior_pair is None)
|
||||
file_size, width, height, image_format = _coerce_file_info(metadata)
|
||||
# ComfyUI's NaN/Infinity are stored as null: the snapshot is strict JSON.
|
||||
snapshots = {
|
||||
column: dumps_strict(metadata[key], sort_keys=True)
|
||||
for column, key in (
|
||||
("workflow_data", "workflow"),
|
||||
("prompt_metadata", "prompt"),
|
||||
("parameters", "parameters"),
|
||||
)
|
||||
if key in metadata
|
||||
}
|
||||
info = {
|
||||
"file_size": file_size,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"format": image_format,
|
||||
}
|
||||
snapshots.update({key: value for key, value in info.items() if value is not None})
|
||||
changed = False
|
||||
if existing:
|
||||
image_id = existing["id"]
|
||||
# Compared as stored text, whichever form the row is in.
|
||||
current = resolve_image_row(DocumentReader(conn), existing)
|
||||
updates = {
|
||||
key: value for key, value in snapshots.items() if current[key] != value
|
||||
}
|
||||
if updates:
|
||||
replaced = [
|
||||
existing[reference]
|
||||
for column, reference in TEXT_COLUMNS
|
||||
if column in updates
|
||||
]
|
||||
image_id = conn.execute(
|
||||
"INSERT INTO generated_images("
|
||||
+ ",".join(columns)
|
||||
+ ") VALUES("
|
||||
+ ",".join("?" for _ in columns)
|
||||
+ ")",
|
||||
values,
|
||||
).lastrowid
|
||||
linked = conn.execute(
|
||||
"SELECT 1 FROM image_prompt_pairs WHERE pair_id=? "
|
||||
"AND image_id=? AND provenance=?",
|
||||
(pair_id, image_id, encoded),
|
||||
).fetchone()
|
||||
db._link_image_to_pair(conn, pair_id, image_id, provenance)
|
||||
status = (
|
||||
"added"
|
||||
if existing is None
|
||||
else "enriched" if changed or not linked else "unchanged"
|
||||
)
|
||||
return {
|
||||
"status": status,
|
||||
"pairs_added": pairs_added,
|
||||
"links_added": int(not linked),
|
||||
"pair_id": pair_id,
|
||||
"image_id": image_id,
|
||||
}
|
||||
updates = _document_form(conn, updates)
|
||||
conn.execute(
|
||||
"UPDATE generated_images SET "
|
||||
+ ",".join(f"{key}=?" for key in updates)
|
||||
+ " WHERE id=?",
|
||||
[*updates.values(), image_id],
|
||||
)
|
||||
release_documents(conn, replaced)
|
||||
changed = True
|
||||
else:
|
||||
stored = _document_form(conn, snapshots)
|
||||
columns = ["prompt_id", "image_path", "filename", "file_path", *stored]
|
||||
values = [
|
||||
pair["positive_prompt_id"],
|
||||
image_path,
|
||||
os.path.basename(image_path),
|
||||
normalized,
|
||||
*stored.values(),
|
||||
]
|
||||
image_id = conn.execute(
|
||||
"INSERT INTO generated_images("
|
||||
+ ",".join(columns)
|
||||
+ ") VALUES("
|
||||
+ ",".join("?" for _ in columns)
|
||||
+ ")",
|
||||
values,
|
||||
).lastrowid
|
||||
linked = conn.execute(
|
||||
"SELECT 1 FROM image_prompt_pairs WHERE pair_id=? "
|
||||
"AND image_id=? AND provenance=?",
|
||||
(pair_id, image_id, encoded),
|
||||
).fetchone()
|
||||
db._link_image_to_pair(conn, pair_id, image_id, provenance)
|
||||
status = (
|
||||
"added"
|
||||
if existing is None
|
||||
else "enriched" if changed or not linked else "unchanged"
|
||||
)
|
||||
return {
|
||||
"status": status,
|
||||
"pairs_added": pairs_added,
|
||||
"links_added": int(not linked),
|
||||
"pair_id": pair_id,
|
||||
"image_id": image_id,
|
||||
}
|
||||
|
||||
|
||||
def _document_form(conn: Any, values: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""``values`` with the workflow and prompt graph texts stored once.
|
||||
|
||||
Each of those columns becomes NULL beside its document id (new form);
|
||||
every other column is unchanged.
|
||||
"""
|
||||
texts = {
|
||||
column: values[column]
|
||||
for column, _reference in TEXT_COLUMNS
|
||||
if column in values
|
||||
}
|
||||
rest = {key: value for key, value in values.items() if key not in texts}
|
||||
return {**rest, **stored_values(conn, texts)}
|
||||
|
||||
@@ -0,0 +1,648 @@
|
||||
"""Needs review items (#249): persist and read capture-bound ambiguous evidence.
|
||||
|
||||
An item is one ambiguous resolver result for one annotation byte record (the
|
||||
whole-file SHA-256 of one physical file at one locator), keyed by
|
||||
``(byte_id, resolver_version, evidence_hash)``. Recording is idempotent: the
|
||||
same evidence for the same bytes is a duplicate and never touches an existing
|
||||
item, its state, revision or decisions. Changed bytes are a new byte record
|
||||
and therefore a new pending item; the old one stays.
|
||||
|
||||
Callers pass verified facts (the capture's digest, size and ``mtime_ns``);
|
||||
nothing here reads the filesystem. Writes run in one ``_pair_transaction``
|
||||
fenced by the storage epoch the capture was taken under.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Iterator
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
from .annotation_schema import (
|
||||
BYTES_TABLE,
|
||||
FILES_TABLE,
|
||||
METADATA_TABLE as ANNOTATION_METADATA_TABLE,
|
||||
)
|
||||
from .annotations import _new_id, _validate_locator, _validate_target
|
||||
from .models import normalize_image_path, utc_now_iso
|
||||
from .review_changes import notify_review_change
|
||||
from .review_schema import (
|
||||
DECISIONS_TABLE,
|
||||
ITEMS_TABLE,
|
||||
MAX_CANDIDATES,
|
||||
MAX_SPELLED_PATH_BYTES,
|
||||
METADATA_TABLE,
|
||||
canonical_evidence,
|
||||
evidence_digest,
|
||||
evidence_offers_choice,
|
||||
)
|
||||
|
||||
# Only this module's table constants are interpolated into the SQL below.
|
||||
_INSERT_ITEM = (
|
||||
f"INSERT OR IGNORE INTO {ITEMS_TABLE}(id,byte_id,spelled_path," # nosec B608
|
||||
"resolver_version,evidence_hash,evidence,candidate_count,limited,state,"
|
||||
"revision,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,'pending',0,?,?)"
|
||||
)
|
||||
_BUMP_REVIEW_INDEX = (
|
||||
f"UPDATE {METADATA_TABLE} SET " # nosec B608
|
||||
"review_index_revision=review_index_revision+1 WHERE singleton_id=1"
|
||||
)
|
||||
# A later spelling of the same locator (a case-only rename on a case-folding
|
||||
# filesystem) replaces the recorded one, so targets keep formatting.
|
||||
_REFRESH_SPELLING = (
|
||||
f"UPDATE {ITEMS_TABLE} SET spelled_path=? " # nosec B608
|
||||
f"WHERE spelled_path<>? AND byte_id IN (SELECT b.id FROM {BYTES_TABLE} b "
|
||||
f"JOIN {FILES_TABLE} f ON f.id=b.file_id WHERE f.locator=?)"
|
||||
)
|
||||
_REVIEW_INDEX = (
|
||||
f"SELECT review_index_revision FROM {METADATA_TABLE} " # nosec B608
|
||||
"WHERE singleton_id=1"
|
||||
)
|
||||
_BUMP_IMAGE_INDEX = (
|
||||
f"UPDATE {ANNOTATION_METADATA_TABLE} SET " # nosec B608
|
||||
"image_index_revision=image_index_revision+1 WHERE singleton_id=1"
|
||||
)
|
||||
_ITEMS_OF_LOCATOR = (
|
||||
"SELECT i.id,i.byte_id,i.resolver_version,i.evidence_hash," # nosec B608
|
||||
"i.evidence,i.candidate_count,i.limited,i.state,i.revision,i.created_at,"
|
||||
f"i.updated_at,b.sha256 FROM {FILES_TABLE} f "
|
||||
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
||||
"WHERE f.locator=? ORDER BY i.created_at,i.id"
|
||||
)
|
||||
_QUEUED = (
|
||||
f"SELECT 1 FROM {FILES_TABLE} f " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
||||
"WHERE f.locator=? AND b.size=? AND b.mtime_ns=? AND i.resolver_version=? "
|
||||
"LIMIT 1"
|
||||
)
|
||||
_PENDING_AT = (
|
||||
f"SELECT 1 FROM {FILES_TABLE} f " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
||||
"WHERE f.locator=? AND b.size=? AND b.mtime_ns=? AND i.resolver_version=? "
|
||||
"AND i.state='pending' LIMIT 1"
|
||||
)
|
||||
# Bytes verified at this stat whose item carries a decision (any version).
|
||||
_DECIDED_AT = (
|
||||
f"SELECT 1 FROM {FILES_TABLE} f " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
||||
"WHERE f.locator=? AND b.size=? AND b.mtime_ns=? "
|
||||
"AND (i.state<>'pending' OR i.revision<>0 OR EXISTS (SELECT 1 FROM "
|
||||
f"{DECISIONS_TABLE} d WHERE d.item_id=i.id)) LIMIT 1"
|
||||
)
|
||||
# The other items of one byte record (another version or evidence), with
|
||||
# whether each may be superseded: pending, revision 0, no decision row.
|
||||
_OTHER_ITEMS_OF_BYTES = (
|
||||
"SELECT i.id, (i.state='pending' AND i.revision=0 AND NOT EXISTS " # nosec B608
|
||||
f"(SELECT 1 FROM {DECISIONS_TABLE} d WHERE d.item_id=i.id)) "
|
||||
f"FROM {ITEMS_TABLE} i WHERE i.byte_id=? "
|
||||
"AND NOT (i.resolver_version=? AND i.evidence_hash=?)"
|
||||
)
|
||||
# Pending items of the bytes verified at one stat, with their versions.
|
||||
_PENDING_AT_STAT = (
|
||||
f"SELECT i.id,i.resolver_version FROM {FILES_TABLE} f " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
||||
"WHERE f.locator=? AND b.size=? AND b.mtime_ns=? AND i.state='pending'"
|
||||
)
|
||||
# Pending items of one file: the candidates a link settles.
|
||||
_PENDING_OF_LOCATOR = (
|
||||
f"SELECT i.id FROM {FILES_TABLE} f " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
||||
"WHERE f.locator=? AND i.state='pending'"
|
||||
)
|
||||
# Pending items of one file with their evidence: an untraced file's settle.
|
||||
_PENDING_EVIDENCE_OF_LOCATOR = (
|
||||
f"SELECT i.id,i.evidence FROM {FILES_TABLE} f " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"JOIN {ITEMS_TABLE} i ON i.byte_id=b.id "
|
||||
"WHERE f.locator=? AND i.state='pending'"
|
||||
)
|
||||
# Only an undecided item goes: a decision row (RESTRICT) is never deleted.
|
||||
_SETTLE_ITEM = (
|
||||
f"DELETE FROM {ITEMS_TABLE} WHERE id=? AND state='pending' " # nosec B608
|
||||
f"AND revision=0 AND NOT EXISTS (SELECT 1 FROM {DECISIONS_TABLE} d "
|
||||
f"WHERE d.item_id={ITEMS_TABLE}.id)"
|
||||
)
|
||||
_EPOCH = "SELECT storage_epoch FROM execution_history_metadata WHERE singleton_id=1"
|
||||
_PENDING_COUNT = (
|
||||
f"SELECT COUNT(*) FROM {ITEMS_TABLE} WHERE state='pending'" # nosec B608
|
||||
)
|
||||
_QUEUE_COLUMNS = (
|
||||
"id",
|
||||
"state",
|
||||
"revision",
|
||||
"created_at",
|
||||
"candidate_count",
|
||||
"limited",
|
||||
"spelled_path",
|
||||
"locator",
|
||||
)
|
||||
_QUEUE_SELECT = (
|
||||
"SELECT i.id,i.state,i.revision,i.created_at,i.candidate_count," # nosec B608
|
||||
f"i.limited,i.spelled_path,f.locator FROM {ITEMS_TABLE} i "
|
||||
f"JOIN {BYTES_TABLE} b ON b.id=i.byte_id "
|
||||
f"JOIN {FILES_TABLE} f ON f.id=b.file_id "
|
||||
)
|
||||
_QUEUE_FIRST = _QUEUE_SELECT + (
|
||||
"WHERE i.state='pending' ORDER BY i.created_at,i.id LIMIT ?"
|
||||
)
|
||||
_QUEUE_AFTER = _QUEUE_SELECT + (
|
||||
"WHERE i.state='pending' AND (i.created_at,i.id)>(?,?) "
|
||||
"ORDER BY i.created_at,i.id LIMIT ?"
|
||||
)
|
||||
_DETAIL_COLUMNS = _QUEUE_COLUMNS + (
|
||||
"byte_id",
|
||||
"resolver_version",
|
||||
"evidence_hash",
|
||||
"evidence",
|
||||
"updated_at",
|
||||
"sha256",
|
||||
)
|
||||
_DETAIL = (
|
||||
"SELECT i.id,i.state,i.revision,i.created_at,i.candidate_count," # nosec B608
|
||||
"i.limited,i.spelled_path,f.locator,i.byte_id,i.resolver_version,"
|
||||
"i.evidence_hash,i.evidence,i.updated_at,b.sha256 "
|
||||
f"FROM {ITEMS_TABLE} i JOIN {BYTES_TABLE} b ON b.id=i.byte_id "
|
||||
f"JOIN {FILES_TABLE} f ON f.id=b.file_id WHERE i.id=?"
|
||||
)
|
||||
# The legacy rows and pair links the ingester's compatibility rules read.
|
||||
_DEPENDENT_IMAGES = (
|
||||
"SELECT gi.id,p.hash,gi.image_path FROM generated_images gi "
|
||||
"JOIN prompts p ON p.id=gi.prompt_id "
|
||||
"WHERE gi.file_path=? OR gi.image_path=? ORDER BY gi.id"
|
||||
)
|
||||
_DEPENDENT_LINKS = (
|
||||
"SELECT DISTINCT ip.pair_id,n.hash FROM image_prompt_pairs ip "
|
||||
"JOIN prompt_pairs pp ON pp.id=ip.pair_id "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
|
||||
"WHERE ip.image_id=? ORDER BY ip.pair_id"
|
||||
)
|
||||
# Rows whose spelling ends with the file's name: the candidates for another
|
||||
# spelling of the same physical file (the link summaries' filename fallback).
|
||||
_SAME_NAME_ROWS = (
|
||||
"SELECT id,image_path FROM generated_images "
|
||||
"WHERE image_path LIKE ? ESCAPE '\\' ORDER BY id"
|
||||
)
|
||||
_TITLES = (
|
||||
"SELECT p.hash,s.title FROM prompts p "
|
||||
"JOIN prompt_library_state s ON s.prompt_id=p.id "
|
||||
"WHERE p.hash IN ({marks}) AND s.title IS NOT NULL ORDER BY p.id"
|
||||
)
|
||||
_ITEM_ID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
|
||||
_ITEM_COLUMNS = (
|
||||
"id",
|
||||
"byte_id",
|
||||
"resolver_version",
|
||||
"evidence_hash",
|
||||
"evidence",
|
||||
"candidate_count",
|
||||
"limited",
|
||||
"state",
|
||||
"revision",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"sha256",
|
||||
)
|
||||
|
||||
|
||||
def _validate_spelling(spelled_path: object, locator: str) -> None:
|
||||
if (
|
||||
not isinstance(spelled_path, str)
|
||||
or not spelled_path
|
||||
or len(spelled_path.encode("utf-8", "surrogatepass")) > MAX_SPELLED_PATH_BYTES
|
||||
or os.path.normcase(spelled_path) != locator
|
||||
):
|
||||
raise ValueError("spelled_path must be the file's spelling of its locator")
|
||||
|
||||
|
||||
def dependent_images(conn: sqlite3.Connection, spellings) -> list:
|
||||
"""``(id, image_path, positive_hash)`` of every legacy row of the file.
|
||||
|
||||
``spellings`` are the paths the file is linked under (the Apply path and
|
||||
every alias of the same physical file); each is matched as the ingester
|
||||
matches one path. Rows are distinct and ordered by id.
|
||||
"""
|
||||
found = {}
|
||||
for spelled in dict.fromkeys(spellings):
|
||||
for row in conn.execute(
|
||||
_DEPENDENT_IMAGES, (normalize_image_path(spelled), spelled)
|
||||
):
|
||||
found[row[0]] = (row[0], row[2], row[1])
|
||||
return [found[key] for key in sorted(found)]
|
||||
|
||||
|
||||
def same_name_rows(conn: sqlite3.Connection, name: str) -> list:
|
||||
"""``(id, image_path)`` of rows whose spelling may end with ``name``."""
|
||||
# Imported here: operations composes this module's mixin.
|
||||
from .operations import escape_like
|
||||
|
||||
return [
|
||||
(row[0], row[1])
|
||||
for row in conn.execute(_SAME_NAME_ROWS, (f"%{escape_like(name)}",))
|
||||
]
|
||||
|
||||
|
||||
def dependency_fingerprint(
|
||||
conn: sqlite3.Connection, image_path: str, aliases=()
|
||||
) -> str:
|
||||
"""SHA-256 of what Apply's compatibility depends on for ``image_path``.
|
||||
|
||||
The exact legacy rows of the file under ``image_path`` and every alias
|
||||
spelling of it, with their positive hashes, and each row's pair links
|
||||
with the negative hash, read as the ingester reads them. Unrelated prompt
|
||||
changes (tags, titles) do not move it.
|
||||
"""
|
||||
rows = []
|
||||
for image_id, _spelled, positive_hash in dependent_images(
|
||||
conn, (image_path, *aliases)
|
||||
):
|
||||
links = conn.execute(_DEPENDENT_LINKS, (image_id,)).fetchall()
|
||||
rows.append([image_id, positive_hash, [list(link) for link in links]])
|
||||
text = canonical_evidence({"v": 1, "images": rows})
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class ReviewOperations:
|
||||
"""PromptDatabase mixin: insert and read Needs review items."""
|
||||
|
||||
def record_review_item(
|
||||
self,
|
||||
*,
|
||||
locator: str,
|
||||
spelled_path: str,
|
||||
sha256: str,
|
||||
size: int,
|
||||
mtime_ns: int,
|
||||
epoch: str,
|
||||
resolver_version: int,
|
||||
evidence: dict,
|
||||
candidate_count: int,
|
||||
limited: bool,
|
||||
) -> bool:
|
||||
"""Store one ambiguous result for these exact bytes; True when new.
|
||||
|
||||
The byte record is created (or its verified fingerprint refreshed)
|
||||
like an observe. ``spelled_path`` is the file's own spelling (it must
|
||||
fold to ``locator``); targets are formatted from it. Raises
|
||||
``LibraryCommandError("storage-changed")`` when the storage epoch is
|
||||
no longer ``epoch`` (a restore since the capture) and ``ValueError``
|
||||
for invalid facts or evidence.
|
||||
|
||||
One file never holds two pending items for the same bytes: the new
|
||||
item supersedes the byte record's other items (an older resolver or
|
||||
semantics version) when every one is pending and undecided (the safe
|
||||
delete of the self-heal); when one carries a decision or a later
|
||||
revision, it is kept and nothing new is stored (False).
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_spelling(spelled_path, locator)
|
||||
if type(resolver_version) is not int or resolver_version < 1:
|
||||
raise ValueError("resolver_version must be a positive integer")
|
||||
if (
|
||||
type(candidate_count) is not int
|
||||
or not 0 <= candidate_count <= MAX_CANDIDATES
|
||||
):
|
||||
raise ValueError(f"candidate_count must be between 0 and {MAX_CANDIDATES}")
|
||||
if not isinstance(limited, bool) or (limited and candidate_count):
|
||||
raise ValueError("A limited item carries no candidates")
|
||||
text = canonical_evidence(evidence)
|
||||
evidence_hash = evidence_digest(text)
|
||||
with self._pair_transaction() as conn:
|
||||
self._library_epoch(conn, epoch)
|
||||
now = utc_now_iso()
|
||||
file_id = self._annotation_owner(conn, locator, now)
|
||||
byte_id, _revision, created, moved = self._annotation_byte(
|
||||
conn, file_id, sha256, size, mtime_ns, now
|
||||
)
|
||||
if created or moved:
|
||||
conn.execute(_BUMP_IMAGE_INDEX)
|
||||
# The byte record is created or refreshed even when the item is
|
||||
# refused below (an observe would do the same); that commits
|
||||
# with the image index bump, and the review queue is unchanged.
|
||||
others = conn.execute(
|
||||
_OTHER_ITEMS_OF_BYTES, (byte_id, resolver_version, evidence_hash)
|
||||
).fetchall()
|
||||
if any(not removable for _id, removable in others):
|
||||
return False
|
||||
superseded = sum(
|
||||
conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
||||
for item_id, _removable in others
|
||||
)
|
||||
cursor = conn.execute(
|
||||
_INSERT_ITEM,
|
||||
(
|
||||
_new_id(),
|
||||
byte_id,
|
||||
spelled_path,
|
||||
resolver_version,
|
||||
evidence_hash,
|
||||
text,
|
||||
candidate_count,
|
||||
int(limited),
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
inserted = cursor.rowcount > 0
|
||||
refreshed = _refresh_spelling(conn, locator, spelled_path)
|
||||
if inserted or refreshed or superseded:
|
||||
conn.execute(_BUMP_REVIEW_INDEX)
|
||||
if inserted or refreshed or superseded:
|
||||
notify_review_change(self.model.db_path)
|
||||
return inserted
|
||||
|
||||
def refresh_review_spelling(self, locator: str, spelled_path: str) -> bool:
|
||||
"""Record ``spelled_path`` as the spelling of every item at ``locator``.
|
||||
|
||||
The rescan's skip calls it for a file it does not copy again: a
|
||||
case-only rename keeps size and ``mtime_ns``, and a target formatted
|
||||
from the old spelling would be refused. True when a spelling changed
|
||||
(the review index revision moves; item revisions do not, so an open
|
||||
decision stays valid).
|
||||
"""
|
||||
_validate_locator(locator)
|
||||
_validate_spelling(spelled_path, locator)
|
||||
with self._pair_transaction() as conn:
|
||||
refreshed = _refresh_spelling(conn, locator, spelled_path)
|
||||
if refreshed:
|
||||
conn.execute(_BUMP_REVIEW_INDEX)
|
||||
if refreshed:
|
||||
notify_review_change(self.model.db_path)
|
||||
return refreshed
|
||||
|
||||
def review_index_revision(self) -> int:
|
||||
"""The queue's revision (moved by every queue change)."""
|
||||
row = self.model.get_connection().execute(_REVIEW_INDEX).fetchone()
|
||||
if row is None:
|
||||
raise sqlite3.DatabaseError("Missing review queue metadata")
|
||||
return row[0]
|
||||
|
||||
def review_queue_version(self) -> dict:
|
||||
"""``{library_epoch, review_revision}`` of the queue, from one snapshot.
|
||||
|
||||
The pair a list read answers: equal pairs mean the same queue (a
|
||||
restore moves the epoch, every queue change moves the revision).
|
||||
"""
|
||||
with self._annotation_read() as conn:
|
||||
epoch = conn.execute(_EPOCH).fetchone()
|
||||
revision = conn.execute(_REVIEW_INDEX).fetchone()
|
||||
if epoch is None or revision is None:
|
||||
raise sqlite3.DatabaseError("Missing review queue metadata")
|
||||
return {"library_epoch": epoch[0], "review_revision": revision[0]}
|
||||
|
||||
def review_items_for(self, locator: str) -> list:
|
||||
"""Every item of the file at ``locator``, oldest first, evidence parsed."""
|
||||
_validate_locator(locator)
|
||||
return list(self._iter_review_items(locator))
|
||||
|
||||
def _iter_review_items(self, locator: str) -> Iterator[dict]:
|
||||
conn = self.model.get_connection()
|
||||
for row in conn.execute(_ITEMS_OF_LOCATOR, (locator,)):
|
||||
item = dict(zip(_ITEM_COLUMNS, tuple(row)))
|
||||
item["evidence"] = json.loads(item["evidence"])
|
||||
item["limited"] = bool(item["limited"])
|
||||
yield item
|
||||
|
||||
# -- queue reads (#249 E3) -------------------------------------------------
|
||||
|
||||
def review_fingerprint_queued(
|
||||
self, locator: str, size: int, mtime_ns: int, resolver_version: int
|
||||
) -> bool:
|
||||
"""True when an item cites a byte record verified at this exact stat.
|
||||
|
||||
A rescan uses it to skip the copy and hash of a file it already
|
||||
queued: the same ``(size, mtime_ns)`` the bytes were verified with.
|
||||
"""
|
||||
row = (
|
||||
self.model.get_connection()
|
||||
.execute(_QUEUED, (locator, size, mtime_ns, resolver_version))
|
||||
.fetchone()
|
||||
)
|
||||
return row is not None
|
||||
|
||||
def review_decided_at(self, locator: str, size: int, mtime_ns: int) -> bool:
|
||||
"""True when bytes verified at this stat carry a decided item.
|
||||
|
||||
Any version: a newer resolver or table never adds an item beside a
|
||||
decision, so the rescan need not copy those bytes again.
|
||||
"""
|
||||
row = (
|
||||
self.model.get_connection()
|
||||
.execute(_DECIDED_AT, (locator, size, mtime_ns))
|
||||
.fetchone()
|
||||
)
|
||||
return row is not None
|
||||
|
||||
def review_pending_at(
|
||||
self, locator: str, size: int, mtime_ns: int, resolver_version: int
|
||||
) -> bool:
|
||||
"""True when a *pending* item cites bytes verified at this exact stat.
|
||||
|
||||
``review_fingerprint_queued`` also matches decided items; a file whose
|
||||
only item is decided is not in the queue.
|
||||
"""
|
||||
row = (
|
||||
self.model.get_connection()
|
||||
.execute(_PENDING_AT, (locator, size, mtime_ns, resolver_version))
|
||||
.fetchone()
|
||||
)
|
||||
return row is not None
|
||||
|
||||
def review_file_linked(self, spellings) -> bool:
|
||||
"""True when a saved row links the file under any of ``spellings``."""
|
||||
with self._annotation_read() as conn:
|
||||
return bool(dependent_images(conn, spellings))
|
||||
|
||||
def settle_linked_review_items(self, files) -> int:
|
||||
"""Remove the pending, undecided items of files that are now linked.
|
||||
|
||||
``files`` holds ``(locator, spellings)`` per file. An item leaves the
|
||||
queue only when it is still ``pending`` at revision 0 with no decision
|
||||
row and a saved row links its file under one of ``spellings``; a
|
||||
decided item is never touched. Files without a pending item cost one
|
||||
indexed read and open no write transaction. Returns how many items
|
||||
were removed (the review index revision moves when any was).
|
||||
"""
|
||||
files = [(locator, tuple(spellings)) for locator, spellings in files]
|
||||
for locator, _spellings in files:
|
||||
_validate_locator(locator)
|
||||
conn = self.model.get_connection()
|
||||
candidates = [
|
||||
(locator, spellings)
|
||||
for locator, spellings in files
|
||||
if conn.execute(_PENDING_OF_LOCATOR, (locator,)).fetchone() is not None
|
||||
]
|
||||
if not candidates:
|
||||
return 0
|
||||
removed = 0
|
||||
with self._pair_transaction() as conn:
|
||||
for locator, spellings in candidates:
|
||||
if not dependent_images(conn, spellings):
|
||||
continue
|
||||
for (item_id,) in conn.execute(
|
||||
_PENDING_OF_LOCATOR, (locator,)
|
||||
).fetchall():
|
||||
removed += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
||||
if removed:
|
||||
conn.execute(_BUMP_REVIEW_INDEX)
|
||||
if removed:
|
||||
notify_review_change(self.model.db_path)
|
||||
return removed
|
||||
|
||||
def settle_stale_review_items(
|
||||
self, locator: str, size: int, mtime_ns: int, versions
|
||||
) -> int:
|
||||
"""Remove pending, undecided items of the bytes verified at this stat
|
||||
whose ``resolver_version`` is not one of ``versions``.
|
||||
|
||||
The caller found these bytes untraced under the current versions, so
|
||||
an older version's item could only refuse Apply. The same safe delete
|
||||
as the other settles; one indexed read when nothing is pending.
|
||||
"""
|
||||
_validate_locator(locator)
|
||||
current = set(versions)
|
||||
conn = self.model.get_connection()
|
||||
stale = [
|
||||
item_id
|
||||
for item_id, version in conn.execute(
|
||||
_PENDING_AT_STAT, (locator, size, mtime_ns)
|
||||
)
|
||||
if version not in current
|
||||
]
|
||||
if not stale:
|
||||
return 0
|
||||
removed = 0
|
||||
with self._pair_transaction() as conn:
|
||||
for item_id in stale:
|
||||
removed += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
||||
if removed:
|
||||
conn.execute(_BUMP_REVIEW_INDEX)
|
||||
if removed:
|
||||
notify_review_change(self.model.db_path)
|
||||
return removed
|
||||
|
||||
def settle_untraced_review_items(self, locators) -> int:
|
||||
"""Remove the pending, undecided items that offer nothing to choose.
|
||||
|
||||
``locators`` are files the caller found ambiguous without a choice
|
||||
(fewer than two candidates carry a prompt). Of their items, one leaves
|
||||
the queue only when its own stored evidence offers no choice either
|
||||
(``evidence_offers_choice``) and it is still ``pending`` at revision 0
|
||||
with no decision row, the same safe delete as
|
||||
``settle_linked_review_items``. Files without a pending item cost one
|
||||
indexed read and open no write transaction. Returns how many items
|
||||
were removed (the review index revision moves when any was).
|
||||
"""
|
||||
locators = list(locators)
|
||||
for locator in locators:
|
||||
_validate_locator(locator)
|
||||
conn = self.model.get_connection()
|
||||
pending = [
|
||||
locator
|
||||
for locator in locators
|
||||
if conn.execute(_PENDING_OF_LOCATOR, (locator,)).fetchone() is not None
|
||||
]
|
||||
if not pending:
|
||||
return 0
|
||||
removed = 0
|
||||
with self._pair_transaction() as conn:
|
||||
for locator in pending:
|
||||
rows = conn.execute(_PENDING_EVIDENCE_OF_LOCATOR, (locator,)).fetchall()
|
||||
for item_id, evidence in rows:
|
||||
if evidence_offers_choice(json.loads(evidence)):
|
||||
continue
|
||||
removed += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
||||
if removed:
|
||||
conn.execute(_BUMP_REVIEW_INDEX)
|
||||
return removed
|
||||
|
||||
def review_queue_page(self, *, limit: int, after: tuple | None = None) -> dict:
|
||||
"""One page of pending items, oldest first, from one snapshot.
|
||||
|
||||
``after`` is the ``(created_at, id)`` keyset of the last item already
|
||||
shown. Returns ``{epoch, revision, total, items, more}``; ``total`` is
|
||||
the exact pending count and ``items`` carry their ``spelled_path`` and
|
||||
``locator`` (server-side only).
|
||||
"""
|
||||
if type(limit) is not int or limit < 1:
|
||||
raise ValueError("limit must be a positive integer")
|
||||
with self._annotation_read() as conn:
|
||||
epoch = conn.execute(_EPOCH).fetchone()
|
||||
revision = conn.execute(_REVIEW_INDEX).fetchone()
|
||||
if epoch is None or revision is None:
|
||||
raise sqlite3.DatabaseError("Missing review queue metadata")
|
||||
total = conn.execute(_PENDING_COUNT).fetchone()[0]
|
||||
if after is None:
|
||||
rows = conn.execute(_QUEUE_FIRST, (limit + 1,)).fetchall()
|
||||
else:
|
||||
rows = conn.execute(_QUEUE_AFTER, (*after, limit + 1)).fetchall()
|
||||
items = [_queue_row(row, _QUEUE_COLUMNS) for row in rows[:limit]]
|
||||
return {
|
||||
"epoch": epoch[0],
|
||||
"revision": revision[0],
|
||||
"total": total,
|
||||
"items": items,
|
||||
"more": len(rows) > limit,
|
||||
}
|
||||
|
||||
def review_item(self, item_id: str) -> dict | None:
|
||||
"""One item with its parsed evidence and the pending total, or None."""
|
||||
if not isinstance(item_id, str) or _ITEM_ID.fullmatch(item_id) is None:
|
||||
return None
|
||||
with self._annotation_read() as conn:
|
||||
epoch = conn.execute(_EPOCH).fetchone()
|
||||
if epoch is None:
|
||||
raise sqlite3.DatabaseError("Missing execution history metadata")
|
||||
row = conn.execute(_DETAIL, (item_id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
item = _queue_row(row, _DETAIL_COLUMNS)
|
||||
item["evidence"] = json.loads(item["evidence"])
|
||||
item["epoch"] = epoch[0]
|
||||
item["total"] = conn.execute(_PENDING_COUNT).fetchone()[0]
|
||||
return item
|
||||
|
||||
def review_dependency_fingerprint(self, image_path: str, aliases=()) -> str:
|
||||
with self._annotation_read() as conn:
|
||||
return dependency_fingerprint(conn, image_path, aliases)
|
||||
|
||||
def review_same_name_rows(self, name: str) -> list:
|
||||
"""``(id, image_path)`` of legacy rows that may spell a file ``name``."""
|
||||
with self._annotation_read() as conn:
|
||||
return same_name_rows(conn, name)
|
||||
|
||||
def stored_prompt_titles(self, texts) -> dict:
|
||||
"""Library titles of saved prompts by text (dedup hash), when set."""
|
||||
hashes = {generate_prompt_hash(text): text for text in texts if text.strip()}
|
||||
if not hashes:
|
||||
return {}
|
||||
marks = ",".join("?" * len(hashes))
|
||||
found: dict[str, tuple] = {}
|
||||
conn = self.model.get_connection()
|
||||
for prompt_hash, title in conn.execute(
|
||||
_TITLES.format(marks=marks), list(hashes)
|
||||
):
|
||||
text = hashes[prompt_hash]
|
||||
found[text] = found.get(text, ()) + (title,)
|
||||
return found
|
||||
|
||||
|
||||
def _refresh_spelling(conn: sqlite3.Connection, locator: str, spelled: str) -> bool:
|
||||
return conn.execute(_REFRESH_SPELLING, (spelled, spelled, locator)).rowcount > 0
|
||||
|
||||
|
||||
def _queue_row(row, columns) -> dict:
|
||||
item = dict(zip(columns, tuple(row)))
|
||||
item["limited"] = bool(item["limited"])
|
||||
return item
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Needs review queue change notices (#249 PR F2).
|
||||
|
||||
Every committed change to the pending queue (an item recorded by the scan or
|
||||
the monitor, a spelling refresh, a decision) calls ``notify_review_change``
|
||||
after its transaction, from whichever thread wrote it. The review event
|
||||
stream (``py/api/review_events.py``) subscribes, so an open Needs review page
|
||||
re-reads its queue when it changes instead of polling. Observers are keyed by
|
||||
the database file, like ``subscribe_history_restore``; an observer that fails
|
||||
is logged and never fails the write.
|
||||
"""
|
||||
|
||||
import os
|
||||
import threading
|
||||
from typing import Callable
|
||||
|
||||
try:
|
||||
from ..utils.logging_config import get_logger
|
||||
except ImportError:
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
_logger = get_logger("prompt_manager.database.review_changes")
|
||||
_lock = threading.Lock()
|
||||
_observers: dict[str, dict[object, Callable[[], None]]] = {}
|
||||
|
||||
|
||||
def _identity(db_path: str) -> str:
|
||||
return os.path.normcase(os.path.realpath(db_path))
|
||||
|
||||
|
||||
def subscribe_review_changes(
|
||||
db_path: str, callback: Callable[[], None]
|
||||
) -> Callable[[], None]:
|
||||
"""Call ``callback()`` after each committed queue change; return the unsubscribe."""
|
||||
identity = _identity(db_path)
|
||||
token = object()
|
||||
with _lock:
|
||||
_observers.setdefault(identity, {})[token] = callback
|
||||
|
||||
def unsubscribe() -> None:
|
||||
with _lock:
|
||||
observers = _observers.get(identity)
|
||||
if observers is not None:
|
||||
observers.pop(token, None)
|
||||
if not observers:
|
||||
_observers.pop(identity, None)
|
||||
|
||||
return unsubscribe
|
||||
|
||||
|
||||
def observer_count(db_path: str) -> int:
|
||||
"""How many observers watch ``db_path`` (tests and diagnostics)."""
|
||||
with _lock:
|
||||
return len(_observers.get(_identity(db_path), {}))
|
||||
|
||||
|
||||
def notify_review_change(db_path: str) -> None:
|
||||
"""Tell every observer of ``db_path``; call it after the commit."""
|
||||
with _lock:
|
||||
callbacks = tuple(_observers.get(_identity(db_path), {}).values())
|
||||
for callback in callbacks:
|
||||
try:
|
||||
callback()
|
||||
except Exception:
|
||||
_logger.warning("Needs review change observer failed", exc_info=True)
|
||||
|
||||
|
||||
def rekey_review_changes(old_path: str, new_path: str) -> None:
|
||||
"""Move the observers of ``old_path`` to ``new_path`` (Move library)."""
|
||||
old, new = _identity(old_path), _identity(new_path)
|
||||
if old == new:
|
||||
return
|
||||
with _lock:
|
||||
observers = _observers.pop(old, None)
|
||||
if observers:
|
||||
_observers.setdefault(new, {}).update(observers)
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Needs review decisions (#249 E3): explicit, auditable and compatible-only.
|
||||
|
||||
A decision is one row of ``media_review_decisions`` for one item revision.
|
||||
``selected`` links the chosen candidate's pair to the file through the public
|
||||
ingester's compatible-only rules (``_ingest_resolved_pair_in``) in the same
|
||||
``_pair_transaction`` as the decision row, the item's new state and revision
|
||||
and the review index revision; any conflict or later failure rolls all of it
|
||||
back. ``unresolved`` records the decision and links nothing. No decision
|
||||
counts a run, touches usage or calls an execution writer.
|
||||
|
||||
Callers verify the file first (the capture must be the item's bytes and the
|
||||
current file must still hash to them); nothing here reads the filesystem.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
from .annotation_schema import BYTES_TABLE
|
||||
from .annotations import _new_id
|
||||
from .library_commands import LibraryCommandError
|
||||
from .models import utc_now_iso
|
||||
from .review import (
|
||||
_BUMP_REVIEW_INDEX,
|
||||
_DETAIL,
|
||||
_DETAIL_COLUMNS,
|
||||
_PENDING_COUNT,
|
||||
_queue_row,
|
||||
dependency_fingerprint,
|
||||
dependent_images,
|
||||
same_name_rows,
|
||||
)
|
||||
from .review_changes import notify_review_change
|
||||
from .review_schema import DECISIONS_TABLE, ITEMS_TABLE, canonical_evidence
|
||||
|
||||
DECISION_KIND = "user-selection"
|
||||
_INSERT_DECISION = (
|
||||
f"INSERT INTO {DECISIONS_TABLE}(id,item_id,decision_revision," # nosec B608
|
||||
"created_at,outcome,candidate_hash,evidence,pair_id,image_id) "
|
||||
"VALUES(?,?,?,?,?,?,?,?,?)"
|
||||
)
|
||||
_ADVANCE_ITEM = (
|
||||
f"UPDATE {ITEMS_TABLE} SET state=?,revision=revision+1," # nosec B608
|
||||
"updated_at=? WHERE id=? AND revision=?"
|
||||
)
|
||||
_ITEM_BYTES = (
|
||||
f"SELECT b.sha256 FROM {ITEMS_TABLE} i " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.id=i.byte_id WHERE i.id=?"
|
||||
)
|
||||
|
||||
|
||||
class ReviewCommandError(LibraryCommandError):
|
||||
"""A decision lost a race or would rewrite an association.
|
||||
|
||||
``code`` is ``review-conflict`` (the item was decided or changed since it
|
||||
was read), ``association-conflict`` (the choice is not compatible with
|
||||
the file's saved links, or those links changed), ``file-changed`` (the
|
||||
item is not about these bytes) or ``review-missing``. ``item`` is the
|
||||
current item read inside the transaction, ``total`` the pending count and
|
||||
``fingerprint`` the current dependency fingerprint, when known;
|
||||
``changed`` marks an association conflict caused by links that changed
|
||||
since the reviewer read them (rather than an incompatible choice).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, code, epoch, *, item=None, total=None, fingerprint=None, changed=False
|
||||
):
|
||||
super().__init__(code, epoch)
|
||||
self.item = item
|
||||
self.total = total
|
||||
self.fingerprint = fingerprint
|
||||
self.changed = changed
|
||||
# The public view of ``item``, set by the API when it formats one.
|
||||
self.view = None
|
||||
|
||||
|
||||
class _Rollback(Exception):
|
||||
"""Raised inside the transaction so the context manager rolls back."""
|
||||
|
||||
def __init__(self, error: ReviewCommandError):
|
||||
super().__init__(error.code)
|
||||
self.error = error
|
||||
|
||||
|
||||
class ReviewDecisionOperations:
|
||||
"""PromptDatabase mixin: record one decision on one Needs review item."""
|
||||
|
||||
def decide_review_item(
|
||||
self,
|
||||
*,
|
||||
item_id: str,
|
||||
expected_epoch: str,
|
||||
expected_revision: int,
|
||||
sha256: str,
|
||||
outcome: str,
|
||||
candidate: dict | None = None,
|
||||
image_path: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
expected_fingerprint: str | None = None,
|
||||
aliases: tuple = (),
|
||||
known_rows: frozenset | None = None,
|
||||
) -> dict:
|
||||
"""Record the decision; return ``{item, total, pair}``.
|
||||
|
||||
``selected`` needs the evidence ``candidate`` (with its ``hash``), the
|
||||
``image_path`` the scan and the monitor link under, the captured
|
||||
``metadata`` and the ``expected_fingerprint`` the reviewer saw.
|
||||
``aliases`` are the other spellings legacy rows of the same physical
|
||||
file use (a symlinked, junctioned or mapped root) and ``known_rows``
|
||||
the ids of every same-name row the caller classified; a same-name row
|
||||
it did not see is a changed association.
|
||||
Raises ``LibraryCommandError("storage-changed")`` after a restore and
|
||||
:class:`ReviewCommandError` for every refused decision, with zero
|
||||
writes either way.
|
||||
"""
|
||||
if outcome not in ("selected", "unresolved"):
|
||||
raise ValueError("outcome must be selected or unresolved")
|
||||
if outcome == "selected" and (
|
||||
not isinstance(candidate, dict) or not image_path
|
||||
):
|
||||
raise ValueError("A selected decision needs a candidate and a file")
|
||||
try:
|
||||
with self._pair_transaction() as conn:
|
||||
decided = self._decide_in(
|
||||
conn,
|
||||
item_id=item_id,
|
||||
expected_epoch=expected_epoch,
|
||||
expected_revision=expected_revision,
|
||||
sha256=sha256,
|
||||
outcome=outcome,
|
||||
candidate=candidate,
|
||||
image_path=image_path,
|
||||
metadata=metadata or {},
|
||||
expected_fingerprint=expected_fingerprint,
|
||||
aliases=tuple(aliases),
|
||||
known_rows=known_rows,
|
||||
)
|
||||
except _Rollback as rollback:
|
||||
raise rollback.error from None
|
||||
# After the commit: open Needs review pages re-read their queue.
|
||||
notify_review_change(self.model.db_path)
|
||||
return decided
|
||||
|
||||
def _decide_in(self, conn: sqlite3.Connection, **command) -> dict:
|
||||
epoch = self._library_epoch(conn, command["expected_epoch"])
|
||||
item_id = command["item_id"]
|
||||
current = _current_item(conn, item_id)
|
||||
if current is None:
|
||||
raise _Rollback(ReviewCommandError("review-missing", epoch))
|
||||
total = conn.execute(_PENDING_COUNT).fetchone()[0]
|
||||
if (
|
||||
current["state"] != "pending"
|
||||
or current["revision"] != command["expected_revision"]
|
||||
):
|
||||
raise _Rollback(
|
||||
ReviewCommandError("review-conflict", epoch, item=current, total=total)
|
||||
)
|
||||
if conn.execute(_ITEM_BYTES, (item_id,)).fetchone()[0] != command["sha256"]:
|
||||
raise _Rollback(ReviewCommandError("file-changed", epoch))
|
||||
decision_id = _new_id()
|
||||
pair = None
|
||||
candidate = command["candidate"] if command["outcome"] == "selected" else None
|
||||
if candidate is not None:
|
||||
pair = self._apply_candidate(
|
||||
conn, epoch, current, decision_id, total, **command
|
||||
)
|
||||
evidence = {
|
||||
"kind": DECISION_KIND,
|
||||
"decision_id": decision_id,
|
||||
"outcome": command["outcome"],
|
||||
"reviewed_digest": command["sha256"],
|
||||
"item_evidence_hash": current["evidence_hash"],
|
||||
"resolver_version": current["resolver_version"],
|
||||
"candidate": candidate,
|
||||
"dependency_fingerprint": command["expected_fingerprint"],
|
||||
"pair": pair,
|
||||
}
|
||||
now = utc_now_iso()
|
||||
self._insert_review_decision(
|
||||
conn,
|
||||
(
|
||||
decision_id,
|
||||
item_id,
|
||||
current["revision"] + 1,
|
||||
now,
|
||||
command["outcome"],
|
||||
candidate["hash"] if candidate is not None else None,
|
||||
canonical_evidence(evidence),
|
||||
pair["id"] if pair else None,
|
||||
pair["image_id"] if pair else None,
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
_ADVANCE_ITEM, (command["outcome"], now, item_id, current["revision"])
|
||||
)
|
||||
conn.execute(_BUMP_REVIEW_INDEX)
|
||||
return {
|
||||
"item": _current_item(conn, item_id),
|
||||
"total": conn.execute(_PENDING_COUNT).fetchone()[0],
|
||||
"pair": pair,
|
||||
"decision_id": decision_id,
|
||||
}
|
||||
|
||||
def _apply_candidate(
|
||||
self, conn, epoch, current, decision_id, total, **command
|
||||
) -> dict:
|
||||
"""Link the candidate's pair, compatible-only; raise on any conflict."""
|
||||
# Imported here: pair_ingestion imports operations, which composes this.
|
||||
from .pair_ingestion import _ingest_resolved_pair_in
|
||||
|
||||
image_path = command["image_path"]
|
||||
aliases = command["aliases"]
|
||||
fingerprint = dependency_fingerprint(conn, image_path, aliases)
|
||||
known = command["known_rows"]
|
||||
unseen = known is not None and any(
|
||||
row_id not in known
|
||||
for row_id, _path in same_name_rows(conn, os.path.basename(image_path))
|
||||
)
|
||||
if unseen or fingerprint != command["expected_fingerprint"]:
|
||||
raise _Rollback(
|
||||
ReviewCommandError(
|
||||
"association-conflict",
|
||||
epoch,
|
||||
item=current,
|
||||
total=total,
|
||||
fingerprint=fingerprint,
|
||||
changed=True,
|
||||
)
|
||||
)
|
||||
rows = dependent_images(conn, (image_path, *aliases))
|
||||
if len(rows) > 1:
|
||||
# One file linked under several spellings: no single row to enrich.
|
||||
raise _Rollback(
|
||||
ReviewCommandError(
|
||||
"association-conflict",
|
||||
epoch,
|
||||
item=current,
|
||||
total=total,
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
)
|
||||
if rows:
|
||||
# Enrich the one row of this file under the spelling it was
|
||||
# linked with, so the ingester's compatibility rules see it.
|
||||
image_path = rows[0][1]
|
||||
candidate = command["candidate"]
|
||||
positive, negative = candidate["positive"], candidate["negative"]
|
||||
provenance = {
|
||||
"kind": DECISION_KIND,
|
||||
"decision_id": decision_id,
|
||||
"reviewed_digest": command["sha256"],
|
||||
"candidate_hash": candidate["hash"],
|
||||
"resolver_version": current["resolver_version"],
|
||||
"positive": positive,
|
||||
"negative": negative,
|
||||
}
|
||||
result = _ingest_resolved_pair_in(
|
||||
conn,
|
||||
self,
|
||||
image_path,
|
||||
command["metadata"],
|
||||
{"state": "resolved", "positive": positive, "negative": negative},
|
||||
provenance=provenance,
|
||||
)
|
||||
if result["status"] == "conflict":
|
||||
raise _Rollback(
|
||||
ReviewCommandError(
|
||||
"association-conflict",
|
||||
epoch,
|
||||
item=current,
|
||||
total=total,
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
)
|
||||
return {
|
||||
"id": result["pair_id"],
|
||||
"image_id": result["image_id"],
|
||||
"status": result["status"],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _insert_review_decision(conn: sqlite3.Connection, values: tuple) -> None:
|
||||
conn.execute(_INSERT_DECISION, values)
|
||||
|
||||
|
||||
def _current_item(conn: sqlite3.Connection, item_id: str) -> dict | None:
|
||||
row = conn.execute(_DETAIL, (item_id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
item = _queue_row(row, _DETAIL_COLUMNS)
|
||||
item.pop("evidence")
|
||||
return item
|
||||
@@ -0,0 +1,507 @@
|
||||
"""Needs review schema group (#249): capture-bound ambiguous items and decisions.
|
||||
|
||||
The group installs like ``image_annotations``: optional in old backups,
|
||||
required once a database is current, and valid only beside a valid image
|
||||
annotation group, because every item cites one annotation byte record. It owns
|
||||
|
||||
- ``media_review_metadata``: one row, the group's schema version and the
|
||||
review index revision (moved by every queue change);
|
||||
- ``media_review_items``: one ambiguous resolver result for one byte record,
|
||||
keyed by ``(byte_id, resolver_version, evidence_hash)``, with the file's
|
||||
spelling when it was recorded (the locator is case-folded on Windows, a
|
||||
target is not), its canonical evidence, candidate count, ``limited`` flag,
|
||||
state and revision;
|
||||
- ``media_review_decisions``: the auditable decision history of an item, one
|
||||
row per decision revision, with optional navigation to the pair and image
|
||||
the decision linked (nulled, never cascaded, when those are deleted).
|
||||
|
||||
Evidence is canonical JSON (:func:`canonical_evidence`): one object, sorted
|
||||
keys, no whitespace, ASCII only (so characters, bytes and the 8 MiB bound
|
||||
agree and lone surrogates survive), no NaN or infinity. ``evidence_hash`` is
|
||||
the SHA-256 of the item's evidence text.
|
||||
|
||||
Installation never backfills anything and never touches the filesystem.
|
||||
Validation reads only: catalog objects compared after whitespace compaction,
|
||||
columns and foreign keys through the PRAGMAs, persisted values in single
|
||||
bounded SQL passes, then one streamed pass that re-canonicalizes each evidence
|
||||
document (one row in memory at a time).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Any, Optional
|
||||
|
||||
from .annotation_schema import BYTES_TABLE
|
||||
from .annotation_schema import TABLES as ANNOTATION_TABLES
|
||||
from .annotation_schema import _catalog as _annotation_catalog
|
||||
from .annotation_schema import annotation_schema_error
|
||||
|
||||
try:
|
||||
from ..utils.json_strict import dumps_strict
|
||||
except ImportError:
|
||||
from utils.json_strict import dumps_strict
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
METADATA_TABLE = "media_review_metadata"
|
||||
ITEMS_TABLE = "media_review_items"
|
||||
DECISIONS_TABLE = "media_review_decisions"
|
||||
TABLES = (METADATA_TABLE, ITEMS_TABLE, DECISIONS_TABLE)
|
||||
MAX_EVIDENCE_BYTES = 8 * 1024 * 1024
|
||||
MAX_CANDIDATES = 16
|
||||
# The file's own spelling when it was recorded (bytes of UTF-8 text). Locators
|
||||
# are case-folded on Windows, so a target is formatted from this instead.
|
||||
MAX_SPELLED_PATH_BYTES = 16384
|
||||
ITEM_STATES = ("pending", "selected", "unresolved")
|
||||
DECISION_OUTCOMES = ("selected", "unresolved")
|
||||
_TIMESTAMP_BYTES = 64
|
||||
_HEX64 = "length({c})=64 AND {c} NOT GLOB '*[^0-9a-f]*'"
|
||||
_EVIDENCE_CHECK = f"CHECK(length(CAST(evidence AS BLOB)) <= {MAX_EVIDENCE_BYTES})"
|
||||
|
||||
_TABLE_DDL = {
|
||||
METADATA_TABLE: (
|
||||
f"CREATE TABLE {METADATA_TABLE} (singleton_id INTEGER PRIMARY KEY "
|
||||
"CHECK(singleton_id=1), schema_version INTEGER NOT NULL "
|
||||
f"CHECK(schema_version={SCHEMA_VERSION}), "
|
||||
"review_index_revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(review_index_revision>=0))"
|
||||
),
|
||||
ITEMS_TABLE: (
|
||||
f"CREATE TABLE {ITEMS_TABLE} (id TEXT PRIMARY KEY CHECK(length(id)=36), "
|
||||
f"byte_id TEXT NOT NULL REFERENCES {BYTES_TABLE}(id) ON DELETE RESTRICT, "
|
||||
"spelled_path TEXT NOT NULL CHECK(length(CAST(spelled_path AS BLOB)) "
|
||||
f"BETWEEN 1 AND {MAX_SPELLED_PATH_BYTES}), "
|
||||
"resolver_version INTEGER NOT NULL, evidence_hash TEXT NOT NULL "
|
||||
f"CHECK({_HEX64.format(c='evidence_hash')}), "
|
||||
f"evidence TEXT NOT NULL {_EVIDENCE_CHECK}, "
|
||||
"candidate_count INTEGER NOT NULL "
|
||||
f"CHECK(candidate_count BETWEEN 0 AND {MAX_CANDIDATES}), "
|
||||
"limited INTEGER NOT NULL CHECK(limited IN (0,1)), "
|
||||
"state TEXT NOT NULL CHECK(state IN ('pending','selected','unresolved')), "
|
||||
"revision INTEGER NOT NULL CHECK(revision>=0), created_at TEXT NOT NULL, "
|
||||
"updated_at TEXT NOT NULL, UNIQUE(byte_id, resolver_version, evidence_hash))"
|
||||
),
|
||||
DECISIONS_TABLE: (
|
||||
f"CREATE TABLE {DECISIONS_TABLE} (id TEXT PRIMARY KEY "
|
||||
"CHECK(length(id)=36), item_id TEXT NOT NULL "
|
||||
f"REFERENCES {ITEMS_TABLE}(id) ON DELETE RESTRICT, "
|
||||
"decision_revision INTEGER NOT NULL, created_at TEXT NOT NULL, "
|
||||
"outcome TEXT NOT NULL CHECK(outcome IN ('selected','unresolved')), "
|
||||
"candidate_hash TEXT CHECK(candidate_hash IS NULL OR "
|
||||
f"({_HEX64.format(c='candidate_hash')})), "
|
||||
f"evidence TEXT NOT NULL {_EVIDENCE_CHECK}, "
|
||||
"pair_id INTEGER REFERENCES prompt_pairs(id) ON DELETE SET NULL, "
|
||||
"image_id INTEGER REFERENCES generated_images(id) ON DELETE SET NULL, "
|
||||
"UNIQUE(item_id, decision_revision), "
|
||||
"CHECK((outcome='selected') = (candidate_hash IS NOT NULL)))"
|
||||
),
|
||||
}
|
||||
_INDEX_DDL = {
|
||||
# The queue reads pending items oldest first with a stable tiebreak.
|
||||
"idx_media_review_items_queue": (
|
||||
"CREATE INDEX idx_media_review_items_queue ON "
|
||||
f"{ITEMS_TABLE}(state, created_at, id)"
|
||||
),
|
||||
# Deleting a pair or an image nulls decision navigation by index, not scan.
|
||||
"idx_media_review_decisions_pair": (
|
||||
"CREATE INDEX idx_media_review_decisions_pair ON " f"{DECISIONS_TABLE}(pair_id)"
|
||||
),
|
||||
"idx_media_review_decisions_image": (
|
||||
"CREATE INDEX idx_media_review_decisions_image ON "
|
||||
f"{DECISIONS_TABLE}(image_id)"
|
||||
),
|
||||
}
|
||||
# ``sqlite_autoindex_<table>_<n>``: the TEXT primary key and the UNIQUE key.
|
||||
_AUTOINDEXES = {ITEMS_TABLE: 2, DECISIONS_TABLE: 2}
|
||||
# (name, declared type, not null, default, primary key position)
|
||||
_COLUMNS = {
|
||||
METADATA_TABLE: (
|
||||
("singleton_id", "INTEGER", 0, None, 1),
|
||||
("schema_version", "INTEGER", 1, None, 0),
|
||||
("review_index_revision", "INTEGER", 1, "0", 0),
|
||||
),
|
||||
ITEMS_TABLE: (
|
||||
("id", "TEXT", 0, None, 1),
|
||||
("byte_id", "TEXT", 1, None, 0),
|
||||
("spelled_path", "TEXT", 1, None, 0),
|
||||
("resolver_version", "INTEGER", 1, None, 0),
|
||||
("evidence_hash", "TEXT", 1, None, 0),
|
||||
("evidence", "TEXT", 1, None, 0),
|
||||
("candidate_count", "INTEGER", 1, None, 0),
|
||||
("limited", "INTEGER", 1, None, 0),
|
||||
("state", "TEXT", 1, None, 0),
|
||||
("revision", "INTEGER", 1, None, 0),
|
||||
("created_at", "TEXT", 1, None, 0),
|
||||
("updated_at", "TEXT", 1, None, 0),
|
||||
),
|
||||
DECISIONS_TABLE: (
|
||||
("id", "TEXT", 0, None, 1),
|
||||
("item_id", "TEXT", 1, None, 0),
|
||||
("decision_revision", "INTEGER", 1, None, 0),
|
||||
("created_at", "TEXT", 1, None, 0),
|
||||
("outcome", "TEXT", 1, None, 0),
|
||||
("candidate_hash", "TEXT", 0, None, 0),
|
||||
("evidence", "TEXT", 1, None, 0),
|
||||
("pair_id", "INTEGER", 0, None, 0),
|
||||
("image_id", "INTEGER", 0, None, 0),
|
||||
),
|
||||
}
|
||||
# (column, parent table, parent column, on delete)
|
||||
_FOREIGN_KEYS = {
|
||||
METADATA_TABLE: set(),
|
||||
ITEMS_TABLE: {("byte_id", BYTES_TABLE, "id", "RESTRICT")},
|
||||
DECISIONS_TABLE: {
|
||||
("item_id", ITEMS_TABLE, "id", "RESTRICT"),
|
||||
("pair_id", "prompt_pairs", "id", "SET NULL"),
|
||||
("image_id", "generated_images", "id", "SET NULL"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class _NonCanonical(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _reject_constant(name: str) -> Any:
|
||||
raise _NonCanonical(f"JSON constant {name} is not allowed")
|
||||
|
||||
|
||||
# Built once: validation re-encodes every stored document at startup.
|
||||
_ENCODER = json.JSONEncoder(
|
||||
sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False
|
||||
)
|
||||
_DECODER = json.JSONDecoder(parse_constant=_reject_constant)
|
||||
|
||||
|
||||
def canonical_evidence(value: Any) -> str:
|
||||
"""Serialize one evidence object canonically; ``ValueError`` when invalid.
|
||||
|
||||
The text is a JSON object with sorted keys, no whitespace, ASCII only and
|
||||
no NaN/infinity, at most ``MAX_EVIDENCE_BYTES`` bytes. A non-finite number
|
||||
(ComfyUI stamps ``NaN`` into graphs) is written as ``null``, the store's
|
||||
policy everywhere (``utils.json_strict.dumps_strict``).
|
||||
"""
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError("Review evidence must be a JSON object")
|
||||
try:
|
||||
try:
|
||||
text = _ENCODER.encode(value)
|
||||
except ValueError as error:
|
||||
if "Out of range float" not in str(error):
|
||||
raise
|
||||
text = dumps_strict(
|
||||
value, sort_keys=True, separators=(",", ":"), ensure_ascii=True
|
||||
)
|
||||
except (TypeError, RecursionError) as error:
|
||||
raise ValueError(f"Review evidence is not serializable: {error}") from None
|
||||
if len(text) > MAX_EVIDENCE_BYTES:
|
||||
raise ValueError("Review evidence is larger than 8 MiB")
|
||||
return text
|
||||
|
||||
|
||||
def choosable(candidate: Any) -> bool:
|
||||
"""True when a candidate offers a prompt to choose: it is ``resolved`` and
|
||||
its positive side carries non-blank text. The page disables any other
|
||||
candidate, so it is never something to choose.
|
||||
"""
|
||||
if not isinstance(candidate, dict) or candidate.get("state") != "resolved":
|
||||
return False
|
||||
positive = candidate.get("positive")
|
||||
if not isinstance(positive, dict):
|
||||
return False
|
||||
text = positive.get("text")
|
||||
return isinstance(text, str) and bool(text.strip())
|
||||
|
||||
|
||||
# An item is worth a decision only when at least this many candidates offer
|
||||
# a prompt; with fewer there is nothing to choose between (maintainer test of
|
||||
# #284/#288: 39 items whose candidates were all "unresolved").
|
||||
MIN_CHOOSABLE = 2
|
||||
|
||||
|
||||
def evidence_offers_choice(evidence: Any) -> bool:
|
||||
"""False when stored evidence provably offers nothing to choose.
|
||||
|
||||
Unlimited evidence must list ``MIN_CHOOSABLE`` choosable candidates. A
|
||||
limited item stores none, only ``candidate_total``; it may offer a choice
|
||||
(more than ``MAX_CANDIDATES`` prompts) unless that total is too small.
|
||||
"""
|
||||
if not isinstance(evidence, dict):
|
||||
return False
|
||||
if evidence.get("limited"):
|
||||
total = evidence.get("candidate_total")
|
||||
return type(total) is int and total >= MIN_CHOOSABLE
|
||||
candidates = evidence.get("candidates")
|
||||
if not isinstance(candidates, list):
|
||||
return False
|
||||
return sum(1 for c in candidates if choosable(c)) >= MIN_CHOOSABLE
|
||||
|
||||
|
||||
def evidence_digest(text: str) -> str:
|
||||
"""SHA-256 of canonical evidence text (ASCII, hence UTF-8 identical)."""
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _is_canonical(text: Any) -> bool:
|
||||
if not isinstance(text, str):
|
||||
return False
|
||||
try:
|
||||
value = _DECODER.decode(text)
|
||||
return canonical_evidence(value) == text
|
||||
except (ValueError, RecursionError):
|
||||
return False
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql).lower()
|
||||
|
||||
|
||||
def _uuid(column: str) -> str:
|
||||
"""SQL: *column* is a lowercase hyphenated 36-character UUID string."""
|
||||
return (
|
||||
f"typeof({column}) = 'text' AND length({column}) = 36 "
|
||||
f"AND substr({column}, 9, 1) = '-' AND substr({column}, 14, 1) = '-' "
|
||||
f"AND substr({column}, 19, 1) = '-' AND substr({column}, 24, 1) = '-' "
|
||||
f"AND length(replace({column}, '-', '')) = 32 "
|
||||
f"AND replace({column}, '-', '') NOT GLOB '*[^0-9a-f]*'"
|
||||
)
|
||||
|
||||
|
||||
def _timestamp(column: str) -> str:
|
||||
return (
|
||||
f"typeof({column}) = 'text' "
|
||||
f"AND length(CAST({column} AS BLOB)) <= {_TIMESTAMP_BYTES}"
|
||||
)
|
||||
|
||||
|
||||
def _non_negative(column: str) -> str:
|
||||
return f"typeof({column}) = 'integer' AND {column} >= 0"
|
||||
|
||||
|
||||
def _hex64(column: str) -> str:
|
||||
return f"typeof({column}) = 'text' AND " + _HEX64.format(c=column)
|
||||
|
||||
|
||||
def _evidence(column: str) -> str:
|
||||
# Type and size only: the streamed pass below proves the text canonical
|
||||
# (which implies printable ASCII), so a pattern scan here would read every
|
||||
# document a second time.
|
||||
return (
|
||||
f"typeof({column}) = 'text' "
|
||||
f"AND length(CAST({column} AS BLOB)) BETWEEN 2 AND {MAX_EVIDENCE_BYTES}"
|
||||
)
|
||||
|
||||
|
||||
def _nullable_integer(column: str) -> str:
|
||||
return f"({column} IS NULL OR typeof({column}) = 'integer')"
|
||||
|
||||
|
||||
def _in(column: str, values: tuple) -> str:
|
||||
return f"typeof({column}) = 'text' AND {column} IN ({repr(values)[1:-1]})"
|
||||
|
||||
|
||||
# Every row of a table must satisfy its predicate; one pass each, LIMIT 1.
|
||||
_ROW_PREDICATES = {
|
||||
ITEMS_TABLE: " AND ".join(
|
||||
(
|
||||
_uuid("id"),
|
||||
_uuid("byte_id"),
|
||||
"typeof(spelled_path) = 'text' AND length(CAST(spelled_path AS BLOB)) "
|
||||
f"BETWEEN 1 AND {MAX_SPELLED_PATH_BYTES}",
|
||||
"typeof(resolver_version) = 'integer'",
|
||||
_hex64("evidence_hash"),
|
||||
_evidence("evidence"),
|
||||
_non_negative("candidate_count"),
|
||||
f"candidate_count <= {MAX_CANDIDATES}",
|
||||
"typeof(limited) = 'integer' AND limited IN (0, 1)",
|
||||
_in("state", ITEM_STATES),
|
||||
_non_negative("revision"),
|
||||
_timestamp("created_at"),
|
||||
_timestamp("updated_at"),
|
||||
)
|
||||
),
|
||||
DECISIONS_TABLE: " AND ".join(
|
||||
(
|
||||
_uuid("id"),
|
||||
_uuid("item_id"),
|
||||
_non_negative("decision_revision"),
|
||||
_timestamp("created_at"),
|
||||
_in("outcome", DECISION_OUTCOMES),
|
||||
"(CASE WHEN outcome = 'selected' THEN "
|
||||
f"{_hex64('candidate_hash')} ELSE candidate_hash IS NULL END)",
|
||||
_evidence("evidence"),
|
||||
_nullable_integer("pair_id"),
|
||||
_nullable_integer("image_id"),
|
||||
)
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _expected_catalog() -> set:
|
||||
objects = {("table", name, _compact(sql)) for name, sql in _TABLE_DDL.items()}
|
||||
objects |= {("index", name, _compact(sql)) for name, sql in _INDEX_DDL.items()}
|
||||
objects |= {
|
||||
("index", f"sqlite_autoindex_{table}_{n}", "")
|
||||
for table, count in _AUTOINDEXES.items()
|
||||
for n in range(1, count + 1)
|
||||
}
|
||||
return objects
|
||||
|
||||
|
||||
def _catalog(conn: sqlite3.Connection) -> list:
|
||||
names = TABLES + tuple(_INDEX_DDL)
|
||||
marks = ",".join("?" * len(names))
|
||||
table_marks = ",".join("?" * len(TABLES))
|
||||
return conn.execute( # only "?" placeholders are interpolated
|
||||
"SELECT type,name,sql FROM sqlite_master "
|
||||
f"WHERE name IN ({marks}) OR tbl_name IN ({table_marks})", # nosec B608
|
||||
names + TABLES,
|
||||
).fetchall()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
for table, expected in _COLUMNS.items():
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[4], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({table})")
|
||||
)
|
||||
if columns != expected:
|
||||
return f"Review queue table {table} has invalid columns"
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({table})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS[table]:
|
||||
return f"Review queue table {table} has invalid foreign keys"
|
||||
if conn.execute(f"PRAGMA foreign_key_check({table})").fetchone():
|
||||
return f"Review queue table {table} contains foreign key violations"
|
||||
return None
|
||||
|
||||
|
||||
def _evidence_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
"""Stream every evidence document once; one row is held at a time."""
|
||||
for evidence_hash, text in conn.execute(
|
||||
f"SELECT evidence_hash, evidence FROM {ITEMS_TABLE}" # nosec B608
|
||||
):
|
||||
if not _is_canonical(text) or evidence_digest(text) != evidence_hash:
|
||||
return f"Review queue table {ITEMS_TABLE} contains invalid evidence"
|
||||
for (text,) in conn.execute(
|
||||
f"SELECT evidence FROM {DECISIONS_TABLE}" # nosec B608
|
||||
):
|
||||
if not _is_canonical(text):
|
||||
return f"Review queue table {DECISIONS_TABLE} contains invalid evidence"
|
||||
return None
|
||||
|
||||
|
||||
def _value_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
# Only this module's constants are interpolated into the SQL below.
|
||||
rows = conn.execute(
|
||||
"SELECT COUNT(*), MIN(typeof(singleton_id) = 'integer' " # nosec B608
|
||||
"AND singleton_id = 1 AND typeof(schema_version) = 'integer' "
|
||||
f"AND schema_version = {SCHEMA_VERSION} "
|
||||
f"AND {_non_negative('review_index_revision')}) FROM {METADATA_TABLE}"
|
||||
).fetchone()
|
||||
if rows[0] != 1 or rows[1] != 1:
|
||||
return "Review queue metadata is invalid"
|
||||
for table, predicate in _ROW_PREDICATES.items():
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {table} WHERE NOT ({predicate}) LIMIT 1" # nosec B608
|
||||
).fetchone():
|
||||
return f"Review queue table {table} contains invalid values"
|
||||
return _evidence_error(conn)
|
||||
|
||||
|
||||
def review_without_annotations_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
"""Catalog only: a review group present without its annotation tables.
|
||||
|
||||
Startup asks this before the annotation group installs, which would make
|
||||
a review group copied without it look valid. It reads no rows; the full
|
||||
validation runs once in :func:`migrate_review_state`.
|
||||
"""
|
||||
if not _catalog(conn):
|
||||
return None
|
||||
present = {
|
||||
name for kind, name, _sql in _annotation_catalog(conn) if kind == "table"
|
||||
}
|
||||
if present != set(ANNOTATION_TABLES):
|
||||
return (
|
||||
"Review queue schema requires a valid image annotation schema: "
|
||||
"Missing image annotation schema"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def review_schema_error(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
allow_missing: bool = True,
|
||||
annotations_valid: bool = False,
|
||||
) -> Optional[str]:
|
||||
"""Reject partial or unsupported review schemas and invalid rows.
|
||||
|
||||
Reads only: nothing is ever repaired. A database without any of the
|
||||
group's objects is acceptable for backups that predate it unless
|
||||
``allow_missing`` is false. A present group requires a present, valid
|
||||
image annotation group; ``annotations_valid`` says the caller has just
|
||||
validated it (startup's annotation install), so only its presence is
|
||||
checked again.
|
||||
"""
|
||||
objects = _catalog(conn)
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing review queue schema"
|
||||
if annotations_valid:
|
||||
annotation_error = review_without_annotations_error(conn)
|
||||
else:
|
||||
annotation_error = annotation_schema_error(conn, allow_missing=False)
|
||||
if annotation_error:
|
||||
annotation_error = (
|
||||
"Review queue schema requires a valid image annotation schema: "
|
||||
+ annotation_error
|
||||
)
|
||||
if annotation_error:
|
||||
return annotation_error
|
||||
present = {name for kind, name, _sql in objects if kind == "table"}
|
||||
if present != set(TABLES):
|
||||
return "Incomplete review queue schema"
|
||||
actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
|
||||
if actual != _expected_catalog():
|
||||
return "Review queue has an invalid schema definition"
|
||||
return _structure_error(conn) or _value_error(conn)
|
||||
|
||||
|
||||
def migrate_review_state(conn: sqlite3.Connection) -> None:
|
||||
"""Install the group atomically; validate before and after creating it.
|
||||
|
||||
Runs after ``migrate_annotation_state`` (which has just validated the
|
||||
annotation group): the group is only valid beside the annotation group it
|
||||
cites. A present group is validated once, evidence included.
|
||||
"""
|
||||
conn.execute("SAVEPOINT media_review_schema")
|
||||
try:
|
||||
installed = bool(_catalog(conn))
|
||||
error = review_schema_error(conn, annotations_valid=True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
if installed:
|
||||
# Already complete and valid: nothing to create, and the evidence
|
||||
# pass is the expensive part of validation, so it runs once.
|
||||
conn.execute("RELEASE media_review_schema")
|
||||
return
|
||||
for sql in _TABLE_DDL.values():
|
||||
conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
for sql in _INDEX_DDL.values():
|
||||
conn.execute(sql.replace("INDEX", "INDEX IF NOT EXISTS", 1))
|
||||
conn.execute(
|
||||
f"INSERT OR IGNORE INTO {METADATA_TABLE} "
|
||||
"(singleton_id, schema_version, review_index_revision) VALUES (1, ?, 0)",
|
||||
(SCHEMA_VERSION,),
|
||||
)
|
||||
error = review_schema_error(conn, allow_missing=False, annotations_valid=True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE media_review_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO media_review_schema")
|
||||
conn.execute("RELEASE media_review_schema")
|
||||
raise
|
||||
+30
-4
@@ -3,6 +3,9 @@
|
||||
A count is the number of eligible Library parents that carry the value
|
||||
directly or through one active saved pair, which is exactly what
|
||||
``list_library(tags=[name])`` / ``list_library(category=value)`` return.
|
||||
A tag row's ``image_count`` is the number of files (private locators) with any
|
||||
byte record carrying the tag (#249), so a file counts once however many of its
|
||||
byte versions carry it, and equal bytes at two paths count twice.
|
||||
|
||||
Tag commands run in one ``_pair_transaction``: the storage epoch is checked
|
||||
first, then the exact ``{id, name}`` identity, and the receipt is read on the
|
||||
@@ -26,6 +29,11 @@ except ImportError:
|
||||
from utils.validators import validate_category, validate_tags
|
||||
|
||||
_CATEGORY_DOMAIN = "prompt-manager-vocabulary-category:v1\n"
|
||||
# Files carrying a tag through any of their byte records (#249 image tags).
|
||||
_IMAGE_FILES = (
|
||||
"SELECT mit.tag_id,COUNT(DISTINCT b.file_id) FROM media_image_tags mit "
|
||||
"JOIN media_annotation_bytes b ON b.id=mit.byte_id"
|
||||
)
|
||||
|
||||
|
||||
def _tag_rows(page: str) -> str:
|
||||
@@ -40,9 +48,13 @@ def _tag_rows(page: str) -> str:
|
||||
"WHERE pt.tag_id IN (SELECT id FROM page) AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id "
|
||||
"WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)) "
|
||||
"SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count "
|
||||
"WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)),"
|
||||
"images(tag_id,n) AS (" + _IMAGE_FILES + " "
|
||||
"WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) "
|
||||
"SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count,"
|
||||
"COALESCE(MAX(images.n),0) AS image_count "
|
||||
"FROM page LEFT JOIN owners ON owners.tag_id=page.id "
|
||||
"LEFT JOIN images ON images.tag_id=page.id "
|
||||
"GROUP BY page.id ORDER BY page.id"
|
||||
)
|
||||
|
||||
@@ -61,7 +73,10 @@ _TAG_IMPACT = (
|
||||
"AS negative_variants,"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs pp JOIN prompt_pair_tags ppt "
|
||||
"ON ppt.pair_id=pp.id WHERE ppt.tag_id=:tag AND pp.archived!=0) "
|
||||
"AS archived_negative_variants"
|
||||
"AS archived_negative_variants,"
|
||||
"(SELECT COUNT(DISTINCT b.file_id) FROM media_image_tags mit "
|
||||
"JOIN media_annotation_bytes b ON b.id=mit.byte_id WHERE mit.tag_id=:tag) "
|
||||
"AS images"
|
||||
)
|
||||
_MAX_ID = 2**63 - 1
|
||||
|
||||
@@ -205,7 +220,16 @@ class VocabularyOperations:
|
||||
"SELECT COUNT(DISTINCT pair_id) FROM prompt_pair_tags WHERE tag_id=?",
|
||||
(tag_id,),
|
||||
).fetchone()[0]
|
||||
return {"components_changed": components, "pairs_changed": pairs}
|
||||
images = conn.execute(
|
||||
"SELECT COUNT(DISTINCT b.file_id) FROM media_image_tags mit "
|
||||
"JOIN media_annotation_bytes b ON b.id=mit.byte_id WHERE mit.tag_id=?",
|
||||
(tag_id,),
|
||||
).fetchone()[0]
|
||||
return {
|
||||
"components_changed": components,
|
||||
"pairs_changed": pairs,
|
||||
"images_changed": images,
|
||||
}
|
||||
|
||||
def create_vocabulary_tag(self, name: str, *, expected_epoch: str) -> dict:
|
||||
"""Insert an unused registry row, or reuse the exact existing name."""
|
||||
@@ -250,6 +274,7 @@ class VocabularyOperations:
|
||||
"SELECT id FROM tags WHERE name=?", (new_name,)
|
||||
).fetchone()
|
||||
if target is None:
|
||||
self._bump_image_tag_bytes(conn, tag_id)
|
||||
conn.execute("UPDATE tags SET name=? WHERE id=?", (new_name, tag_id))
|
||||
result_id = tag_id
|
||||
else:
|
||||
@@ -275,6 +300,7 @@ class VocabularyOperations:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
self._vocabulary_identity(conn, epoch, tag_id, name)
|
||||
counts = self._tag_owner_counts(conn, tag_id)
|
||||
self._bump_image_tag_bytes(conn, tag_id)
|
||||
conn.execute("DELETE FROM tags WHERE id=?", (tag_id,))
|
||||
return {"library_epoch": epoch, "deleted": True, **counts}
|
||||
|
||||
|
||||
@@ -0,0 +1,477 @@
|
||||
"""Each workflow and prompt graph stored once, compressed (#249 S6 carry-over).
|
||||
|
||||
``generated_images.workflow_data`` (the UI workflow) and ``prompt_metadata``
|
||||
(the API prompt graph) used to hold the same large JSON text again for every
|
||||
image of a run. Now each distinct text is one ``workflow_documents`` row:
|
||||
|
||||
- ``sha256``: the SHA-256 of the text's exact UTF-8 bytes (unique key);
|
||||
- ``byte_length``: the length of those bytes;
|
||||
- ``data``: ``zlib.compress`` of those bytes.
|
||||
|
||||
An image row references its texts through ``workflow_document_id`` and
|
||||
``prompt_document_id``. Two forms coexist until the migration has rewritten
|
||||
every row, and both stay readable forever:
|
||||
|
||||
- old form: the text inline in ``workflow_data`` / ``prompt_metadata``, no id;
|
||||
- new form: the inline column NULL and the document id set.
|
||||
|
||||
The text is stored and hashed exactly as written, never parsed and
|
||||
re-serialized, because a JSON round trip can change bytes (key order,
|
||||
spacing, number spelling, duplicate keys, ComfyUI's ``NaN`` tokens).
|
||||
|
||||
``read_stored_text`` is the one decoder. Inline text, when present, wins (a
|
||||
row an older app version rewrote inline stays right); otherwise the document
|
||||
is read by its rowid, decompressed within its recorded length and checked
|
||||
against its length and hash. A document that fails a check reads as
|
||||
unreadable (None, logged), never as other text. ``resolve_image_row`` puts
|
||||
the texts back into a ``generated_images`` row and drops the id keys, so
|
||||
every payload keeps its shape.
|
||||
|
||||
No trigger: every restore validator refuses triggers in a restored file. A
|
||||
document is removed in code: ``release_documents`` in the same transaction
|
||||
that drops a row's last reference, and ``sweep_documents`` (at startup, after
|
||||
the migration and before Reclaim space) for anything a path missed.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
import zlib
|
||||
from collections import OrderedDict
|
||||
from typing import Any, Dict, Iterable, Optional, Set
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.workflow_documents")
|
||||
|
||||
TABLE = "workflow_documents"
|
||||
# (inline text column, document id column) on generated_images.
|
||||
TEXT_COLUMNS = (
|
||||
("workflow_data", "workflow_document_id"),
|
||||
("prompt_metadata", "prompt_document_id"),
|
||||
)
|
||||
DOCUMENT_ID_COLUMNS = tuple(reference for _column, reference in TEXT_COLUMNS)
|
||||
INDEXES = {
|
||||
"workflow_document_id": "idx_image_workflow_document",
|
||||
"prompt_document_id": "idx_image_prompt_document",
|
||||
}
|
||||
COMPRESSION_LEVEL = 6
|
||||
# A text larger than this stays inline: a document is decompressed in one
|
||||
# piece, and its recorded length bounds that (a restored file cannot make a
|
||||
# small row expand without limit).
|
||||
MAX_DOCUMENT_BYTES = 256 * 2**20
|
||||
ID_BATCH = 400 # ids bound per statement (under SQLite's old 999 limit)
|
||||
|
||||
_DDL = (
|
||||
f"CREATE TABLE {TABLE} (id INTEGER PRIMARY KEY, sha256 BLOB NOT NULL UNIQUE, "
|
||||
"byte_length INTEGER NOT NULL, data BLOB NOT NULL)"
|
||||
)
|
||||
_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1"
|
||||
_REFERENCE_TYPE = f"INTEGER REFERENCES {TABLE}(id)"
|
||||
|
||||
_UNREFERENCED = (
|
||||
"NOT EXISTS (SELECT 1 FROM generated_images g "
|
||||
"WHERE g.workflow_document_id=workflow_documents.id) "
|
||||
"AND NOT EXISTS (SELECT 1 FROM generated_images g "
|
||||
"WHERE g.prompt_document_id=workflow_documents.id)"
|
||||
)
|
||||
LOOKUP_SQL = (
|
||||
"SELECT id, sha256, byte_length, data FROM workflow_documents WHERE sha256=?"
|
||||
)
|
||||
REPAIR_SQL = "UPDATE workflow_documents SET byte_length=?, data=? WHERE id=?"
|
||||
ALL_DOCUMENTS_SQL = "SELECT sha256, byte_length, data, id FROM workflow_documents"
|
||||
READ_SQL = "SELECT sha256, byte_length, data FROM workflow_documents WHERE id=?"
|
||||
INSERT_SQL = (
|
||||
"INSERT INTO workflow_documents (sha256, byte_length, data) VALUES (?, ?, ?)"
|
||||
)
|
||||
# Built from constants only; the ids are bound as "?" marks.
|
||||
RELEASE_SQL = "DELETE FROM workflow_documents WHERE id IN ({marks}) AND "
|
||||
RELEASE_SQL += _UNREFERENCED
|
||||
# The sweep walks the ids through the small unique index, not the blobs.
|
||||
SWEEP_SQL = (
|
||||
"DELETE FROM workflow_documents WHERE id IN (SELECT id FROM workflow_documents "
|
||||
"INDEXED BY sqlite_autoindex_workflow_documents_1 WHERE "
|
||||
)
|
||||
SWEEP_SQL += _UNREFERENCED + ")"
|
||||
IMAGE_DOCUMENTS_SQL = (
|
||||
"SELECT workflow_document_id, prompt_document_id FROM generated_images WHERE "
|
||||
)
|
||||
|
||||
# Statements whose query plans tests/test_workflow_documents.py records.
|
||||
QUERY_PLAN_STATEMENTS = {
|
||||
"lookup by hash": (LOOKUP_SQL, (b"\0" * 32,)),
|
||||
"read by id": (READ_SQL, (1,)),
|
||||
"release": (RELEASE_SQL.format(marks="?,?"), (1, 2)),
|
||||
"sweep": (SWEEP_SQL, ()),
|
||||
"documents of images": (IMAGE_DOCUMENTS_SQL + "id IN (?,?)", (1, 2)),
|
||||
"documents of prompts": (IMAGE_DOCUMENTS_SQL + "prompt_id IN (?,?)", (1, 2)),
|
||||
}
|
||||
|
||||
|
||||
class DocumentMismatch(Exception):
|
||||
"""A document would not give back exactly the text it was made from."""
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
# ── schema ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _reference_columns(conn: sqlite3.Connection) -> Dict[str, tuple]:
|
||||
return {
|
||||
row[1]: tuple(row)
|
||||
for row in conn.execute("PRAGMA table_info(generated_images)")
|
||||
if row[1] in DOCUMENT_ID_COLUMNS
|
||||
}
|
||||
|
||||
|
||||
def _reference_error(conn: sqlite3.Connection, columns: Dict[str, tuple]):
|
||||
for name, info in columns.items():
|
||||
# (cid, name, type, notnull, default, pk)
|
||||
if (str(info[2]).upper(), info[3], info[4], info[5]) != ("INTEGER", 0, None, 0):
|
||||
return "Workflow document references have an invalid column"
|
||||
keys = {
|
||||
(row[3], row[2], row[4])
|
||||
for row in conn.execute("PRAGMA foreign_key_list(generated_images)")
|
||||
if row[3] in DOCUMENT_ID_COLUMNS
|
||||
}
|
||||
if keys != {(name, TABLE, "id") for name in DOCUMENT_ID_COLUMNS}:
|
||||
return "Workflow document references have invalid foreign keys"
|
||||
for column, index in INDEXES.items():
|
||||
info = conn.execute(f'PRAGMA index_info("{index}")').fetchall()
|
||||
owner = conn.execute(
|
||||
"SELECT tbl_name FROM sqlite_master WHERE type='index' AND name=?",
|
||||
(index,),
|
||||
).fetchone()
|
||||
if [row[2] for row in info] != [column] or not owner:
|
||||
return "Workflow document references are missing their index"
|
||||
if owner[0] != "generated_images":
|
||||
return "Workflow document references are missing their index"
|
||||
return None
|
||||
|
||||
|
||||
def _value_error(conn: sqlite3.Connection):
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM workflow_documents WHERE typeof(sha256)!='blob' "
|
||||
"OR length(sha256)!=32 OR typeof(byte_length)!='integer' OR byte_length<0 "
|
||||
"OR byte_length>? OR typeof(data)!='blob' LIMIT 1",
|
||||
(MAX_DOCUMENT_BYTES,),
|
||||
).fetchone():
|
||||
return "Workflow documents contain invalid values"
|
||||
for column in DOCUMENT_ID_COLUMNS:
|
||||
# typeof() reads no overflow page; a dangling id is a rowid probe.
|
||||
# ``column`` is one of DOCUMENT_ID_COLUMNS, never caller data.
|
||||
sql = (
|
||||
f"SELECT 1 FROM generated_images g WHERE typeof(g.{column}) " # nosec B608
|
||||
f"NOT IN ('null','integer') OR (g.{column} IS NOT NULL AND NOT EXISTS "
|
||||
f"(SELECT 1 FROM workflow_documents d WHERE d.id=g.{column})) LIMIT 1"
|
||||
)
|
||||
if conn.execute(sql).fetchone():
|
||||
return "Workflow document references point at missing documents"
|
||||
return None
|
||||
|
||||
|
||||
def damaged_message(document_id: int) -> str:
|
||||
"""The verify reason for a damaged document: which one, and what to do."""
|
||||
return (
|
||||
f"Workflow document {document_id} is damaged: its text no longer matches "
|
||||
"its checksum, and the images that use it show no workflow. To fix it, "
|
||||
"restore a backup from before the damage, or store a new image with the "
|
||||
"same workflow (that repairs it)."
|
||||
)
|
||||
|
||||
|
||||
def _damaged_document(conn: sqlite3.Connection) -> Optional[int]:
|
||||
"""The first document failing its length or hash check, or None."""
|
||||
for row in conn.execute(ALL_DOCUMENTS_SQL):
|
||||
if _decode(row[:3], row[3], quiet=True) is None:
|
||||
return row[3]
|
||||
return None
|
||||
|
||||
|
||||
def workflow_documents_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True, documents: bool = False
|
||||
) -> Optional[str]:
|
||||
"""Reject an unsupported document schema or invalid stored values (reads only).
|
||||
|
||||
A database from before the dedupe has neither the table nor the id
|
||||
columns; that is acceptable unless ``allow_missing`` is false. Half of
|
||||
the schema never is.
|
||||
"""
|
||||
objects = conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name=? "
|
||||
"OR (type IN ('index','trigger') AND tbl_name=?)",
|
||||
(TABLE, TABLE),
|
||||
).fetchall()
|
||||
has_images = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='generated_images'"
|
||||
).fetchone()
|
||||
columns = _reference_columns(conn) if has_images else {}
|
||||
if not objects and not columns:
|
||||
if allow_missing or not has_images:
|
||||
return None
|
||||
return "Missing workflow document schema"
|
||||
if not objects or set(columns) != set(DOCUMENT_ID_COLUMNS):
|
||||
return "Incomplete workflow document schema"
|
||||
tables = [sql for kind, _name, sql in objects if kind == "table"]
|
||||
others = {(kind, name, sql) for kind, name, sql in objects if kind != "table"}
|
||||
if len(tables) != 1 or others != {("index", _AUTOINDEX, None)}:
|
||||
return "Workflow documents name is reserved by an unsupported schema object"
|
||||
# The exact definition fixes the columns, their types and constraints.
|
||||
if _compact(tables[0]) != _compact(_DDL):
|
||||
return "Workflow documents have an invalid table definition"
|
||||
error = _reference_error(conn, columns) or _value_error(conn)
|
||||
if error is None and documents:
|
||||
# Restore and backup verification only: every document is read.
|
||||
damaged = _damaged_document(conn)
|
||||
if damaged is not None:
|
||||
return damaged_message(damaged)
|
||||
return error
|
||||
|
||||
|
||||
def migrate_workflow_documents(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table, the id columns and their indexes atomically.
|
||||
|
||||
Runs after every older migration (the table rebuilds only run on
|
||||
databases without ``file_path``, before this). Validates before and
|
||||
after, then sweeps documents no row references.
|
||||
"""
|
||||
conn.execute("SAVEPOINT workflow_documents_schema")
|
||||
try:
|
||||
error = workflow_documents_schema_error(conn)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
present = _reference_columns(conn)
|
||||
for column in DOCUMENT_ID_COLUMNS:
|
||||
if column not in present:
|
||||
conn.execute(
|
||||
f"ALTER TABLE generated_images ADD COLUMN {column} "
|
||||
f"{_REFERENCE_TYPE}"
|
||||
)
|
||||
for column, index in INDEXES.items():
|
||||
conn.execute(
|
||||
f"CREATE INDEX IF NOT EXISTS {index} ON generated_images({column})"
|
||||
)
|
||||
error = workflow_documents_schema_error(conn, allow_missing=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
sweep_documents(conn)
|
||||
conn.execute("RELEASE workflow_documents_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO workflow_documents_schema")
|
||||
conn.execute("RELEASE workflow_documents_schema")
|
||||
raise
|
||||
|
||||
|
||||
# ── write ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def storable(text: Any) -> bool:
|
||||
"""Whether ``text`` can be a document (text within MAX_DOCUMENT_BYTES)."""
|
||||
if not isinstance(text, str):
|
||||
return False
|
||||
# UTF-8 takes at most 4 bytes a character: most texts need no encoding.
|
||||
if len(text) * 4 <= MAX_DOCUMENT_BYTES:
|
||||
return True
|
||||
return len(text.encode("utf-8")) <= MAX_DOCUMENT_BYTES
|
||||
|
||||
|
||||
def store_text(
|
||||
conn: sqlite3.Connection, text: Optional[str], verified: Optional[set] = None
|
||||
) -> Optional[int]:
|
||||
"""The id of the document holding exactly ``text``; None for None.
|
||||
|
||||
Reuses the document with the same hash, or inserts a new one whose
|
||||
compressed bytes were checked to decompress to ``text``'s bytes. A reused
|
||||
document is decompressed and checked first; a damaged one (a restored
|
||||
file's corruption under a valid hash) is repaired from ``text``, whose
|
||||
hash is its key, so every row that references it reads right again.
|
||||
``verified``, when given, holds the ids already checked (a migration
|
||||
batch), which are then not read again. Opens a write transaction first
|
||||
when none is open, so a concurrent ``release_documents`` cannot remove a
|
||||
reused document before the caller references it. Raises TypeError for a
|
||||
non-text value and ValueError for a text over MAX_DOCUMENT_BYTES.
|
||||
"""
|
||||
if text is None:
|
||||
return None
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("Only text is stored as a workflow document")
|
||||
raw = text.encode("utf-8")
|
||||
if len(raw) > MAX_DOCUMENT_BYTES:
|
||||
raise ValueError("Text is too large for a workflow document")
|
||||
digest = hashlib.sha256(raw).digest()
|
||||
if not conn.in_transaction:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
row = conn.execute(LOOKUP_SQL, (digest,)).fetchone()
|
||||
if row is not None and verified is not None and row[0] in verified:
|
||||
return row[0]
|
||||
data = zlib.compress(raw, COMPRESSION_LEVEL)
|
||||
if zlib.decompress(data) != raw: # pragma: no cover - zlib is exact
|
||||
raise DocumentMismatch("Compression did not round-trip")
|
||||
if row is None:
|
||||
document_id = conn.execute(INSERT_SQL, (digest, len(raw), data)).lastrowid
|
||||
else:
|
||||
document_id = row[0]
|
||||
if _decode(row[1:], document_id, quiet=True) != text:
|
||||
logger.warning(f"Workflow document {document_id} was damaged; repaired")
|
||||
conn.execute(REPAIR_SQL, (len(raw), data, document_id))
|
||||
if verified is not None:
|
||||
verified.add(document_id)
|
||||
return document_id
|
||||
|
||||
|
||||
def stored_values(
|
||||
conn: sqlite3.Connection, texts: Dict[str, Optional[str]]
|
||||
) -> Dict[str, Any]:
|
||||
"""New-form column values for ``{text column: text}`` (writers).
|
||||
|
||||
Each text becomes ``{column: None, id column: document id}``; None
|
||||
stays ``{column: None, id column: None}``; a value that cannot be a
|
||||
document (not text, or too large) stays inline with no id.
|
||||
"""
|
||||
references = dict(TEXT_COLUMNS)
|
||||
values: Dict[str, Any] = {}
|
||||
for column, text in texts.items():
|
||||
reference = references[column]
|
||||
if text is None or not storable(text):
|
||||
values[column], values[reference] = text, None
|
||||
else:
|
||||
values[column], values[reference] = None, store_text(conn, text)
|
||||
return values
|
||||
|
||||
|
||||
# ── read ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _decode(row, document_id, quiet=False) -> Optional[str]:
|
||||
"""``(sha256, byte_length, data)`` as checked text, or None (logged)."""
|
||||
if row is None:
|
||||
logger.warning(f"Workflow document {document_id} is missing")
|
||||
return None
|
||||
digest, length, data = bytes(row[0]), row[1], row[2]
|
||||
try:
|
||||
if not isinstance(length, int) or not 0 <= length <= MAX_DOCUMENT_BYTES:
|
||||
raise ValueError("invalid length")
|
||||
inflater = zlib.decompressobj()
|
||||
raw = inflater.decompress(bytes(data), length + 1)
|
||||
if len(raw) != length or inflater.unconsumed_tail or not inflater.eof:
|
||||
raise ValueError("length differs")
|
||||
if hashlib.sha256(raw).digest() != digest:
|
||||
raise ValueError("hash differs")
|
||||
return raw.decode("utf-8")
|
||||
except (zlib.error, ValueError, TypeError) as error:
|
||||
if not quiet:
|
||||
logger.warning(f"Workflow document {document_id} is unreadable: {error}")
|
||||
return None
|
||||
|
||||
|
||||
def read_stored_text(
|
||||
conn: sqlite3.Connection, inline: Any, document_id: Optional[int]
|
||||
) -> Any:
|
||||
"""The stored text of one column: inline when present, else the document.
|
||||
|
||||
Returns None when there is neither, or when the document is missing or
|
||||
fails its checks (logged). An inline value is returned unchanged.
|
||||
"""
|
||||
if inline is not None:
|
||||
return inline
|
||||
if document_id is None:
|
||||
return None
|
||||
return _decode(conn.execute(READ_SQL, (document_id,)).fetchone(), document_id)
|
||||
|
||||
|
||||
class DocumentReader:
|
||||
"""``read_stored_text`` with a small memo for list reads on one connection."""
|
||||
|
||||
MEMO = 32
|
||||
|
||||
def __init__(self, conn: sqlite3.Connection):
|
||||
self.conn = conn
|
||||
self._memo: "OrderedDict[int, Optional[str]]" = OrderedDict()
|
||||
|
||||
def text(self, inline: Any, document_id: Optional[int]) -> Any:
|
||||
if inline is not None or document_id is None:
|
||||
return read_stored_text(self.conn, inline, document_id)
|
||||
if document_id in self._memo:
|
||||
self._memo.move_to_end(document_id)
|
||||
return self._memo[document_id]
|
||||
text = read_stored_text(self.conn, None, document_id)
|
||||
self._memo[document_id] = text
|
||||
if len(self._memo) > self.MEMO:
|
||||
self._memo.popitem(last=False)
|
||||
return text
|
||||
|
||||
|
||||
def resolve_image_row(reader: DocumentReader, row) -> Dict[str, Any]:
|
||||
"""A ``generated_images`` row as a dict with its texts read back.
|
||||
|
||||
The id columns are dropped, so a payload built from it has the shape it
|
||||
had before the dedupe. Columns the row did not select stay absent.
|
||||
"""
|
||||
data = dict(row)
|
||||
for column, reference in TEXT_COLUMNS:
|
||||
has_reference = reference in data
|
||||
document_id = data.pop(reference, None)
|
||||
if column in data or has_reference:
|
||||
data[column] = reader.text(data.get(column), document_id)
|
||||
return data
|
||||
|
||||
|
||||
# ── garbage ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _chunks(values):
|
||||
values = list(values)
|
||||
for start in range(0, len(values), ID_BATCH):
|
||||
yield values[start : start + ID_BATCH]
|
||||
|
||||
|
||||
def image_documents(conn: sqlite3.Connection, where: str, params) -> Set[int]:
|
||||
"""The document ids the image rows matching ``where`` reference.
|
||||
|
||||
``where`` is a fixed condition written in this package (``id IN (...)``,
|
||||
``prompt_id IN (...)``), never caller data; values go in ``params``.
|
||||
"""
|
||||
ids: Set[int] = set()
|
||||
sql = IMAGE_DOCUMENTS_SQL + where
|
||||
for row in conn.execute(sql, list(params)): # nosec B608
|
||||
ids.update(value for value in row if value is not None)
|
||||
return ids
|
||||
|
||||
|
||||
def documents_of_rows(
|
||||
conn: sqlite3.Connection, column: str, values: Iterable[Any]
|
||||
) -> Set[int]:
|
||||
"""Document ids of the image rows whose ``column`` is in ``values``.
|
||||
|
||||
``column`` is ``id`` or ``prompt_id`` (the rows a prompt delete cascades to).
|
||||
"""
|
||||
if column not in ("id", "prompt_id"):
|
||||
raise ValueError(f"Unsupported column {column!r}")
|
||||
ids: Set[int] = set()
|
||||
for chunk in _chunks(values):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
ids |= image_documents(conn, f"{column} IN ({marks})", chunk)
|
||||
return ids
|
||||
|
||||
|
||||
def release_documents(conn: sqlite3.Connection, ids: Iterable[Optional[int]]) -> int:
|
||||
"""Delete the documents in ``ids`` that no image row references now.
|
||||
|
||||
Runs in the caller's transaction, right after the delete or update that
|
||||
dropped the references. Returns how many documents were removed.
|
||||
"""
|
||||
removed = 0
|
||||
for chunk in _chunks(sorted({i for i in ids if i is not None})):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
# Only "?" marks are interpolated.
|
||||
sql = RELEASE_SQL.format(marks=marks)
|
||||
removed += conn.execute(sql, chunk).rowcount # nosec B608
|
||||
return removed
|
||||
|
||||
|
||||
def sweep_documents(conn: sqlite3.Connection) -> int:
|
||||
"""Delete every document no image row references; returns the count."""
|
||||
return conn.execute(SWEEP_SQL).rowcount
|
||||
@@ -0,0 +1,455 @@
|
||||
"""The workflow storage migration and Reclaim space (#249 S6 carry-over).
|
||||
|
||||
``migrate_workflow_storage`` rewrites old-form image rows (workflow and
|
||||
prompt graph text inline) to the new form (a ``workflow_documents`` id,
|
||||
``database/workflow_documents.py``) in batches:
|
||||
|
||||
- each batch is one ``BEGIN IMMEDIATE`` transaction over at most
|
||||
``BATCH_ROWS`` rows or about ``BATCH_BYTES`` of text, found by a rowid
|
||||
range search (``NEXT_IDS_SQL``) and read one row at a time, so memory is
|
||||
bounded by one row plus the batch's distinct documents;
|
||||
- each text is stored (or its document reused) and read back through the
|
||||
one accessor before its row changes; a text that would not come back
|
||||
identical fails the batch (``DocumentMismatch``) and rolls it back whole;
|
||||
- a value that is not text (or is over ``MAX_DOCUMENT_BYTES``) stays inline
|
||||
and is counted as protected;
|
||||
- each row runs in its own savepoint: one that cannot be rewritten (text
|
||||
SQLite cannot decode, a text that would not come back) stays inline and
|
||||
is listed, and the rest of the batch goes on;
|
||||
- a batch ends after ``BATCH_SECONDS``, and the write lock is then left
|
||||
free for ``BATCH_PAUSE``, so other writers are never starved.
|
||||
|
||||
A crash or a stop between batches leaves committed batches in the new form
|
||||
and the rest in the old form; both read the same, and a rerun continues
|
||||
from the first old-form row. Only database rows change: no image or video
|
||||
file is opened.
|
||||
|
||||
``reclaim_space`` is the VACUUM afterwards: it sweeps unreferenced
|
||||
documents, rebuilds the file and checkpoints the WAL with TRUNCATE, so the
|
||||
freed pages go back to the disk.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
from contextlib import closing
|
||||
|
||||
from .workflow_documents import (
|
||||
TEXT_COLUMNS,
|
||||
DocumentMismatch,
|
||||
read_stored_text,
|
||||
release_documents,
|
||||
storable,
|
||||
store_text,
|
||||
sweep_documents,
|
||||
)
|
||||
|
||||
OPERATION = "workflow_storage"
|
||||
# The first apply's backup: ``<db>.pre-workflow-storage_<stamp>``, outside the
|
||||
# pruned ``<db>.backup_*`` pool. Only one is ever made; later applies take a
|
||||
# pooled safety backup.
|
||||
PRE_MIGRATION_LABEL = "pre-workflow-storage"
|
||||
BATCH_ROWS = 200
|
||||
BATCH_BYTES = 16 * 2**20 # characters of text rewritten per transaction
|
||||
# A batch holds the write lock: it ends after this long, and the loop then
|
||||
# leaves the lock free for BATCH_PAUSE, longer than SQLite's longest busy
|
||||
# poll (100 ms), so the image monitor and the run counting get in between.
|
||||
BATCH_SECONDS = 0.5
|
||||
BATCH_PAUSE = 0.15
|
||||
|
||||
|
||||
def _row_error(error) -> bool:
|
||||
"""Whether ``error`` is one row's own cause, which keeps only that row inline.
|
||||
|
||||
Text SQLite cannot decode for Python, and a text that would not come
|
||||
back identical. Anything else (a full disk, an I/O error, a malformed
|
||||
database, a bug) fails the whole batch. The decode error is matched by
|
||||
its message: ``sqlite_errorcode`` needs Python 3.11.
|
||||
"""
|
||||
if isinstance(error, (DocumentMismatch, UnicodeError)):
|
||||
return True
|
||||
return isinstance(error, sqlite3.OperationalError) and str(error).startswith(
|
||||
"Could not decode"
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.workflow_storage")
|
||||
_clock = time.monotonic
|
||||
|
||||
# typeof() reads a column's type from the record header: no overflow page.
|
||||
NEXT_IDS_SQL = (
|
||||
"SELECT id FROM generated_images WHERE id>? AND "
|
||||
"(typeof(workflow_data)!='null' OR typeof(prompt_metadata)!='null') "
|
||||
"ORDER BY id LIMIT ?"
|
||||
)
|
||||
ROW_SQL = (
|
||||
"SELECT workflow_data, prompt_metadata, workflow_document_id, "
|
||||
"prompt_document_id FROM generated_images WHERE id=?"
|
||||
)
|
||||
UPDATE_SQL = (
|
||||
"UPDATE generated_images SET {assignments} WHERE id=?" # columns are constants
|
||||
)
|
||||
# One pass over the record headers: rows with inline text, and inline
|
||||
# values that are not text (they stay inline).
|
||||
INLINE_COUNTS_SQL = (
|
||||
"SELECT "
|
||||
"SUM(typeof(workflow_data)='text' OR typeof(prompt_metadata)='text'), "
|
||||
"SUM(typeof(workflow_data) NOT IN ('null','text')) "
|
||||
"+ SUM(typeof(prompt_metadata) NOT IN ('null','text')) "
|
||||
"FROM generated_images"
|
||||
)
|
||||
# octet_length() (SQLite 3.43+) reads no overflow page either.
|
||||
INLINE_BYTES_SQL = (
|
||||
"SELECT SUM(octet_length(workflow_data)), SUM(octet_length(prompt_metadata)) "
|
||||
"FROM generated_images"
|
||||
)
|
||||
|
||||
QUERY_PLAN_STATEMENTS = {
|
||||
"next old-form rows": (NEXT_IDS_SQL, (0, BATCH_ROWS)),
|
||||
"one row": (ROW_SQL, (1,)),
|
||||
}
|
||||
|
||||
|
||||
def _rewrite_row(conn, image_id, row, verified, checked):
|
||||
"""Store the row's inline texts as documents; returns (changed, protected, size).
|
||||
|
||||
``verified`` maps document ids already read back in this batch to their
|
||||
text, and ``checked`` holds the reused ids ``store_text`` already
|
||||
checked, so a document shared by many rows is decompressed once.
|
||||
"""
|
||||
changes, replaced = {}, []
|
||||
protected = size = 0
|
||||
for index, (column, reference) in enumerate(TEXT_COLUMNS):
|
||||
value, current = row[index], row[index + len(TEXT_COLUMNS)]
|
||||
if value is None:
|
||||
continue
|
||||
if not storable(value):
|
||||
protected += 1
|
||||
continue
|
||||
document_id = store_text(conn, value, checked)
|
||||
if document_id not in verified:
|
||||
verified[document_id] = read_stored_text(conn, None, document_id)
|
||||
if verified[document_id] != value:
|
||||
raise DocumentMismatch(f"Image {image_id}: {column} would change")
|
||||
changes[column], changes[reference] = None, document_id
|
||||
if current is not None and current != document_id:
|
||||
replaced.append(current)
|
||||
size += len(value)
|
||||
if changes:
|
||||
assignments = ",".join(f"{key}=?" for key in changes)
|
||||
conn.execute(
|
||||
UPDATE_SQL.format(assignments=assignments),
|
||||
[*changes.values(), image_id],
|
||||
)
|
||||
release_documents(conn, replaced)
|
||||
return bool(changes), protected, size
|
||||
|
||||
|
||||
def migrate_batch(
|
||||
conn,
|
||||
after_id,
|
||||
max_rows=BATCH_ROWS,
|
||||
max_bytes=BATCH_BYTES,
|
||||
max_seconds=BATCH_SECONDS,
|
||||
):
|
||||
"""Rewrite the next batch after ``after_id`` in one transaction.
|
||||
|
||||
The batch ends after ``max_rows`` rows, about ``max_bytes`` of text or
|
||||
``max_seconds``, whichever comes first. Each row runs in its own
|
||||
savepoint: a row that cannot be rewritten (``_row_error``) is rolled
|
||||
back alone, stays inline and is listed in ``errors``; anything else rolls
|
||||
the whole batch back and propagates. Returns ``{last_id, rows,
|
||||
protected, errors, done}``: the last row looked at, rows rewritten,
|
||||
inline values kept (not text), the ids kept by an error and whether
|
||||
there was nothing left to look at.
|
||||
"""
|
||||
if conn.in_transaction:
|
||||
raise RuntimeError("The migration needs its own transaction")
|
||||
started = _clock()
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
ids = [row[0] for row in conn.execute(NEXT_IDS_SQL, (after_id, max_rows))]
|
||||
result = {
|
||||
"last_id": after_id,
|
||||
"rows": 0,
|
||||
"protected": 0,
|
||||
"errors": [],
|
||||
"done": not ids,
|
||||
}
|
||||
verified, checked, size = {}, set(), 0
|
||||
for image_id in ids:
|
||||
conn.execute("SAVEPOINT workflow_row")
|
||||
try:
|
||||
row = conn.execute(ROW_SQL, (image_id,)).fetchone()
|
||||
changed, protected, row_size = _rewrite_row(
|
||||
conn, image_id, row, verified, checked
|
||||
)
|
||||
except Exception as error:
|
||||
if not _row_error(error):
|
||||
raise
|
||||
conn.execute("ROLLBACK TO workflow_row")
|
||||
conn.execute("RELEASE workflow_row")
|
||||
verified.clear() # the row's documents were rolled back too
|
||||
checked.clear()
|
||||
logger.warning(f"Image record {image_id} kept inline: {error}")
|
||||
result["errors"].append(image_id)
|
||||
changed, protected, row_size = False, 0, 0
|
||||
else:
|
||||
conn.execute("RELEASE workflow_row")
|
||||
result["rows"] += int(changed)
|
||||
result["protected"] += protected
|
||||
result["last_id"] = image_id
|
||||
size += row_size
|
||||
if size >= max_bytes or _clock() - started >= max_seconds:
|
||||
break
|
||||
conn.commit()
|
||||
return result
|
||||
except BaseException:
|
||||
conn.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def pre_migration_backups(db_path):
|
||||
"""The kept pre-migration backups beside ``db_path``, earliest first."""
|
||||
db_path = os.path.abspath(db_path)
|
||||
prefix = os.path.basename(db_path) + "." + PRE_MIGRATION_LABEL + "_"
|
||||
folder = os.path.dirname(db_path)
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
return []
|
||||
return sorted(
|
||||
name
|
||||
for name in names
|
||||
if name.startswith(prefix) and os.path.isfile(os.path.join(folder, name))
|
||||
)
|
||||
|
||||
|
||||
def _file_bytes(db_path):
|
||||
return sum(
|
||||
os.path.getsize(path)
|
||||
for path in (db_path, db_path + "-wal")
|
||||
if os.path.exists(path)
|
||||
)
|
||||
|
||||
|
||||
class WorkflowStorageOperations:
|
||||
"""PromptDatabase mixin: the workflow storage migration and Reclaim space."""
|
||||
|
||||
def migrate_workflow_storage(
|
||||
self,
|
||||
*,
|
||||
on_batch=None,
|
||||
should_stop=None,
|
||||
max_rows=BATCH_ROWS,
|
||||
max_bytes=BATCH_BYTES,
|
||||
) -> dict:
|
||||
"""Rewrite every old-form row, batch by batch (blocking).
|
||||
|
||||
``on_batch(progress)`` runs after each committed batch with the
|
||||
running totals ``{rows, protected, batches, last_id}``;
|
||||
``should_stop()`` is asked before each batch. Each committed batch is
|
||||
followed by ``BATCH_PAUSE`` with the write lock free. Returns
|
||||
``{rows, protected, batches, stopped, errors, documents_removed}``
|
||||
(``errors``: the ids of rows kept inline by an error). Unreferenced
|
||||
documents are swept once every row was looked at.
|
||||
"""
|
||||
totals = {"rows": 0, "protected": 0, "batches": 0, "last_id": 0}
|
||||
errors = []
|
||||
stopped = False
|
||||
while True:
|
||||
if should_stop is not None and should_stop():
|
||||
stopped = True
|
||||
break
|
||||
batch = self.migrate_workflow_batch(totals["last_id"], max_rows, max_bytes)
|
||||
if batch["done"]:
|
||||
break
|
||||
time.sleep(BATCH_PAUSE) # leave the write lock to other writers
|
||||
errors += batch["errors"]
|
||||
totals["rows"] += batch["rows"]
|
||||
totals["protected"] += batch["protected"]
|
||||
totals["batches"] += 1
|
||||
totals["last_id"] = batch["last_id"]
|
||||
if on_batch is not None:
|
||||
on_batch(dict(totals))
|
||||
removed = 0 if stopped else self.sweep_workflow_documents()
|
||||
return {
|
||||
"rows": totals["rows"],
|
||||
"protected": totals["protected"],
|
||||
"batches": totals["batches"],
|
||||
"stopped": stopped,
|
||||
"errors": errors,
|
||||
"documents_removed": removed,
|
||||
}
|
||||
|
||||
def migrate_workflow_batch(
|
||||
self, after_id, max_rows=BATCH_ROWS, max_bytes=BATCH_BYTES
|
||||
) -> dict:
|
||||
"""One batch on the calling thread's connection (``migrate_batch``)."""
|
||||
return migrate_batch(self.model.get_connection(), after_id, max_rows, max_bytes)
|
||||
|
||||
def sweep_workflow_documents(self) -> int:
|
||||
"""Remove every document no image row references (one transaction)."""
|
||||
conn = self.model.get_connection()
|
||||
with conn:
|
||||
return sweep_documents(conn)
|
||||
|
||||
def workflow_storage_status(self) -> dict:
|
||||
"""``workflow_storage_counts`` on one read-only snapshot (blocking)."""
|
||||
from .maintenance import _read_snapshot
|
||||
|
||||
return _read_snapshot(self.model.db_path, self.workflow_storage_counts)
|
||||
|
||||
def workflow_storage_counts(self, conn) -> dict:
|
||||
"""``{rows, protected, documents, inline_bytes}`` on ``conn`` (reads only).
|
||||
|
||||
``inline_bytes`` is None where SQLite has no ``octet_length``.
|
||||
"""
|
||||
rows, protected = conn.execute(INLINE_COUNTS_SQL).fetchone()
|
||||
documents = conn.execute("SELECT COUNT(*) FROM workflow_documents").fetchone()
|
||||
inline_bytes = None
|
||||
if sqlite3.sqlite_version_info >= (3, 43, 0):
|
||||
row = conn.execute(INLINE_BYTES_SQL).fetchone()
|
||||
inline_bytes = sum(value or 0 for value in row)
|
||||
return {
|
||||
"rows": rows or 0,
|
||||
"protected": protected or 0,
|
||||
"documents": documents[0],
|
||||
"inline_bytes": inline_bytes,
|
||||
}
|
||||
|
||||
def reclaim_space(self) -> dict:
|
||||
"""Sweep, VACUUM and truncate the WAL (blocking); file bytes before/after.
|
||||
|
||||
The caller holds the registry's ``exclusive`` slot. Raises whatever
|
||||
stopped VACUUM; the database is then unchanged.
|
||||
"""
|
||||
db_path = self.model.db_path
|
||||
removed = self.sweep_workflow_documents()
|
||||
before = _file_bytes(db_path)
|
||||
self.model.vacuum_database()
|
||||
with closing(sqlite3.connect(db_path)) as checkpoint:
|
||||
busy = checkpoint.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()[0]
|
||||
return {
|
||||
"before_bytes": before,
|
||||
"after_bytes": _file_bytes(db_path),
|
||||
"documents_removed": removed,
|
||||
"checkpoint_busy": bool(busy),
|
||||
}
|
||||
|
||||
# ── preview ───────────────────────────────────────────────────────
|
||||
|
||||
def _plan_workflow_storage(self):
|
||||
"""The reviewed plan of the migration (one read-only snapshot)."""
|
||||
from .maintenance import (
|
||||
MaintenanceDraft,
|
||||
_Fingerprint,
|
||||
_megabytes,
|
||||
_plural,
|
||||
_read_snapshot,
|
||||
)
|
||||
|
||||
def read(conn):
|
||||
sizes = [
|
||||
conn.execute(f"PRAGMA {name}").fetchone()[0]
|
||||
for name in ("page_size", "page_count", "freelist_count")
|
||||
]
|
||||
return sizes, self.workflow_storage_counts(conn)
|
||||
|
||||
(page_size, page_count, free_pages), counts = _read_snapshot(
|
||||
self.model.db_path, read
|
||||
)
|
||||
size = page_size * page_count
|
||||
details = [
|
||||
{
|
||||
"label": "Database file",
|
||||
"detail": f"{_megabytes(size)}, of which "
|
||||
f"{_megabytes(free_pages * page_size)} is free pages",
|
||||
},
|
||||
{
|
||||
"label": "Image records",
|
||||
"detail": f"{_plural(counts['rows'], 'record')} keep their workflow "
|
||||
"or prompt graph inline; each distinct text is stored once, "
|
||||
"compressed",
|
||||
},
|
||||
]
|
||||
if not counts["rows"]:
|
||||
details[1]["detail"] = NOTHING_TO_DO
|
||||
if counts["inline_bytes"] is not None:
|
||||
details.append(
|
||||
{
|
||||
"label": "Inline text",
|
||||
"detail": f"{_megabytes(counts['inline_bytes'])} now",
|
||||
}
|
||||
)
|
||||
if counts["documents"]:
|
||||
details.append(
|
||||
{
|
||||
"label": "Already stored once",
|
||||
"detail": _plural(counts["documents"], "workflow document"),
|
||||
}
|
||||
)
|
||||
name = os.path.basename(self.model.db_path)
|
||||
kept = pre_migration_backups(self.model.db_path)
|
||||
backup = (
|
||||
POOLED_BACKUP_NOTE.format(
|
||||
name=f"{name}.backup_<date and time>", kept=kept[0]
|
||||
)
|
||||
if kept
|
||||
else BACKUP_NOTE.format(
|
||||
name=f"{name}.{PRE_MIGRATION_LABEL}_<date and time>"
|
||||
)
|
||||
)
|
||||
details += [
|
||||
{"label": "Backup", "detail": backup},
|
||||
{"label": "Reclaim space", "detail": RECLAIM_NOTE},
|
||||
{"label": "Older versions", "detail": OLDER_VERSIONS_NOTE},
|
||||
]
|
||||
protected = counts["protected"]
|
||||
return MaintenanceDraft(
|
||||
operation=OPERATION,
|
||||
fingerprint=_Fingerprint(OPERATION).hexdigest(),
|
||||
targets=[],
|
||||
scope={
|
||||
"count": counts["rows"],
|
||||
"label": (
|
||||
_plural(counts["rows"], "image record") + " to rewrite"
|
||||
if counts["rows"]
|
||||
else "Nothing to rewrite"
|
||||
),
|
||||
},
|
||||
protected={
|
||||
"count": protected,
|
||||
"reasons": (
|
||||
[f"{_plural(protected, 'stored value')} not text; kept as is"]
|
||||
if protected
|
||||
else []
|
||||
),
|
||||
},
|
||||
media={"deletes_files": False, "label": MEDIA_NOTE},
|
||||
details=details,
|
||||
)
|
||||
|
||||
|
||||
MEDIA_NOTE = "No image or video file is opened or changed; only the database."
|
||||
RECLAIM_NOTE = (
|
||||
"When every record is rewritten, Reclaim space runs alone to give the freed "
|
||||
"space back to the disk (VACUUM). The database is locked while it runs."
|
||||
)
|
||||
BACKUP_NOTE = (
|
||||
"Always taken first, whatever the safety backup setting: {name} beside "
|
||||
"the database, verified, and kept until you remove it (it is never pruned)."
|
||||
)
|
||||
POOLED_BACKUP_NOTE = (
|
||||
"A safety backup is taken first, whatever the safety backup setting: {name} "
|
||||
"among the pooled backups; the pre-migration backup {kept} is already kept."
|
||||
)
|
||||
NOTHING_TO_DO = "Every image record already stores its workflow once."
|
||||
OLDER_VERSIONS_NOTE = (
|
||||
"This version stores the workflows of images it records in the new form, "
|
||||
"whether or not this job runs, and this job rewrites the rest. An older "
|
||||
"Prompt Manager version shows those images without their workflow until you "
|
||||
"update again. The way back to an older version is the earliest "
|
||||
"pre-workflow-storage backup."
|
||||
)
|
||||
@@ -12,6 +12,7 @@ code. This plan proposes implementation order and acceptance gates.
|
||||
- The native Prompt Library merged in [#210](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/210) for [#208](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/208): one positive parent, saved negative variants, full selected text and a shared filtered image gallery. It does not invent chronological runs from counters or scan dates.
|
||||
- [Durable history storage](prompt-history-storage.md) records job attempts and outcomes from [#212](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/212), frozen prompt contexts from [#213](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/213), and observed run-owned output locators from [#214](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/214). Those locators do not verify file bytes or establish gallery-image associations. That association contract and the chronological execution timeline remain pending under [#209](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/209).
|
||||
- [#215](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/215) brings the shell, Images and natural-height Prompt Library presentation into the approved geometry and themes while retaining their existing operations. [#216](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/216) adds the shared current-file inspector with exact generation facts, scoped provenance, raw/download evidence and loaded-collection viewer controls, including a slideshow that pauses at native video. This delivery does not establish whole-application parity, change the default entry point or constitute a release. Recorded chronological run/context browsing continues under [#209](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/209); image tagging/review, System consolidation and remaining organization capabilities continue under #196.
|
||||
- Local Inspect (#249, PRs L1–L2) replaces the `#/inspect` handoff with the concept's native "Inspect a file" page: one local image or video previews inline and opens the shared viewer and full inspector on a record read entirely in the browser, with nothing uploaded, imported or stored. The legacy metadata page (`web/metadata.html`, served at `/prompt_manager/gallery.html`) stays reachable from Overview; its capabilities map to the shared inspector, except its one-click COPY ALL summary, whose ruled replacement ("Copy settings summary") is pending. Recorded deviations D60–D66 are in [TESTING.md](../TESTING.md). Local roles equal to the server's resolver follow in PR L3.
|
||||
|
||||
## Scope and compatibility
|
||||
|
||||
|
||||
@@ -29,6 +29,12 @@ Each role has `status` (`resolved`, `absent`, `unresolved`, or `ambiguous`), `te
|
||||
(string only when resolved, otherwise null), and `source`. Inspection does not
|
||||
index files, alter saved prompts, run scans, or decode image pixels. PNG text and
|
||||
video probe output are bounded; unsupported or malformed evidence remains
|
||||
unresolved rather than becoming sendable placeholder text.
|
||||
unresolved rather than becoming sendable placeholder text. Videos are read from
|
||||
one container carrier: the per-key tags written by core SaveVideo/SaveWEBM and
|
||||
current VideoHelperSuite (`prompt`, `workflow`, `parameters`,
|
||||
`comfyassets_prompt_pairs`), else the old VideoHelperSuite `comment` envelope,
|
||||
else `description`. The admin scan reads the same carrier, so Images and
|
||||
ingestion agree on a video's prompts (see TESTING.md for the order and budgets;
|
||||
the live monitor processes images only).
|
||||
|
||||
Python route changes require restarting ComfyUI. Browser assets require a refresh.
|
||||
|
||||
+126
-15
@@ -7,6 +7,17 @@ Split into domain-specific mixins for maintainability:
|
||||
- LoggingRoutesMixin: log management endpoints
|
||||
- AutotagRoutesMixin: auto-tagging model management and tagging
|
||||
- JobRoutesMixin: the session job registry behind Activity (py/api/jobs.py)
|
||||
- ImageTagJobsMixin: bulk Add tags / Auto-tag as image-tags jobs
|
||||
- ThumbnailJobsMixin: thumbnail plan, thumbnail job and the gallery stream on it
|
||||
- AutoThumbnailMixin: previews made on first view, on capture and after a scan
|
||||
- ReviewRoutesMixin: the Needs review queue, item detail and decisions
|
||||
- ReviewRecordingMixin: Needs review recording for the scan and the monitor
|
||||
- ReviewEventsMixin: the Needs review change stream an open page re-reads on
|
||||
- SystemRoutesMixin: System health check, Overview and usage statistics
|
||||
- LibraryLocationRoutesMixin: where the library lives, Move library, Keep it here
|
||||
- MaintenanceRoutesMixin: reviewed maintenance plans (preview, apply)
|
||||
- WorkflowStorageRoutesMixin: store each workflow once, then Reclaim space
|
||||
- DuplicateFilesMixin: duplicate image files as find and removal jobs
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -14,6 +25,7 @@ import datetime
|
||||
import functools
|
||||
import gzip as gzip_module
|
||||
import json
|
||||
import math
|
||||
import ntpath
|
||||
import os
|
||||
import re
|
||||
@@ -26,11 +38,25 @@ from .prompts import PromptRoutesMixin
|
||||
from .pairs import PairRoutesMixin
|
||||
from .library import LibraryRoutesMixin
|
||||
from .vocabulary import VocabularyRoutesMixin
|
||||
from .annotations import AnnotationRoutesMixin
|
||||
from .review import ReviewRoutesMixin
|
||||
from .review_recording import ReviewRecordingMixin
|
||||
from .review_events import ReviewEventsMixin
|
||||
from .saved_views import SavedViewRoutesMixin
|
||||
from .history import HistoryRoutesMixin
|
||||
from .images import ImageRoutesMixin
|
||||
from .admin import AdminRoutesMixin
|
||||
from .settings_routes import SettingsRoutesMixin
|
||||
from .job_routes import JobRoutesMixin
|
||||
from .image_tag_jobs import ImageTagJobsMixin
|
||||
from .thumbnail_jobs import ThumbnailJobsMixin
|
||||
from .thumbnail_auto import AutoThumbnailMixin
|
||||
from .system import SystemRoutesMixin
|
||||
from .library_location import LibraryLocationRoutesMixin
|
||||
from .maintenance import MaintenanceRoutesMixin
|
||||
from .workflow_storage import WorkflowStorageRoutesMixin
|
||||
from .duplicate_files import DuplicateFilesMixin
|
||||
from .maintenance_plans import PlanStore
|
||||
from .jobs import JobRegistry
|
||||
from .logging_routes import LoggingRoutesMixin
|
||||
from .autotag_routes import AutotagRoutesMixin
|
||||
@@ -104,16 +130,20 @@ STATIC_MIME_TYPES = {
|
||||
".html": "text/html",
|
||||
}
|
||||
_STATIC_SEPARATORS = re.compile(r"[\\/]+")
|
||||
# Served by exact request path beside the extension allow-list: the browser's
|
||||
# copy of the inspection node table (local Inspect resolves with it, #249).
|
||||
STATIC_EXACT_FILES = {("js", "inspection-semantics.json"): "application/json"}
|
||||
|
||||
|
||||
def _resolve_static_file(root, filepath):
|
||||
def _resolve_static_file(root, filepath, allowed=STATIC_MIME_TYPES):
|
||||
"""Real path of ``filepath`` under ``root``, or None when it must not be served.
|
||||
|
||||
The request path is untrusted. It is rejected when it is empty, carries a
|
||||
``..`` segment, is absolute on any platform (a leading slash or backslash,
|
||||
a drive letter such as ``C:`` or a UNC ``\\\\server\\share`` prefix all
|
||||
re-anchor ``os.path.join`` on Windows), has an extension outside
|
||||
STATIC_MIME_TYPES, or resolves (symlinks included) outside ``root``.
|
||||
``allowed`` (STATIC_MIME_TYPES), or resolves (symlinks included) outside
|
||||
``root``.
|
||||
"""
|
||||
if not isinstance(filepath, str) or not filepath:
|
||||
return None
|
||||
@@ -126,7 +156,7 @@ def _resolve_static_file(root, filepath):
|
||||
segments = _STATIC_SEPARATORS.split(filepath)
|
||||
if any(segment in ("", ".", "..") for segment in segments):
|
||||
return None
|
||||
if os.path.splitext(filepath)[1].lower() not in STATIC_MIME_TYPES:
|
||||
if os.path.splitext(filepath)[1].lower() not in allowed:
|
||||
return None
|
||||
try:
|
||||
root_real = Path(os.path.normcase(os.path.realpath(root)))
|
||||
@@ -252,12 +282,25 @@ class PromptManagerAPI(
|
||||
PairRoutesMixin,
|
||||
LibraryRoutesMixin,
|
||||
VocabularyRoutesMixin,
|
||||
AnnotationRoutesMixin,
|
||||
ReviewRoutesMixin,
|
||||
ReviewRecordingMixin,
|
||||
ReviewEventsMixin,
|
||||
SavedViewRoutesMixin,
|
||||
HistoryRoutesMixin,
|
||||
PromptRoutesMixin,
|
||||
ImageRoutesMixin,
|
||||
AdminRoutesMixin,
|
||||
SettingsRoutesMixin,
|
||||
JobRoutesMixin,
|
||||
ImageTagJobsMixin,
|
||||
ThumbnailJobsMixin,
|
||||
AutoThumbnailMixin,
|
||||
SystemRoutesMixin,
|
||||
LibraryLocationRoutesMixin,
|
||||
MaintenanceRoutesMixin,
|
||||
WorkflowStorageRoutesMixin,
|
||||
DuplicateFilesMixin,
|
||||
LoggingRoutesMixin,
|
||||
AutotagRoutesMixin,
|
||||
LoraIntegrationMixin,
|
||||
@@ -285,6 +328,12 @@ class PromptManagerAPI(
|
||||
self.db = db if db is not None else PromptDatabase()
|
||||
self._history_read_fence = None
|
||||
self._history_reads_closed = False
|
||||
# Private media captures of the annotation routes (py/api/annotations.py),
|
||||
# created on first use with their restore subscription.
|
||||
self._captures = None
|
||||
self._captures_unsubscribe = None
|
||||
# Set by close_annotation_captures (shutdown): no store is recreated.
|
||||
self._captures_closed = False
|
||||
self._cached_output_dir = None # Lazy-cached by _find_comfyui_output_dir()
|
||||
self._html_cache = {} # Cached HTML file contents keyed by path
|
||||
self._gallery_cache = {} # dict: path_str -> (files, timestamp)
|
||||
@@ -308,8 +357,31 @@ class PromptManagerAPI(
|
||||
self._jobs = JobRegistry(self.logger)
|
||||
# The latest metadata scan job (also in self._jobs), if one was started.
|
||||
self._scan_job = None
|
||||
# The current duplicate scan (a py/api/jobs.Job outside the registry).
|
||||
self._duplicate_job = None
|
||||
# Duplicate-file finds by job id, with the groups each one holds for
|
||||
# review (py/api/duplicate_files.py); the last few only.
|
||||
self._duplicate_finds = {}
|
||||
# The last System health report this session and the database
|
||||
# generation it describes, bumped when a restore replaces self.db
|
||||
# (py/api/system.py).
|
||||
self._health_report = None
|
||||
self._db_generation = 0
|
||||
# Reviewed maintenance plans under single-use tokens; a restart
|
||||
# forgets them (py/api/maintenance.py).
|
||||
self._maintenance_plans = PlanStore()
|
||||
|
||||
# A safety backup interrupted by a crash leaves its temp copy behind.
|
||||
try:
|
||||
from .maintenance import sweep_safety_temps
|
||||
|
||||
swept = sweep_safety_temps(self.db.model.db_path)
|
||||
if swept:
|
||||
self.logger.info(f"Removed {len(swept)} interrupted backup copies")
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not sweep interrupted backup copies: {e}")
|
||||
|
||||
# A crash between Move library's switch and its rename left the old
|
||||
# node-folder files under their own names (#249 S6b): rename them now.
|
||||
self.finish_interrupted_library_move()
|
||||
|
||||
# Run cleanup on initialization to remove any existing duplicates
|
||||
try:
|
||||
@@ -352,6 +424,24 @@ class PromptManagerAPI(
|
||||
"""409 envelope for a long job that is already running."""
|
||||
return _json_error(f"{label} already running", 409)
|
||||
|
||||
async def _run_held(self, kind, title, admission, operation):
|
||||
"""Run a legacy-page operation under a registry hold (D80).
|
||||
|
||||
While ``operation()`` runs, the registry lists it as a running,
|
||||
non-cancellable job, so workspace jobs its admission class conflicts
|
||||
with are refused, naming it. When the registry refuses the hold, the
|
||||
legacy 409 envelope names the running job and nothing is done. A
|
||||
response of 400 or above ends the hold ``failed``.
|
||||
"""
|
||||
busy = self._jobs.blocker(kind, admission=admission)
|
||||
if busy is not None:
|
||||
return self._long_job_busy_response(busy.title)
|
||||
async with self._jobs.hold(kind, title, admission=admission) as job:
|
||||
response = await operation()
|
||||
if getattr(response, "status", 200) >= 400:
|
||||
job.finish("failed")
|
||||
return response
|
||||
|
||||
async def _run_in_executor(self, func, *args, **kwargs):
|
||||
"""Run a blocking function in the default thread pool executor.
|
||||
|
||||
@@ -441,12 +531,20 @@ class PromptManagerAPI(
|
||||
self._register_pair_routes(routes)
|
||||
self._register_library_routes(routes)
|
||||
self._register_vocabulary_routes(routes)
|
||||
self._register_annotation_routes(routes)
|
||||
self._register_review_routes(routes)
|
||||
self._register_review_event_routes(routes)
|
||||
self._register_saved_view_routes(routes)
|
||||
self._register_history_routes(routes)
|
||||
self._register_prompt_routes(routes)
|
||||
self._register_image_routes(routes)
|
||||
self._register_admin_routes(routes)
|
||||
self._register_settings_routes(routes)
|
||||
self._register_job_routes(routes)
|
||||
self._register_system_routes(routes)
|
||||
self._register_library_location_routes(routes)
|
||||
self._register_maintenance_routes(routes)
|
||||
self._register_duplicate_routes(routes)
|
||||
self._register_logging_routes(routes)
|
||||
self._register_autotag_routes(routes)
|
||||
self._register_lora_routes(routes)
|
||||
@@ -468,15 +566,20 @@ class PromptManagerAPI(
|
||||
def _serve_static_file(self, subdir, filepath):
|
||||
"""Serve ``web/<subdir>/<filepath>``; 403 outside it, 404 when missing."""
|
||||
root = os.path.join(_get_project_root(), "web", subdir)
|
||||
file_path = _resolve_static_file(root, filepath)
|
||||
exact = STATIC_EXACT_FILES.get((subdir, filepath))
|
||||
# An exact file passes the same containment check, its extension only.
|
||||
allowed = STATIC_MIME_TYPES if exact is None else {".json"}
|
||||
file_path = _resolve_static_file(root, filepath, allowed)
|
||||
if file_path is None:
|
||||
return web.Response(text="Forbidden", status=403)
|
||||
content_type = (
|
||||
exact or STATIC_MIME_TYPES[os.path.splitext(file_path)[1].lower()]
|
||||
)
|
||||
if not os.path.isfile(file_path):
|
||||
return web.Response(text="Not Found", status=404)
|
||||
ext = os.path.splitext(file_path)[1].lower()
|
||||
with open(file_path, "rb") as f:
|
||||
content = f.read()
|
||||
return web.Response(body=content, content_type=STATIC_MIME_TYPES[ext])
|
||||
return web.Response(body=content, content_type=content_type)
|
||||
|
||||
def _serve_html_page(self, filename, title, cache=True):
|
||||
"""Serve ``web/<filename>``; 404 when missing, 500 when unreadable."""
|
||||
@@ -560,12 +663,12 @@ class PromptManagerAPI(
|
||||
return prompts
|
||||
|
||||
def _clean_nan_recursive(self, obj):
|
||||
"""Recursively clean NaN values from nested data structures."""
|
||||
"""Recursively replace NaN and Infinity in nested data with None."""
|
||||
if isinstance(obj, dict):
|
||||
return {key: self._clean_nan_recursive(value) for key, value in obj.items()}
|
||||
elif isinstance(obj, list):
|
||||
return [self._clean_nan_recursive(item) for item in obj]
|
||||
elif isinstance(obj, float) and str(obj) == "nan":
|
||||
elif isinstance(obj, float) and not math.isfinite(obj):
|
||||
return None
|
||||
else:
|
||||
return obj
|
||||
@@ -688,6 +791,13 @@ class PromptManagerAPI(
|
||||
Returns:
|
||||
List[Path]: Validated directory paths, possibly empty.
|
||||
"""
|
||||
return self._output_dirs_with_origin()[0]
|
||||
|
||||
def _output_dirs_with_origin(self):
|
||||
"""``(dirs, auto_detected)``: the ``_get_all_output_dirs`` list, and
|
||||
whether it comes from detection or ComfyUI's output rather than from
|
||||
a valid configured root (False when there is no directory at all).
|
||||
"""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
configured = list(GalleryConfig.MONITORING_DIRECTORIES)
|
||||
@@ -706,15 +816,16 @@ class PromptManagerAPI(
|
||||
output_dirs.append(Path(real))
|
||||
|
||||
if output_dirs:
|
||||
return output_dirs
|
||||
return output_dirs, False
|
||||
|
||||
if configured:
|
||||
# Every configured root was rejected: never let auto-detection
|
||||
# resurrect it, serve ComfyUI's own output directory instead.
|
||||
return self._comfyui_output_dir_list()
|
||||
|
||||
fallback = self._find_comfyui_output_dir()
|
||||
return [Path(fallback)] if fallback else []
|
||||
fallback_dirs = self._comfyui_output_dir_list()
|
||||
else:
|
||||
fallback = self._find_comfyui_output_dir()
|
||||
fallback_dirs = [Path(fallback)] if fallback else []
|
||||
return fallback_dirs, bool(fallback_dirs)
|
||||
|
||||
def _comfyui_output_dir_list(self):
|
||||
"""ComfyUI's output directory as a one-element list, or [] if unknown."""
|
||||
|
||||
+543
-406
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,121 @@
|
||||
"""Request validation and the worker of the Tag dialog's Generate (#249 B2).
|
||||
|
||||
``POST /prompt_manager/image-annotations/suggest`` asks one of legacy's four
|
||||
engines for the top N tags of a captured image. Options are validated
|
||||
strictly, per engine: WD14 classifiers take two thresholds, the language
|
||||
models (GGUF and HuggingFace JoyCaption) take the instructions; every engine
|
||||
takes the suggestion count and the GPU switch. Nothing here touches the
|
||||
filesystem or the database.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
WD14_MODELS = frozenset({"wd14-swinv2", "wd14-vit"})
|
||||
LLM_MODELS = frozenset({"gguf", "hf"})
|
||||
MODEL_TYPES = WD14_MODELS | LLM_MODELS
|
||||
|
||||
MAX_INSTRUCTION_CHARS = 4000
|
||||
MAX_INSTRUCTION_BYTES = 16 * 1024
|
||||
|
||||
SUGGEST_FIELDS = frozenset(
|
||||
{"target", "capture_token", "expected_epoch", "model_type", "options"}
|
||||
)
|
||||
_COMMON = frozenset({"suggestion_limit", "use_gpu"})
|
||||
_WD14_REQUIRED = frozenset({"general_threshold", "character_threshold"})
|
||||
_LLM_REQUIRED = frozenset({"instructions"})
|
||||
|
||||
UNAVAILABLE_HINT = "Download it from the legacy Auto-tag page."
|
||||
|
||||
|
||||
class OptionError(ValueError):
|
||||
"""A suggest request whose model type or options are not acceptable."""
|
||||
|
||||
|
||||
def _threshold(name: str, value) -> float:
|
||||
if type(value) not in (int, float) or not math.isfinite(value):
|
||||
raise OptionError(f"{name} must be a finite number")
|
||||
if not 0 <= value <= 1:
|
||||
raise OptionError(f"{name} must be between 0 and 1")
|
||||
return float(value)
|
||||
|
||||
|
||||
def _instructions(value) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise OptionError("instructions must be non-empty text")
|
||||
if len(value) > MAX_INSTRUCTION_CHARS:
|
||||
raise OptionError(
|
||||
f"instructions must be at most {MAX_INSTRUCTION_CHARS} characters"
|
||||
)
|
||||
try:
|
||||
encoded = value.encode("utf-8")
|
||||
except UnicodeEncodeError:
|
||||
raise OptionError("instructions must be valid Unicode text") from None
|
||||
if len(encoded) > MAX_INSTRUCTION_BYTES:
|
||||
raise OptionError("instructions are too long")
|
||||
return value
|
||||
|
||||
|
||||
def validate_options(model_type, options, *, default_limit: int, max_limit: int):
|
||||
"""Normalised ``(model_type, options)`` or :class:`OptionError`.
|
||||
|
||||
The answer echoes every option the request will run with, the defaults
|
||||
(suggestion count from the settings, GPU on) filled in.
|
||||
"""
|
||||
if not isinstance(model_type, str) or model_type not in MODEL_TYPES:
|
||||
raise OptionError("model_type must be one of " + ", ".join(sorted(MODEL_TYPES)))
|
||||
if not isinstance(options, dict):
|
||||
raise OptionError("options must be an object")
|
||||
required = _WD14_REQUIRED if model_type in WD14_MODELS else _LLM_REQUIRED
|
||||
keys = set(options)
|
||||
if not required <= keys or not keys <= required | _COMMON:
|
||||
raise OptionError(
|
||||
f"options for {model_type} take exactly "
|
||||
+ ", ".join(sorted(required))
|
||||
+ " and optionally "
|
||||
+ ", ".join(sorted(_COMMON))
|
||||
)
|
||||
limit = options.get("suggestion_limit", default_limit)
|
||||
if type(limit) is not int or not 1 <= limit <= max_limit:
|
||||
raise OptionError(f"suggestion_limit must be an integer from 1 to {max_limit}")
|
||||
use_gpu = options.get("use_gpu", True)
|
||||
if not isinstance(use_gpu, bool):
|
||||
raise OptionError("use_gpu must be true or false")
|
||||
normalised = {"use_gpu": use_gpu, "suggestion_limit": limit}
|
||||
if model_type in WD14_MODELS:
|
||||
for name in sorted(_WD14_REQUIRED):
|
||||
normalised[name] = _threshold(name, options[name])
|
||||
else:
|
||||
normalised["instructions"] = _instructions(options["instructions"])
|
||||
return model_type, normalised
|
||||
|
||||
|
||||
def unavailable_message(model_name: str) -> str:
|
||||
return f"{model_name} is not downloaded. {UNAVAILABLE_HINT}"
|
||||
|
||||
|
||||
def run_suggestions(service, capture_path: str, model_type: str, options: dict):
|
||||
"""Blocking: load (or swap to) *model_type*, then suggest for the capture.
|
||||
|
||||
The caller holds the model gate for the whole call; nothing is ever
|
||||
downloaded (``load_model`` refuses a model that is not on disk).
|
||||
"""
|
||||
if not service.is_model_loaded() or service.get_loaded_model_type() != model_type:
|
||||
service.load_model(model_type, options["use_gpu"])
|
||||
return service.generate_tag_suggestions(
|
||||
capture_path,
|
||||
limit=options["suggestion_limit"],
|
||||
prompt=options.get("instructions"),
|
||||
general_threshold=options.get("general_threshold"),
|
||||
character_threshold=options.get("character_threshold"),
|
||||
)
|
||||
|
||||
|
||||
def suggestion_rows(pairs) -> list:
|
||||
"""``[{name, confidence}]``; a non-finite score is reported as unknown."""
|
||||
rows = []
|
||||
for name, score in pairs:
|
||||
confidence = None
|
||||
if score is not None and math.isfinite(score):
|
||||
confidence = float(score)
|
||||
rows.append({"name": str(name), "confidence": confidence})
|
||||
return rows
|
||||
@@ -0,0 +1,781 @@
|
||||
"""Image annotation routes: observe, capture preview and byte-exact tag saves.
|
||||
|
||||
Every route names its file through the listing identity ``{root_index,
|
||||
root_scope, path}`` and checks the scope against the current gallery roots
|
||||
before any file access. Observe copies the file into a private capture (the
|
||||
token the preview and Save refer to), Save re-hashes the current file and
|
||||
writes only when the bytes are still the captured ones. Responses carry no
|
||||
server path and no locator.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import functools
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .annotation_suggest import (
|
||||
SUGGEST_FIELDS,
|
||||
OptionError,
|
||||
run_suggestions,
|
||||
suggestion_rows,
|
||||
unavailable_message,
|
||||
validate_options,
|
||||
)
|
||||
from .autotag_routes import model_busy_response
|
||||
from .images import _forbidden, _has_traversal, _unavailable_link, _verified_title
|
||||
from .prompts import bad_request
|
||||
from .. import model_gate
|
||||
|
||||
try:
|
||||
from ...database.annotations import (
|
||||
MAX_IMAGE_TAGS,
|
||||
AnnotationCommandError,
|
||||
ImageTagLimitError,
|
||||
normalize_image_tags,
|
||||
)
|
||||
from ...database.execution_history import subscribe_history_restore
|
||||
from ...database.library_commands import LibraryCommandError
|
||||
from ...utils.media_capture import CAPTURE_EXTENSIONS, CaptureError, CaptureStore
|
||||
except ImportError:
|
||||
from database.annotations import (
|
||||
MAX_IMAGE_TAGS,
|
||||
AnnotationCommandError,
|
||||
ImageTagLimitError,
|
||||
normalize_image_tags,
|
||||
)
|
||||
from database.execution_history import subscribe_history_restore
|
||||
from database.library_commands import LibraryCommandError
|
||||
from utils.media_capture import CAPTURE_EXTENSIONS, CaptureError, CaptureStore
|
||||
|
||||
# Request bodies of the annotation routes are small (a target plus a tag set).
|
||||
MAX_BODY_BYTES = 64 * 1024
|
||||
MAX_TARGET_PATH = 1024
|
||||
CAPTURE_URL = "/prompt_manager/image-annotations/capture/"
|
||||
_PREVIEW_CHUNK = 64 * 1024
|
||||
|
||||
_TARGET_FIELDS = frozenset({"root_index", "root_scope", "path"})
|
||||
_OBSERVE_FIELDS = frozenset({"target"})
|
||||
_SAVE_FIELDS = frozenset(
|
||||
{
|
||||
"target",
|
||||
"capture_token",
|
||||
"sha256",
|
||||
"expected_epoch",
|
||||
"expected_tag_revision",
|
||||
"tags",
|
||||
}
|
||||
)
|
||||
_HEX64 = re.compile(r"[0-9a-f]{64}")
|
||||
_UUID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
|
||||
# secrets.token_urlsafe(32) is 43 characters; the bound only rejects junk.
|
||||
_TOKEN = re.compile(r"[A-Za-z0-9_-]{1,128}")
|
||||
# The store is created on first use by the event loop, the scan's executor or
|
||||
# the monitor's worker (Needs review), so its creation is serialized.
|
||||
_CAPTURE_STORE_LOCK = threading.Lock()
|
||||
|
||||
_PREVIEW_TYPES = {
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".webp": "image/webp",
|
||||
".gif": "image/gif",
|
||||
".bmp": "image/bmp",
|
||||
".tiff": "image/tiff",
|
||||
".tif": "image/tiff",
|
||||
".mp4": "video/mp4",
|
||||
".m4v": "video/mp4",
|
||||
".webm": "video/webm",
|
||||
".mov": "video/quicktime",
|
||||
".mkv": "video/x-matroska",
|
||||
".avi": "video/x-msvideo",
|
||||
".wmv": "video/x-ms-wmv",
|
||||
}
|
||||
|
||||
_STATUS = {
|
||||
"review-capacity": 429,
|
||||
"unsupported-media": 415,
|
||||
"file-changed": 409,
|
||||
"source-changed": 409,
|
||||
"storage-changed": 409,
|
||||
"tag-conflict": 409,
|
||||
"capture-expired": 409,
|
||||
"model-unavailable": 409,
|
||||
"tag-limit": 400,
|
||||
"metadata-conflict": 409,
|
||||
"vocabulary-conflict": 409,
|
||||
}
|
||||
_MESSAGES = {
|
||||
"source-changed": (
|
||||
"The source changed. Refresh the image collection before tagging this file."
|
||||
),
|
||||
"file-changed": "The file changed since it was previewed. Reopen the Tag dialog.",
|
||||
"review-capacity": "Too many previews are open. Close a Tag dialog and try again.",
|
||||
"storage-changed": "Prompt storage was restored; reopen the Tag dialog.",
|
||||
"tag-conflict": (
|
||||
"The tags of this image changed since the dialog opened; review them first."
|
||||
),
|
||||
"capture-expired": "The preview expired. Open the Tag dialog again.",
|
||||
"tag-limit": f"An image can carry at most {MAX_IMAGE_TAGS} tags.",
|
||||
}
|
||||
# Any other storage code (a conflict a later command adds) still answers a
|
||||
# coded envelope rather than a bare 500.
|
||||
_DEFAULT_STATUS = 409
|
||||
_DEFAULT_MESSAGE = "The tags could not be saved. Reopen the Tag dialog and try again."
|
||||
_OTHER_FILE = "The preview belongs to another file. Open the Tag dialog again."
|
||||
|
||||
|
||||
def preview_content_type(extension: str) -> str:
|
||||
"""The media type the preview is served as; never sniffed from bytes."""
|
||||
return _PREVIEW_TYPES.get(extension.lower(), "application/octet-stream")
|
||||
|
||||
|
||||
def _error(status: int, code: str, message: str, **extra) -> web.Response:
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": message, **extra},
|
||||
status=status,
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
|
||||
def _coded(code: str, message: str | None = None, **extra) -> web.Response:
|
||||
status = _STATUS.get(code, _DEFAULT_STATUS)
|
||||
return _error(
|
||||
status, code, message or _MESSAGES.get(code, _DEFAULT_MESSAGE), **extra
|
||||
)
|
||||
|
||||
|
||||
def _open_capture(path: str):
|
||||
"""Worker thread: the capture file opened for reading."""
|
||||
return open(path, "rb")
|
||||
|
||||
|
||||
async def _settled(future, cleanup):
|
||||
"""Await an executor *future*; a cancelled caller still cleans up after it.
|
||||
|
||||
The future is shielded, so the worker call always runs to its end. When the
|
||||
caller is cancelled first, ``cleanup(result)`` runs once the call is done
|
||||
(``cleanup(None)`` when it failed): a handle is closed, a capture released.
|
||||
"""
|
||||
try:
|
||||
return await asyncio.shield(future)
|
||||
except asyncio.CancelledError:
|
||||
future.add_done_callback(functools.partial(_cleanup_result, cleanup))
|
||||
raise
|
||||
|
||||
|
||||
def _cleanup_result(cleanup, future) -> None:
|
||||
if future.cancelled() or future.exception() is not None:
|
||||
cleanup(None)
|
||||
else:
|
||||
cleanup(future.result())
|
||||
|
||||
|
||||
def _submit(func, *args, **kwargs):
|
||||
return asyncio.get_running_loop().run_in_executor(
|
||||
None, functools.partial(func, *args, **kwargs)
|
||||
)
|
||||
|
||||
|
||||
def _capture_error(exc: CaptureError) -> web.Response:
|
||||
"""The envelope for a capture outcome; messages never name a path."""
|
||||
if exc.code not in _STATUS:
|
||||
return _error(500, exc.code, "The file could not be captured.")
|
||||
return _coded(exc.code, _MESSAGES.get(exc.code) or str(exc))
|
||||
|
||||
|
||||
def _target(data: object):
|
||||
"""``((index, scope, relative), None)`` for a target, else ``(None, error)``.
|
||||
|
||||
Only the listing identity is accepted: an int root index, the root's
|
||||
64-hex scope and a bounded root-relative POSIX media path, never an
|
||||
absolute, drive-qualified, backslash or ``..`` path. Nothing here touches
|
||||
the filesystem or the database.
|
||||
"""
|
||||
if not isinstance(data, dict) or set(data) != _TARGET_FIELDS:
|
||||
return None, bad_request("target must name root_index, root_scope and path")
|
||||
index, scope, relative = data["root_index"], data["root_scope"], data["path"]
|
||||
if type(index) is not int or index < 0:
|
||||
return None, bad_request("Invalid gallery root")
|
||||
if not isinstance(scope, str) or _HEX64.fullmatch(scope) is None:
|
||||
return None, bad_request("Invalid gallery root scope")
|
||||
if not isinstance(relative, str) or not relative or len(relative) > MAX_TARGET_PATH:
|
||||
return None, bad_request("path must be a relative media path")
|
||||
parts = relative.split("/")
|
||||
if (
|
||||
"\0" in relative
|
||||
or "\\" in relative
|
||||
or relative.startswith("/")
|
||||
or _has_traversal(relative)
|
||||
or any(part in ("", ".", "..") or ":" in part for part in parts)
|
||||
):
|
||||
return None, _forbidden()
|
||||
if os.path.splitext(parts[-1])[1].lower() not in CAPTURE_EXTENSIONS:
|
||||
return None, _forbidden("Only media files can be tagged")
|
||||
return (index, scope, relative), None
|
||||
|
||||
|
||||
def _expires_at(seconds: float | None) -> str:
|
||||
moment = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
|
||||
seconds=max(0.0, seconds or 0.0)
|
||||
)
|
||||
return moment.replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _capture_info(store: CaptureStore, capture) -> dict:
|
||||
return {
|
||||
"token": capture.token,
|
||||
"expires_at": _expires_at(store.expires_in(capture.token)),
|
||||
"preview_url": CAPTURE_URL + capture.token,
|
||||
"sha256": capture.sha256,
|
||||
"size": capture.size,
|
||||
"width": capture.width,
|
||||
"height": capture.height,
|
||||
"media_type": capture.media_type,
|
||||
"generate": {
|
||||
"supported": capture.inferable,
|
||||
"reason": capture.inferable_reason,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class AnnotationRoutesMixin:
|
||||
"""Observe, capture preview and tag save routes over PR A's storage."""
|
||||
|
||||
def _register_annotation_routes(self, routes: web.RouteTableDef) -> None:
|
||||
routes.post("/prompt_manager/image-annotations/observe")(self.observe_image)
|
||||
routes.get(CAPTURE_URL + "{token}")(self.get_capture_preview)
|
||||
routes.delete(CAPTURE_URL + "{token}")(self.release_capture)
|
||||
routes.put("/prompt_manager/image-annotations/tags")(self.save_image_tags)
|
||||
routes.post("/prompt_manager/image-annotations/suggest")(
|
||||
self.suggest_image_tags
|
||||
)
|
||||
|
||||
# -- capture store lifetime ------------------------------------------------
|
||||
|
||||
def _capture_store(self) -> CaptureStore:
|
||||
"""The process's capture store, created on first use.
|
||||
|
||||
Created lazily so an API object that never observes a file (most tests,
|
||||
a server that never opens the Tag dialog) owns no temp directory. A
|
||||
restore of the database dooms every capture (``invalidate_all``).
|
||||
Thread-safe: Needs review recording calls it off the event loop.
|
||||
Raises ``CaptureError`` after ``close_annotation_captures``.
|
||||
"""
|
||||
with _CAPTURE_STORE_LOCK:
|
||||
if getattr(self, "_captures_closed", False):
|
||||
# A recording still in flight at shutdown must not create a
|
||||
# new temp directory after the cleanup removed the old one.
|
||||
raise CaptureError("The capture store is closed")
|
||||
if self._captures is None:
|
||||
self._captures = CaptureStore()
|
||||
if self._captures_unsubscribe is None:
|
||||
self._captures_unsubscribe = subscribe_history_restore(
|
||||
self.db.model.db_path, self._invalidate_captures
|
||||
)
|
||||
return self._captures
|
||||
|
||||
def _invalidate_captures(self) -> None:
|
||||
store = self._captures
|
||||
if store is not None:
|
||||
store.invalidate_all()
|
||||
|
||||
def close_annotation_captures(self, *, final: bool = False) -> None:
|
||||
"""Remove every capture file and the owned directory. Idempotent.
|
||||
|
||||
Taken under the store lock. ``final`` (the shutdown cleanup) also makes
|
||||
``_capture_store`` refuse to create another store, so a recording
|
||||
still in flight cannot leak a new temp directory after it.
|
||||
"""
|
||||
with _CAPTURE_STORE_LOCK:
|
||||
if final:
|
||||
self._captures_closed = True
|
||||
unsubscribe, self._captures_unsubscribe = self._captures_unsubscribe, None
|
||||
store, self._captures = self._captures, None
|
||||
if unsubscribe is not None:
|
||||
unsubscribe()
|
||||
if store is not None:
|
||||
store.close()
|
||||
|
||||
async def cleanup_annotation_captures(self, app: web.Application) -> None:
|
||||
self.close_annotation_captures(final=True)
|
||||
|
||||
# -- shared request steps --------------------------------------------------
|
||||
|
||||
async def _annotation_body(self, request: web.Request, fields: frozenset):
|
||||
data, error = await self._read_json_body(request, max_bytes=MAX_BODY_BYTES)
|
||||
if error is not None:
|
||||
return None, error
|
||||
if set(data) != fields:
|
||||
return None, bad_request(
|
||||
"Request must contain exactly " + ", ".join(sorted(fields))
|
||||
)
|
||||
return data, None
|
||||
|
||||
def _scoped_root(self, index: int, scope: str):
|
||||
"""``(root, None)`` when the scope names a current root, else an error."""
|
||||
roots = list(self._get_all_output_dirs())
|
||||
error = self._gallery_scope_error(
|
||||
{"root_scope": scope, "root": str(index)}, roots
|
||||
)
|
||||
if error is not None:
|
||||
if error.status == 409:
|
||||
return None, _coded("source-changed")
|
||||
return None, error
|
||||
return roots[index], None
|
||||
|
||||
# -- observe ---------------------------------------------------------------
|
||||
|
||||
async def observe_image(self, request: web.Request) -> web.Response:
|
||||
"""POST ``{target}``: capture the file and answer its saved tag set.
|
||||
|
||||
The capture is the byte-exact copy the preview and Save refer to; the
|
||||
tags are those of exactly these bytes (an empty set at revision 0 for
|
||||
unseen bytes). Observing never writes tags; equal bytes refresh the
|
||||
record's verified fingerprint (S1).
|
||||
"""
|
||||
data, error = await self._annotation_body(request, _OBSERVE_FIELDS)
|
||||
if error is not None:
|
||||
return error
|
||||
target, error = _target(data["target"])
|
||||
if error is not None:
|
||||
return error
|
||||
index, scope, relative = target
|
||||
root, error = self._scoped_root(index, scope)
|
||||
if error is not None:
|
||||
return error
|
||||
try:
|
||||
store = self._capture_store()
|
||||
except CaptureError as exc: # the server is shutting down
|
||||
return _capture_error(exc)
|
||||
try:
|
||||
epoch = await self._run_in_executor(self.db.get_execution_epoch)
|
||||
capture = await _settled(
|
||||
_submit(
|
||||
store.capture,
|
||||
root,
|
||||
relative,
|
||||
epoch=epoch,
|
||||
scope=scope,
|
||||
root_index=index,
|
||||
),
|
||||
# A cancelled observe never hands its capture out: release it.
|
||||
lambda made: made is not None and store.release(made.token),
|
||||
)
|
||||
except CaptureError as exc:
|
||||
return _capture_error(exc)
|
||||
except Exception:
|
||||
self.logger.exception("Image observation failed")
|
||||
return _error(500, "observe-failed", "The image could not be observed.")
|
||||
try:
|
||||
stored, title = await self._run_in_executor(
|
||||
self._observe_capture, store, capture, root, relative
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
store.release(capture.token)
|
||||
raise
|
||||
except Exception:
|
||||
store.release(capture.token)
|
||||
self.logger.exception("Image observation failed")
|
||||
return _error(500, "observe-failed", "The image could not be observed.")
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"library_epoch": capture.epoch,
|
||||
"target": {"root_index": index, "root_scope": scope, "path": relative},
|
||||
"capture": _capture_info(store, capture),
|
||||
"title": title,
|
||||
"filename": relative.rsplit("/", 1)[-1],
|
||||
"tags": stored["tags"],
|
||||
"tag_revision": stored["tag_revision"],
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
def _observe_capture(self, store: CaptureStore, capture, root, relative: str):
|
||||
"""Blocking part of observe: fingerprint refresh, tag read, title."""
|
||||
self.db.refresh_image_fingerprint(
|
||||
capture.locator, capture.sha256, capture.size, capture.mtime_ns
|
||||
)
|
||||
stored = self.db.read_image_tags(capture.locator, capture.sha256)
|
||||
return stored, self._observed_title(store, capture, root, relative)
|
||||
|
||||
def _observed_title(self, store: CaptureStore, capture, root, relative: str):
|
||||
"""The dialog's title under the card and inspector rule (``_verified_title``).
|
||||
|
||||
A stored Library title always shows; a first-line title only when the
|
||||
captured bytes' own records confirm it. An ambiguous association has no
|
||||
title. Videos read no card metadata, so only a stored title shows.
|
||||
"""
|
||||
path = str(Path(root) / relative)
|
||||
summary = self._workspace_link_summaries([path]).get(path)
|
||||
if summary is None:
|
||||
summary = _unavailable_link()
|
||||
positive_title = None
|
||||
if (
|
||||
capture.media_type != "video"
|
||||
and summary.get("title_source") == "text"
|
||||
and store.acquire(capture.token) is not None
|
||||
):
|
||||
try:
|
||||
facts = self._read_card_facts(Path(capture.path), parse=True)
|
||||
finally:
|
||||
store.release_use(capture.token)
|
||||
positive_title = facts.positive_title if facts is not None else None
|
||||
return _verified_title(summary, positive_title)
|
||||
|
||||
# -- capture preview -------------------------------------------------------
|
||||
|
||||
async def get_capture_preview(self, request: web.Request) -> web.StreamResponse:
|
||||
"""GET the captured bytes; the token is pinned while they stream."""
|
||||
token = request.match_info["token"]
|
||||
store = self._captures
|
||||
if store is None or _TOKEN.fullmatch(token) is None:
|
||||
return _error(404, "capture-expired", _MESSAGES["capture-expired"])
|
||||
capture = store.acquire(token)
|
||||
if capture is None:
|
||||
return _error(404, "capture-expired", _MESSAGES["capture-expired"])
|
||||
pinned = True
|
||||
try:
|
||||
_root, error = self._scoped_root(capture.root_index, capture.scope)
|
||||
if error is not None:
|
||||
# The roots changed under the token: it is no longer usable.
|
||||
store.release(token)
|
||||
return error
|
||||
try:
|
||||
handle = await _settled(
|
||||
_submit(_open_capture, capture.path),
|
||||
functools.partial(_close_then_release, store, token),
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
# Cancelled while the file opens: the pin ends only after the
|
||||
# handle closes (Windows cannot unlink an open file).
|
||||
pinned = False
|
||||
raise
|
||||
except OSError:
|
||||
store.release(token)
|
||||
return _error(404, "capture-expired", _MESSAGES["capture-expired"])
|
||||
try:
|
||||
return await self._stream_capture(request, capture, handle)
|
||||
finally:
|
||||
handle.close()
|
||||
finally:
|
||||
if pinned:
|
||||
store.release_use(token)
|
||||
|
||||
@staticmethod
|
||||
async def _stream_capture(request, capture, handle) -> web.StreamResponse:
|
||||
"""The capture's bytes, or the one byte range a ``Range`` header asks for.
|
||||
|
||||
The response is written here, inside the caller's pin, so the handle
|
||||
is closed and the pin ended only after the last byte is sent.
|
||||
"""
|
||||
size = capture.size
|
||||
headers = {
|
||||
"Content-Type": preview_content_type(Path(capture.path).suffix),
|
||||
"Accept-Ranges": "bytes",
|
||||
"Cache-Control": "no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
}
|
||||
status, start, end = 200, 0, size
|
||||
if "Range" in request.headers:
|
||||
wanted = _byte_range(request, size)
|
||||
if wanted is None:
|
||||
return web.Response(
|
||||
status=416,
|
||||
headers={**headers, "Content-Range": f"bytes */{size}"},
|
||||
)
|
||||
status, (start, end) = 206, wanted
|
||||
headers["Content-Range"] = f"bytes {start}-{end - 1}/{size}"
|
||||
headers["Content-Length"] = str(end - start)
|
||||
response = web.StreamResponse(status=status, headers=headers)
|
||||
await response.prepare(request)
|
||||
loop = asyncio.get_running_loop()
|
||||
if start:
|
||||
await loop.run_in_executor(None, handle.seek, start)
|
||||
remaining = end - start
|
||||
while remaining > 0 and request.method != "HEAD":
|
||||
chunk = await loop.run_in_executor(
|
||||
None, handle.read, min(_PREVIEW_CHUNK, remaining)
|
||||
)
|
||||
if not chunk:
|
||||
break
|
||||
remaining -= len(chunk)
|
||||
await response.write(chunk)
|
||||
await response.write_eof()
|
||||
return response
|
||||
|
||||
async def release_capture(self, request: web.Request) -> web.Response:
|
||||
"""DELETE a capture token; releasing an unknown token is a success."""
|
||||
token = request.match_info["token"]
|
||||
store = self._captures
|
||||
if store is not None and _TOKEN.fullmatch(token) is not None:
|
||||
store.release(token)
|
||||
return web.json_response(
|
||||
{"success": True}, headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
|
||||
# -- save ------------------------------------------------------------------
|
||||
|
||||
async def save_image_tags(self, request: web.Request) -> web.Response:
|
||||
"""PUT the tag set of the previewed bytes, re-hashing the current file.
|
||||
|
||||
Writes only when the file still holds the captured bytes
|
||||
(``file-changed`` with zero writes otherwise), under the storage epoch
|
||||
and the tag revision the dialog read; an identical retry is
|
||||
``replayed``. The hash never runs inside a database transaction.
|
||||
"""
|
||||
data, error = await self._annotation_body(request, _SAVE_FIELDS)
|
||||
if error is not None:
|
||||
return error
|
||||
target, error = _target(data["target"])
|
||||
if error is not None:
|
||||
return error
|
||||
guards, error = _save_guards(data)
|
||||
if error is not None:
|
||||
return error
|
||||
token, sha256, expected_epoch, revision, tags = guards
|
||||
index, scope, relative = target
|
||||
root, error = self._scoped_root(index, scope)
|
||||
if error is not None:
|
||||
return error
|
||||
store = self._captures
|
||||
capture = store.get(token) if store is not None else None
|
||||
if capture is None:
|
||||
return _coded("capture-expired")
|
||||
if capture.root_index != index or capture.scope != scope:
|
||||
return _coded("capture-expired", _OTHER_FILE)
|
||||
if sha256 != capture.sha256:
|
||||
return bad_request("sha256 does not match the preview")
|
||||
if expected_epoch != capture.epoch:
|
||||
current = await self._run_in_executor(self.db.get_execution_epoch)
|
||||
return _coded("storage-changed", library_epoch=current)
|
||||
try:
|
||||
facts = await self._run_in_executor(
|
||||
_verify_captured_file, store, capture, root, relative
|
||||
)
|
||||
except CaptureError as exc:
|
||||
return _capture_error(exc)
|
||||
if facts is None or facts.locator != capture.locator:
|
||||
return _coded("capture-expired", _OTHER_FILE)
|
||||
if facts.sha256 != capture.sha256:
|
||||
return _coded("file-changed")
|
||||
try:
|
||||
receipt = await self._run_in_executor(
|
||||
self.db.replace_image_tags,
|
||||
locator=facts.locator,
|
||||
sha256=facts.sha256,
|
||||
size=facts.size,
|
||||
mtime_ns=facts.mtime_ns,
|
||||
expected_epoch=expected_epoch,
|
||||
expected_tag_revision=revision,
|
||||
tags=tags,
|
||||
)
|
||||
except AnnotationCommandError as exc:
|
||||
current = exc.current or {}
|
||||
return _coded(
|
||||
exc.code,
|
||||
library_epoch=exc.epoch,
|
||||
tags=current.get("tags", []),
|
||||
tag_revision=current.get("tag_revision", 0),
|
||||
)
|
||||
except LibraryCommandError as exc:
|
||||
return _coded(exc.code, library_epoch=exc.epoch)
|
||||
except ValueError as exc:
|
||||
return bad_request(str(exc))
|
||||
except Exception:
|
||||
self.logger.exception("Image tag save failed")
|
||||
return _error(500, "save-failed", "The tags could not be saved.")
|
||||
return web.json_response(
|
||||
{"success": True, **receipt}, headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
|
||||
# -- suggest ---------------------------------------------------------------
|
||||
|
||||
async def suggest_image_tags(self, request: web.Request) -> web.Response:
|
||||
"""POST: the top N tags of the captured bytes from one legacy engine.
|
||||
|
||||
Runs only on the private capture (pinned for the whole inference),
|
||||
under the shared model gate (``model-busy`` while anything else holds
|
||||
it, including a legacy batch), never downloads (``model-unavailable``)
|
||||
and never writes.
|
||||
"""
|
||||
from ..autotag import MODELS, get_autotag_service
|
||||
from ..config import MAX_SUGGESTION_LIMIT, AutotagConfig
|
||||
|
||||
data, error = await self._annotation_body(request, SUGGEST_FIELDS)
|
||||
if error is not None:
|
||||
return error
|
||||
target, error = _target(data["target"])
|
||||
if error is not None:
|
||||
return error
|
||||
try:
|
||||
model_type, options = validate_options(
|
||||
data["model_type"],
|
||||
data["options"],
|
||||
default_limit=AutotagConfig.suggestion_limit(),
|
||||
max_limit=MAX_SUGGESTION_LIMIT,
|
||||
)
|
||||
except OptionError as exc:
|
||||
return bad_request(str(exc))
|
||||
token, epoch = data["capture_token"], data["expected_epoch"]
|
||||
if not isinstance(token, str) or _TOKEN.fullmatch(token) is None:
|
||||
return bad_request("Invalid capture token")
|
||||
if not isinstance(epoch, str) or _UUID.fullmatch(epoch) is None:
|
||||
return bad_request("expected_epoch must be a canonical UUID")
|
||||
index, scope, relative = target
|
||||
root, error = self._scoped_root(index, scope)
|
||||
if error is not None:
|
||||
return error
|
||||
store = self._captures
|
||||
capture = store.acquire(token) if store is not None else None
|
||||
if capture is None:
|
||||
return _coded("capture-expired")
|
||||
started = False
|
||||
try:
|
||||
error = await self._suggest_guards(capture, target, root, epoch)
|
||||
if error is not None:
|
||||
return error
|
||||
service = get_autotag_service()
|
||||
status = await self._run_in_executor(service.get_models_status)
|
||||
if not status.get(model_type, {}).get("downloaded"):
|
||||
name = MODELS.get(model_type, {}).get("name", model_type)
|
||||
return _coded("model-unavailable", unavailable_message(name))
|
||||
owner = model_gate.new_owner("image-annotations")
|
||||
if not model_gate.try_acquire(owner):
|
||||
return model_busy_response()
|
||||
try:
|
||||
# From here the pin is the gate's to end: the worker ends it,
|
||||
# or ``on_skipped`` does when the worker never starts.
|
||||
started = True
|
||||
pairs = await model_gate.run(
|
||||
owner,
|
||||
_pinned_suggestions,
|
||||
store,
|
||||
token,
|
||||
service,
|
||||
capture,
|
||||
model_type,
|
||||
options,
|
||||
on_skipped=functools.partial(store.release_use, token),
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Tag suggestion failed")
|
||||
return _error(
|
||||
500,
|
||||
"suggest-failed",
|
||||
"Auto-tagging failed. Check server logs for details.",
|
||||
)
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
finally:
|
||||
if not started:
|
||||
store.release_use(token)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"model_type": model_type,
|
||||
"options": options,
|
||||
"suggestions": suggestion_rows(pairs),
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
async def _suggest_guards(self, capture, target, root, epoch):
|
||||
"""The capture must be this target's, of this epoch, and inferable."""
|
||||
index, scope, relative = target
|
||||
expected = os.path.normcase(
|
||||
os.path.join(os.path.realpath(root), *relative.split("/"))
|
||||
)
|
||||
if (
|
||||
capture.root_index != index
|
||||
or capture.scope != scope
|
||||
or capture.locator != expected
|
||||
):
|
||||
return _coded("capture-expired", _OTHER_FILE)
|
||||
if epoch != capture.epoch:
|
||||
current = await self._run_in_executor(self.db.get_execution_epoch)
|
||||
return _coded("storage-changed", library_epoch=current)
|
||||
if not capture.inferable:
|
||||
return _coded("unsupported-media", capture.inferable_reason)
|
||||
return None
|
||||
|
||||
|
||||
def _close_then_release(store: CaptureStore, token: str, handle) -> None:
|
||||
"""Close a preview handle a cancelled request opened, then end its pin."""
|
||||
try:
|
||||
if handle is not None:
|
||||
handle.close()
|
||||
finally:
|
||||
store.release_use(token)
|
||||
|
||||
|
||||
def _byte_range(request: web.Request, size: int):
|
||||
"""``(start, end)`` of the one satisfiable byte range asked for, else None.
|
||||
|
||||
A malformed, multi-range or unsatisfiable header is None (416), as
|
||||
aiohttp's own file responses answer it.
|
||||
"""
|
||||
try:
|
||||
wanted = request.http_range
|
||||
except ValueError:
|
||||
return None
|
||||
start, stop = wanted.start, wanted.stop
|
||||
if start is None:
|
||||
return None
|
||||
if start < 0: # a suffix: the last -start bytes
|
||||
start = max(0, size + start)
|
||||
stop = size
|
||||
stop = size if stop is None else min(stop, size)
|
||||
if start >= size or stop <= start:
|
||||
return None
|
||||
return start, stop
|
||||
|
||||
|
||||
def _pinned_suggestions(store, token, service, capture, model_type, options):
|
||||
"""Worker thread: infer on the capture, then end its pin (even on error)."""
|
||||
try:
|
||||
return run_suggestions(service, capture.path, model_type, options)
|
||||
finally:
|
||||
store.release_use(token)
|
||||
|
||||
|
||||
def _verify_captured_file(store: CaptureStore, capture, root, relative: str):
|
||||
"""Re-hash the current file, or None when the token names another file.
|
||||
|
||||
The cheap locator comparison runs first, so a token sent for another
|
||||
target never costs a whole-file hash.
|
||||
"""
|
||||
expected = os.path.normcase(
|
||||
os.path.join(os.path.realpath(root), *relative.split("/"))
|
||||
)
|
||||
if expected != capture.locator:
|
||||
return None
|
||||
return store.verify(root, relative)
|
||||
|
||||
|
||||
def _save_guards(data: dict):
|
||||
"""Validated ``(token, sha256, epoch, revision, tags)`` or ``(None, error)``."""
|
||||
token = data["capture_token"]
|
||||
if not isinstance(token, str) or _TOKEN.fullmatch(token) is None:
|
||||
return None, bad_request("Invalid capture token")
|
||||
sha256 = data["sha256"]
|
||||
if not isinstance(sha256, str) or _HEX64.fullmatch(sha256) is None:
|
||||
return None, bad_request("sha256 must be 64 lowercase hexadecimal characters")
|
||||
epoch = data["expected_epoch"]
|
||||
if not isinstance(epoch, str) or _UUID.fullmatch(epoch) is None:
|
||||
return None, bad_request("expected_epoch must be a canonical UUID")
|
||||
revision = data["expected_tag_revision"]
|
||||
if type(revision) is not int or revision < 0:
|
||||
return None, bad_request("expected_tag_revision must be a non-negative integer")
|
||||
try:
|
||||
tags = normalize_image_tags(data["tags"])
|
||||
except ImageTagLimitError as exc:
|
||||
return None, _error(400, "tag-limit", str(exc))
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
return (token, sha256, epoch, revision, tags), None
|
||||
+111
-27
@@ -8,12 +8,26 @@ from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.parse import quote
|
||||
|
||||
from .jobs import AUTOTAG_BATCH_JOB_KIND
|
||||
from .prompts import thumbnail_url_for
|
||||
from .. import model_gate
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
_INTERNAL_ERROR = "An internal error occurred. Check server logs for details."
|
||||
_BUSY_ERROR = "Auto-tagging is already running"
|
||||
MODEL_BUSY_ERROR = (
|
||||
"A tagging model is busy with another operation. Try again when it finishes."
|
||||
)
|
||||
|
||||
|
||||
def model_busy_response() -> web.Response:
|
||||
"""``409 model-busy``: another operation holds the shared model gate."""
|
||||
return web.json_response(
|
||||
{"success": False, "code": "model-busy", "error": MODEL_BUSY_ERROR},
|
||||
status=409,
|
||||
)
|
||||
|
||||
|
||||
# Only these may be handed to the tagging engine or listed by the scanner.
|
||||
_MEDIA_EXTENSIONS = frozenset(
|
||||
@@ -50,6 +64,12 @@ def path_is_within(candidate, root) -> bool:
|
||||
return child != parent and child.is_relative_to(parent)
|
||||
|
||||
|
||||
async def _refusal(title):
|
||||
"""The single error event of an auto-tag batch refused by another job."""
|
||||
message = f"{title} already running"
|
||||
yield {"type": "error", "status": "error", "message": message}
|
||||
|
||||
|
||||
class AutotagRoutesMixin:
|
||||
"""Mixin providing auto-tagging API endpoints."""
|
||||
|
||||
@@ -86,6 +106,7 @@ class AutotagRoutesMixin:
|
||||
"""Get status of available AutoTag models."""
|
||||
try:
|
||||
from ..autotag import get_autotag_service
|
||||
from ..config import AutotagConfig
|
||||
|
||||
service = get_autotag_service()
|
||||
models_status = service.get_models_status()
|
||||
@@ -99,6 +120,7 @@ class AutotagRoutesMixin:
|
||||
"loaded_model_type": service.get_loaded_model_type(),
|
||||
"wd14_general_threshold": service.wd14_general_threshold,
|
||||
"wd14_character_threshold": service.wd14_character_threshold,
|
||||
"suggestion_limit": AutotagConfig.suggestion_limit(),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -138,6 +160,7 @@ class AutotagRoutesMixin:
|
||||
async def download_autotag_model(self, request):
|
||||
"""Download an AutoTag model with streaming progress (POST only)."""
|
||||
model_type = request.match_info.get("model_type")
|
||||
owner = model_gate.new_owner("legacy-download")
|
||||
|
||||
async def events():
|
||||
try:
|
||||
@@ -160,8 +183,8 @@ class AutotagRoutesMixin:
|
||||
def progress_callback(status: str, progress: float):
|
||||
pass
|
||||
|
||||
success = await self._run_in_executor(
|
||||
service.download_model, model_type, progress_callback
|
||||
success = await model_gate.run(
|
||||
owner, service.download_model, model_type, progress_callback
|
||||
)
|
||||
if success:
|
||||
yield {
|
||||
@@ -178,7 +201,12 @@ class AutotagRoutesMixin:
|
||||
"message": _INTERNAL_ERROR,
|
||||
}
|
||||
|
||||
return await self._stream_sse(request, events())
|
||||
if not model_gate.try_acquire(owner):
|
||||
return model_busy_response()
|
||||
try:
|
||||
return await self._stream_sse(request, events())
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
|
||||
async def _read_start_params(self, request) -> Optional[dict]:
|
||||
"""Merge query-string and optional JSON-body parameters for autotag/start.
|
||||
@@ -239,8 +267,9 @@ class AutotagRoutesMixin:
|
||||
return
|
||||
|
||||
yield {"type": "progress", "progress": 0, "status": "Loading model..."}
|
||||
run = self._gated_runner(opts.get("gate_owner"))
|
||||
try:
|
||||
await self._run_in_executor(service.load_model, model_type, True)
|
||||
await run(service.load_model, model_type, True)
|
||||
except Exception:
|
||||
self.logger.exception("Failed to load model")
|
||||
yield {
|
||||
@@ -338,7 +367,7 @@ class AutotagRoutesMixin:
|
||||
continue
|
||||
|
||||
try:
|
||||
tags = await self._run_in_executor(
|
||||
tags = await run(
|
||||
service.generate_tags,
|
||||
allowed_path,
|
||||
general_threshold=opts["general_threshold"],
|
||||
@@ -403,6 +432,8 @@ class AutotagRoutesMixin:
|
||||
"general_threshold and character_threshold must be numeric"
|
||||
)
|
||||
|
||||
errors = [] # an error event ends the run; Activity lists it failed
|
||||
|
||||
async def events():
|
||||
try:
|
||||
from ..autotag import get_autotag_service
|
||||
@@ -410,23 +441,65 @@ class AutotagRoutesMixin:
|
||||
async for event in self._autotag_batch_events(
|
||||
get_autotag_service(), opts
|
||||
):
|
||||
if event.get("type") == "error":
|
||||
errors.append(event)
|
||||
yield event
|
||||
except Exception:
|
||||
self.logger.exception("AutoTag error")
|
||||
errors.append(_INTERNAL_ERROR)
|
||||
yield {
|
||||
"type": "error",
|
||||
"message": _INTERNAL_ERROR,
|
||||
}
|
||||
|
||||
# The batch writes prompt tags: a records hold on the registry, so it
|
||||
# never runs beside a scan, restore or the workspace tagging job.
|
||||
busy = self._jobs.blocker(AUTOTAG_BATCH_JOB_KIND, admission="records")
|
||||
if busy is not None:
|
||||
if busy.kind == AUTOTAG_BATCH_JOB_KIND: # unchanged payload
|
||||
return web.json_response(
|
||||
{"success": False, "error": _BUSY_ERROR}, status=409
|
||||
)
|
||||
# Both legacy pages read this route as a stream, where a 409 reads
|
||||
# as a lost connection: refuse with one error frame instead (the
|
||||
# admin page keys on ``type``, the gallery on ``status``).
|
||||
return await self._stream_sse(request, _refusal(busy.title))
|
||||
async with self._jobs.hold(
|
||||
AUTOTAG_BATCH_JOB_KIND,
|
||||
"Auto-tag images (legacy page)",
|
||||
admission="records",
|
||||
) as job:
|
||||
response = await self._start_autotag_batch(request, opts, events)
|
||||
if response.status >= 400 or errors:
|
||||
job.finish("failed")
|
||||
return response
|
||||
|
||||
async def _start_autotag_batch(self, request, opts, events):
|
||||
if not _autotag_batch_lock.acquire(blocking=False):
|
||||
return web.json_response(
|
||||
{"success": False, "error": _BUSY_ERROR}, status=409
|
||||
)
|
||||
try:
|
||||
return await self._stream_sse(request, events())
|
||||
opts["gate_owner"] = model_gate.new_owner("legacy-batch")
|
||||
if not model_gate.try_acquire(opts["gate_owner"]):
|
||||
return model_busy_response()
|
||||
try:
|
||||
return await self._stream_sse(request, events())
|
||||
finally:
|
||||
model_gate.release(opts["gate_owner"])
|
||||
finally:
|
||||
_autotag_batch_lock.release()
|
||||
|
||||
def _gated_runner(self, owner):
|
||||
"""Executor calls under the model gate *owner* holds (plain when None)."""
|
||||
if owner is None:
|
||||
return self._run_in_executor
|
||||
|
||||
async def run(func, *args, **kwargs):
|
||||
return await model_gate.run(owner, func, *args, **kwargs)
|
||||
|
||||
return run
|
||||
|
||||
def _resolve_allowed_image_path(self, raw_path: str) -> Optional[str]:
|
||||
"""Return the real path when it is a media file inside an output directory."""
|
||||
if Path(str(raw_path)).suffix.lower() not in _MEDIA_EXTENSIONS:
|
||||
@@ -549,22 +622,28 @@ class AutotagRoutesMixin:
|
||||
from ..autotag import get_autotag_service
|
||||
|
||||
service = get_autotag_service()
|
||||
owner = model_gate.new_owner("legacy-single")
|
||||
if not model_gate.try_acquire(owner):
|
||||
return model_busy_response()
|
||||
try:
|
||||
if (
|
||||
not service.is_model_loaded()
|
||||
or service.get_loaded_model_type() != model_type
|
||||
):
|
||||
await model_gate.run(owner, service.load_model, model_type, use_gpu)
|
||||
|
||||
if (
|
||||
not service.is_model_loaded()
|
||||
or service.get_loaded_model_type() != model_type
|
||||
):
|
||||
await self._run_in_executor(service.load_model, model_type, use_gpu)
|
||||
if custom_prompt:
|
||||
service.custom_prompt = custom_prompt
|
||||
|
||||
if custom_prompt:
|
||||
service.custom_prompt = custom_prompt
|
||||
|
||||
tags = await self._run_in_executor(
|
||||
service.generate_tags,
|
||||
image_path,
|
||||
general_threshold=general_threshold,
|
||||
character_threshold=character_threshold,
|
||||
)
|
||||
tags = await model_gate.run(
|
||||
owner,
|
||||
service.generate_tags,
|
||||
image_path,
|
||||
general_threshold=general_threshold,
|
||||
character_threshold=character_threshold,
|
||||
)
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
@@ -636,14 +715,19 @@ class AutotagRoutesMixin:
|
||||
from ..autotag import get_autotag_service
|
||||
|
||||
service = get_autotag_service()
|
||||
owner = model_gate.new_owner("legacy-unload")
|
||||
if not model_gate.try_acquire(owner):
|
||||
return model_busy_response()
|
||||
try:
|
||||
if not service.is_model_loaded():
|
||||
return web.json_response(
|
||||
{"success": True, "message": "No model was loaded"}
|
||||
)
|
||||
|
||||
if not service.is_model_loaded():
|
||||
return web.json_response(
|
||||
{"success": True, "message": "No model was loaded"}
|
||||
)
|
||||
|
||||
model_type = service.get_loaded_model_type()
|
||||
service.unload_model()
|
||||
model_type = service.get_loaded_model_type()
|
||||
service.unload_model()
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
|
||||
@@ -0,0 +1,779 @@
|
||||
"""Duplicate image files on the job registry (#249 System S6, Task 17).
|
||||
|
||||
- ``POST /prompt_manager/jobs/duplicates`` starts the find (202 ``{job}``):
|
||||
kind ``duplicates``, "Find duplicate image files", admission ``read``,
|
||||
cancellable, outcome group ``groups_found``. It hashes the media files of
|
||||
every effective source (``_hash_media_groups``, shared with the legacy
|
||||
scan) and the server keeps the groups it found under the job id: content
|
||||
hash, root-relative paths and each file's source index. A find is refused
|
||||
while a duplicate removal runs, and one find runs at a time.
|
||||
- ``GET /prompt_manager/maintenance/duplicates/{job_id}?offset&limit`` pages
|
||||
those groups, at most ``MAX_GROUP_PAGE`` at a time: ``{success, job:{id,
|
||||
state}, total, offset, limit, groups:[{group_id, size, count, files}]}``.
|
||||
Each file names its root-relative ``path``, its source's public label and
|
||||
index and the Images serve URLs; never an absolute path.
|
||||
- ``POST /prompt_manager/maintenance/preview {operation:"duplicate_files",
|
||||
options:{job_id, keep:[{group_id, root_index, path}]}}`` reviews a removal:
|
||||
in each group named, the files kept are the ones listed and every other
|
||||
copy is removed. Each keeper and each removal is checked against the held
|
||||
group, with containment decided as text before any filesystem access
|
||||
(``file_problem``): inside its source, a media file outside
|
||||
``thumbnails``, then a regular file (not a link) whose resolved path is
|
||||
still inside the source, with the same size and content hash. A removal
|
||||
that fails, or whose kept copy fails, is protected and listed with the
|
||||
reason. The plan carries the files' database links, which move to the
|
||||
kept copy.
|
||||
- ``POST /prompt_manager/maintenance/apply {token}`` runs the reviewed plan
|
||||
as the ``duplicate_cleanup`` job: admission ``records``, not cancellable,
|
||||
groups ``removed``, ``kept`` and ``errors``. It checks the sources are
|
||||
unchanged, takes the safety backup the policy promises (the database only;
|
||||
removed files are not in it), then for each removal rechecks its kept copy
|
||||
and the file itself (``file_problem``) and removes it with every handle
|
||||
closed (``_unlink``), together with its thumbnail. A file that changed is
|
||||
kept; the first unlink that fails stops the job, a locked file with the
|
||||
coded ``LOCKED_FILE`` message (never the OS text), and the files after it
|
||||
are not attempted. The database links of the files actually removed then
|
||||
move to their kept copy (``PromptDatabase.move_file_links``).
|
||||
|
||||
The legacy gallery routes (``/scan_duplicates``, its stream and status, and
|
||||
``/delete_duplicate_images``) keep their shapes on the same registry kinds;
|
||||
see ``py/api/admin.py``.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .admin import DUPLICATE_FIND_LEGACY_TITLE, _remove_thumbnail
|
||||
from .duplicate_review import ( # noqa: F401 (CHANGED, OUTSIDE_SOURCE: public)
|
||||
CHANGED,
|
||||
DELETE_FAILED,
|
||||
FIND_CANCELLED,
|
||||
FIND_ERROR_EVENT,
|
||||
FIND_FAILED,
|
||||
GONE,
|
||||
KEEPER,
|
||||
KEEPER_CHANGED,
|
||||
KEEPER_GONE,
|
||||
KEEPER_WENT,
|
||||
KEPT_COPY,
|
||||
LOCKED_FILE,
|
||||
MAYBE_SAME_FILE,
|
||||
NO_FILE_ID_ROW,
|
||||
NOT_ATTEMPTED,
|
||||
OUTSIDE_SOURCE,
|
||||
RECORDS_FAILED,
|
||||
ROW_DETAILS,
|
||||
SAME_FILE,
|
||||
SOURCES_CHANGED,
|
||||
SOURCES_CHANGED_REVIEW,
|
||||
HeldFind,
|
||||
SourcesChanged,
|
||||
_fingerprint,
|
||||
_group_order,
|
||||
_identity,
|
||||
_parse_options,
|
||||
_plural,
|
||||
_public_plan,
|
||||
_select,
|
||||
file_problem,
|
||||
human_bytes,
|
||||
still_kept,
|
||||
)
|
||||
from .duplicate_spellings import Spellings, copy_move
|
||||
from .images import _favorite_root_key, _output_image_entry
|
||||
from .job_routes import busy_response
|
||||
from .jobs import (
|
||||
DUPLICATE_CLEANUP_GROUP_NAMES,
|
||||
DUPLICATE_CLEANUP_JOB_KIND,
|
||||
DUPLICATE_GROUP_NAMES,
|
||||
DUPLICATES_JOB_KIND,
|
||||
JobBusy,
|
||||
)
|
||||
from .maintenance import DUPLICATE_FILES, _fail_job, _failure, _invalid, backup_note
|
||||
|
||||
try:
|
||||
from ...database.duplicate_copies import FILE_LINK_SQL
|
||||
except ImportError:
|
||||
from database.duplicate_copies import FILE_LINK_SQL
|
||||
|
||||
DUPLICATE_FIND_TITLE = "Find duplicate image files"
|
||||
DUPLICATE_CLEANUP_TITLE = "Remove duplicate image files"
|
||||
MAX_GROUP_PAGE = 10 # groups a page lists (D69: ten at a time)
|
||||
MAX_HELD_FINDS = 3 # finds whose groups the server keeps for review
|
||||
favorite_root_key = _favorite_root_key
|
||||
|
||||
|
||||
def query_plan_sql():
|
||||
"""The link reconciliation's statements (EXPLAIN QUERY PLAN checks)."""
|
||||
return list(FILE_LINK_SQL)
|
||||
|
||||
|
||||
def _unlink(path):
|
||||
"""Remove one file (the caller closed every handle on it)."""
|
||||
os.remove(path)
|
||||
|
||||
|
||||
class DuplicateFilesMixin:
|
||||
"""Mixin providing the duplicate-file find, its groups and its removal."""
|
||||
|
||||
def _register_duplicate_routes(self, routes):
|
||||
@routes.post("/prompt_manager/jobs/duplicates")
|
||||
async def start_duplicate_find_route(request):
|
||||
return await self.start_duplicate_find(request)
|
||||
|
||||
@routes.get("/prompt_manager/maintenance/duplicates/{job_id}")
|
||||
async def duplicate_groups_route(request):
|
||||
return await self.duplicate_groups(request)
|
||||
|
||||
# ── find ──────────────────────────────────────────────────────────
|
||||
|
||||
async def start_duplicate_find(self, request):
|
||||
"""Start the duplicate find (202), or 409 naming the job in the way."""
|
||||
try:
|
||||
job = self._start_duplicate_find(origin="workspace")
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
def _start_duplicate_find(self, origin):
|
||||
"""Register a find job (on the loop); raises JobBusy.
|
||||
|
||||
A workspace find walks every effective source; a gallery one the
|
||||
gallery's output folder, as the legacy scan did. Both resolve their
|
||||
folders when the job begins, so a source change later cannot move it.
|
||||
"""
|
||||
cleanup = self._jobs.active(DUPLICATE_CLEANUP_JOB_KIND)
|
||||
if cleanup is not None:
|
||||
raise JobBusy(cleanup)
|
||||
held = HeldFind(origin)
|
||||
if origin == "workspace":
|
||||
title = DUPLICATE_FIND_TITLE
|
||||
else:
|
||||
title = DUPLICATE_FIND_LEGACY_TITLE
|
||||
job = self._jobs.start(
|
||||
DUPLICATES_JOB_KIND,
|
||||
title,
|
||||
admission="read",
|
||||
cancellable=True,
|
||||
groups=DUPLICATE_GROUP_NAMES,
|
||||
runner=lambda job: self._run_duplicate_find(job, held),
|
||||
)
|
||||
held.job_id = job.id
|
||||
self._remember_find(held)
|
||||
return job
|
||||
|
||||
def _remember_find(self, held):
|
||||
finds = self._duplicate_finds
|
||||
finds[held.job_id] = held
|
||||
for job_id in list(finds):
|
||||
if self._jobs.get(job_id) is None:
|
||||
del finds[job_id]
|
||||
while len(finds) > MAX_HELD_FINDS:
|
||||
del finds[next(iter(finds))]
|
||||
|
||||
def _find_covers(self, job, legacy_dir):
|
||||
held = self._duplicate_finds.get(job.id)
|
||||
return held is not None and held.covers(legacy_dir)
|
||||
|
||||
def _latest_duplicate_find(self):
|
||||
"""The running find, else the newest finished one this session."""
|
||||
for job in self._jobs.list():
|
||||
if job.kind == DUPLICATES_JOB_KIND and job.id in self._duplicate_finds:
|
||||
return job
|
||||
return None
|
||||
|
||||
def _resolve_find_roots(self, held):
|
||||
"""Fill in where a find looks (blocking): sources and the gallery folder."""
|
||||
listed = [os.path.abspath(str(d)) for d in self._get_all_output_dirs()]
|
||||
held.legacy_dir = self._find_comfyui_output_dir()
|
||||
if held.origin == "workspace":
|
||||
held.roots = list(enumerate(listed))
|
||||
return
|
||||
if not held.legacy_dir or not os.path.isdir(held.legacy_dir):
|
||||
self.logger.warning("ComfyUI output directory not found")
|
||||
held.roots = []
|
||||
return
|
||||
legacy = os.path.abspath(held.legacy_dir)
|
||||
index = next(
|
||||
(i for i, d in enumerate(listed) if _identity(d) == _identity(legacy)),
|
||||
None,
|
||||
)
|
||||
held.roots = [(index, legacy)]
|
||||
|
||||
def _find_progress(self, job, done, total):
|
||||
if done < job.processed:
|
||||
return # hashing threads report out of order; keep the count rising
|
||||
pct = int(done / total * 100) if total else 100
|
||||
job.update(
|
||||
processed=done,
|
||||
total=total,
|
||||
status_line=f"Hashing file {done:,} of {total:,}",
|
||||
)
|
||||
job.publish(
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": pct,
|
||||
"status": f"Hashing file {done}/{total}...",
|
||||
"processed": done,
|
||||
"found": 0,
|
||||
}
|
||||
)
|
||||
|
||||
async def _run_duplicate_find(self, job, held):
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def report(done, total): # from the hashing threads
|
||||
loop.call_soon_threadsafe(self._find_progress, job, done, total)
|
||||
|
||||
try:
|
||||
job.update(status_line="Looking for media files")
|
||||
job.publish(
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 0,
|
||||
"status": "Collecting media files...",
|
||||
"processed": 0,
|
||||
"found": 0,
|
||||
}
|
||||
)
|
||||
await self._run_in_executor(self._resolve_find_roots, held)
|
||||
job.scope["output_folders"] = len(held.roots)
|
||||
job.log_line(f"Searching {_plural(len(held.roots), 'source')}")
|
||||
groups = await self._run_in_executor(
|
||||
self._hash_media_groups,
|
||||
[root for _index, root in held.roots],
|
||||
report,
|
||||
lambda: job.cancelled,
|
||||
)
|
||||
if job.cancelled:
|
||||
job.update(status_line="Cancelled; nothing was changed.")
|
||||
job.publish({"type": "error", "message": FIND_CANCELLED})
|
||||
job.finish("cancelled")
|
||||
return
|
||||
await self._finish_find(job, held, groups)
|
||||
except Exception:
|
||||
self.logger.exception("Duplicate search failed")
|
||||
job.update(status_line=FIND_FAILED)
|
||||
job.publish({"type": "error", "message": FIND_ERROR_EVENT})
|
||||
job.finish("failed")
|
||||
|
||||
async def _finish_find(self, job, held, groups):
|
||||
for group in groups:
|
||||
group["group_id"] = group["hash"]
|
||||
for record in group["files"]:
|
||||
index, root = held.roots[record.pop("slot")]
|
||||
record["root_index"] = index
|
||||
record["root"] = root
|
||||
groups.sort(key=_group_order)
|
||||
held.groups = groups
|
||||
labels = {root: self._public_path(root) for _index, root in held.roots}
|
||||
rows = [
|
||||
{
|
||||
"path": group["files"][0]["rel"],
|
||||
"root": labels[group["files"][0]["root"]],
|
||||
"root_index": group["files"][0]["root_index"],
|
||||
"detail": f"{len(group['files'])} identical files · "
|
||||
f"{human_bytes(group['size'])} each",
|
||||
}
|
||||
for group in groups
|
||||
]
|
||||
job.record_counted("groups_found", len(groups), rows)
|
||||
line = f"Found {_plural(len(groups), 'group')} of duplicate files"
|
||||
job.log_line(line)
|
||||
legacy = []
|
||||
if held.covers(held.legacy_dir) and held.legacy_dir:
|
||||
legacy = await self._run_in_executor(
|
||||
self._legacy_duplicate_groups, groups, held.legacy_dir
|
||||
)
|
||||
# Every file was read by now (the last reports may still be queued).
|
||||
job.update(progress=100, status_line=line, processed=job.total)
|
||||
job.publish(
|
||||
{
|
||||
"type": "complete",
|
||||
"processed": job.processed,
|
||||
"found": len(legacy),
|
||||
"duplicates": legacy,
|
||||
"message": f"Found {len(legacy)} groups of duplicate images",
|
||||
}
|
||||
)
|
||||
|
||||
# ── groups ────────────────────────────────────────────────────────
|
||||
|
||||
def _held_result(self, job_id):
|
||||
"""``(held, None)`` for a finished find's groups, else an error reply."""
|
||||
held = self._duplicate_finds.get(job_id)
|
||||
job = self._jobs.get(job_id)
|
||||
if held is None or job is None or job.kind != DUPLICATES_JOB_KIND:
|
||||
return None, _failure(
|
||||
"No duplicate search with this id. Search again.", 404, "not-found"
|
||||
)
|
||||
if job.running:
|
||||
return None, _failure(
|
||||
"The duplicate search is still running.", 409, "job-running"
|
||||
)
|
||||
if held.groups is None:
|
||||
return None, _failure(
|
||||
"This search did not finish. Search again.", 409, "no-groups"
|
||||
)
|
||||
return held, None
|
||||
|
||||
async def duplicate_groups(self, request):
|
||||
"""One page of a finished find's groups (at most MAX_GROUP_PAGE)."""
|
||||
try:
|
||||
offset = int(request.query.get("offset", "0"))
|
||||
limit = int(request.query.get("limit", str(MAX_GROUP_PAGE)))
|
||||
except ValueError:
|
||||
return _invalid("offset and limit must be whole numbers")
|
||||
if offset < 0 or not 0 < limit <= MAX_GROUP_PAGE:
|
||||
return _invalid(f"limit must be 1 to {MAX_GROUP_PAGE}; offset 0 or more")
|
||||
job_id = request.match_info["job_id"]
|
||||
held, error = self._held_result(job_id)
|
||||
if error is not None:
|
||||
return error
|
||||
page = held.groups[offset : offset + limit]
|
||||
groups = await self._run_in_executor(self._describe_groups, page)
|
||||
job = self._jobs.get(job_id)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"job": {"id": job.id, "state": job.state},
|
||||
"total": len(held.groups),
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"groups": groups,
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
def _describe_groups(self, groups):
|
||||
"""Public group rows (blocking: one stat per file)."""
|
||||
return [
|
||||
{
|
||||
"group_id": group["group_id"],
|
||||
"size": group["size"],
|
||||
"count": len(group["files"]),
|
||||
"files": [self._describe_copy(record) for record in group["files"]],
|
||||
}
|
||||
for group in groups
|
||||
]
|
||||
|
||||
def _describe_copy(self, record):
|
||||
label = self._public_path(record["root"])
|
||||
base = {
|
||||
"filename": os.path.basename(record["rel"]),
|
||||
"path": record["rel"],
|
||||
"root": label,
|
||||
"root_index": record["root_index"],
|
||||
"url": None,
|
||||
"thumbnail_url": None,
|
||||
"size": record["size"],
|
||||
"modified_time": record["mtime"],
|
||||
"media_type": "image",
|
||||
"missing": False,
|
||||
}
|
||||
if record["root_index"] is None:
|
||||
return base
|
||||
try:
|
||||
entry = _output_image_entry(
|
||||
Path(record["abs"]), Path(record["root"]), record["root_index"], label
|
||||
)
|
||||
except (OSError, ValueError):
|
||||
return {**base, "missing": True}
|
||||
return {
|
||||
**base,
|
||||
"url": entry["url"],
|
||||
"thumbnail_url": entry["thumbnail_url"],
|
||||
"size": entry["size"],
|
||||
"modified_time": entry["modified_time"],
|
||||
"media_type": entry["media_type"],
|
||||
}
|
||||
|
||||
# ── review ────────────────────────────────────────────────────────
|
||||
|
||||
def _listed_sources(self):
|
||||
"""``{identity: index}`` of the effective sources now (blocking)."""
|
||||
return {
|
||||
_identity(d): index for index, d in enumerate(self._get_all_output_dirs())
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _source_listed(listed, index, root):
|
||||
return index is not None and listed.get(_identity(root)) == index
|
||||
|
||||
async def preview_duplicate_files(self, options):
|
||||
"""The ``duplicate_files`` plan for one finished find and its keepers."""
|
||||
job_id, keep, message = _parse_options(options)
|
||||
if message:
|
||||
return _invalid(message)
|
||||
held, error = self._held_result(job_id)
|
||||
if error is not None:
|
||||
return error
|
||||
selection, message = _select(held, keep)
|
||||
if message:
|
||||
return _invalid(message)
|
||||
db = self.db
|
||||
identity = self._database_identity(db)
|
||||
try:
|
||||
review = await self._run_in_executor(
|
||||
self._review_duplicate_files, db, selection
|
||||
)
|
||||
except SourcesChanged:
|
||||
return _failure(SOURCES_CHANGED_REVIEW, 409, "sources-changed")
|
||||
except Exception:
|
||||
self.logger.exception("Duplicate file review failed")
|
||||
return _failure("Failed to review maintenance", 500)
|
||||
return await self._plan_response(
|
||||
db,
|
||||
identity,
|
||||
DUPLICATE_FILES,
|
||||
review["fingerprint"],
|
||||
review["public"],
|
||||
{
|
||||
"targets": review["targets"],
|
||||
"kept": review["kept"],
|
||||
"protected": review["public"]["protected"]["count"],
|
||||
},
|
||||
)
|
||||
|
||||
def _review_duplicate_files(self, db, selection):
|
||||
"""Check every keeper and removal now (blocking); the plan's parts.
|
||||
|
||||
No file is read: each copy is bound to what the search saw by its
|
||||
size, mtime and inode, and never to its kept copy itself. The removal
|
||||
job hashes each file again right before it unlinks it.
|
||||
"""
|
||||
listed = self._listed_sources()
|
||||
targets, kept, protected = [], [], {}
|
||||
for group, keepers in selection.values():
|
||||
removals = [r for r in group["files"] if r not in keepers]
|
||||
for record in removals:
|
||||
if record["root_index"] is not None and not self._source_listed(
|
||||
listed, record["root_index"], record["root"]
|
||||
):
|
||||
raise SourcesChanged(record["root"])
|
||||
keeper = next(
|
||||
(k for k in keepers if _bound_problem(k, group) is None), None
|
||||
)
|
||||
reviewed = []
|
||||
for record in removals:
|
||||
if keeper is None:
|
||||
problem = KEEPER
|
||||
elif record["root_index"] is None:
|
||||
problem = OUTSIDE_SOURCE
|
||||
else:
|
||||
problem = _bound_problem(record, group, keeper)
|
||||
if problem:
|
||||
protected[problem] = protected.get(problem, 0) + 1
|
||||
continue
|
||||
reviewed.append(self._target(group, record, keeper))
|
||||
if reviewed:
|
||||
targets.extend(reviewed)
|
||||
kept.append(self._copy_row(keeper, KEPT_COPY))
|
||||
records = {}
|
||||
if targets:
|
||||
spellings = self._source_spellings()
|
||||
records = db.count_copy_records(
|
||||
{
|
||||
t["abs"]: {
|
||||
"spellings": list(spellings.of(t["abs"]).values()),
|
||||
"locator": t["locator"],
|
||||
"sha256": t["hash"],
|
||||
}
|
||||
for t in targets
|
||||
}
|
||||
)
|
||||
return {
|
||||
"targets": targets,
|
||||
"kept": kept,
|
||||
"fingerprint": _fingerprint(targets),
|
||||
"public": _public_plan(targets, protected, records),
|
||||
}
|
||||
|
||||
def _source_spellings(self):
|
||||
"""The folder spellings records may name files by (blocking).
|
||||
|
||||
The search walks each source's real path, while a record names a file
|
||||
as it was seen when written: under the folder the image monitor
|
||||
watches, the configured spelling, ComfyUI's output folder, a symlink
|
||||
to it, or a parent of a source added later. ``Spellings`` matches
|
||||
them to each file by containment (``py/api/duplicate_spellings.py``).
|
||||
"""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
listed = [os.path.abspath(str(d)) for d in self._get_all_output_dirs()]
|
||||
candidates = list(self._monitored_directories())
|
||||
candidates += list(GalleryConfig.MONITORING_DIRECTORIES)
|
||||
try:
|
||||
import folder_paths
|
||||
|
||||
candidates.append(folder_paths.get_output_directory())
|
||||
except Exception: # nosec B110 - outside ComfyUI there is none
|
||||
pass
|
||||
candidates.append(self._find_comfyui_output_dir())
|
||||
folders = [
|
||||
os.path.abspath(str(c)) for c in candidates if c and os.path.isabs(str(c))
|
||||
]
|
||||
return Spellings(folders + listed, listed)
|
||||
|
||||
def _target(self, group, record, keeper):
|
||||
return {
|
||||
"group_id": group["group_id"],
|
||||
"hash": group["hash"],
|
||||
"size": record["size"],
|
||||
"root_index": record["root_index"],
|
||||
"root": record["root"],
|
||||
"root_label": self._public_path(record["root"]),
|
||||
"abs": record["abs"],
|
||||
"rel": record["rel"],
|
||||
"keeper_abs": keeper["abs"],
|
||||
"keeper_rel": keeper["rel"],
|
||||
"keeper_root": keeper["root"],
|
||||
"keeper_size": keeper["size"],
|
||||
"locator": os.path.normcase(record["abs"]),
|
||||
"keeper_locator": os.path.normcase(keeper["abs"]),
|
||||
}
|
||||
|
||||
def _copy_row(self, record, detail):
|
||||
return {
|
||||
"path": record["rel"],
|
||||
"root": self._public_path(record["root"]),
|
||||
"root_index": record["root_index"],
|
||||
"detail": detail,
|
||||
}
|
||||
|
||||
# ── removal ───────────────────────────────────────────────────────
|
||||
|
||||
def _start_duplicate_cleanup(self, db, plan, backup_on):
|
||||
"""Register the removal job for a reviewed plan (on the loop)."""
|
||||
find = self._jobs.active(DUPLICATES_JOB_KIND)
|
||||
if find is not None:
|
||||
raise JobBusy(find)
|
||||
return self._jobs.start(
|
||||
DUPLICATE_CLEANUP_JOB_KIND,
|
||||
DUPLICATE_CLEANUP_TITLE,
|
||||
admission="records",
|
||||
cancellable=False,
|
||||
scope={"operation": DUPLICATE_FILES},
|
||||
groups=DUPLICATE_CLEANUP_GROUP_NAMES,
|
||||
runner=lambda job: self._run_duplicate_cleanup(job, db, plan, backup_on),
|
||||
)
|
||||
|
||||
def _sources_unchanged(self, targets):
|
||||
listed = self._listed_sources()
|
||||
return all(
|
||||
self._source_listed(listed, t["root_index"], t["root"]) for t in targets
|
||||
)
|
||||
|
||||
async def _run_duplicate_cleanup(self, job, db, plan, backup_on):
|
||||
targets = plan.payload["targets"]
|
||||
unchanged = await self._run_in_executor(self._sources_unchanged, targets)
|
||||
if not unchanged:
|
||||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return
|
||||
try:
|
||||
await self._remove_duplicates(job, db, plan, backup)
|
||||
finally:
|
||||
if backup:
|
||||
await self._prune_backup_pool(job, db)
|
||||
|
||||
async def _remove_duplicates(self, job, db, plan, backup):
|
||||
targets = plan.payload["targets"]
|
||||
for row in plan.payload["kept"]:
|
||||
job.record("kept", row, f"= {row['root']}/{row['path']} · {KEPT_COPY}")
|
||||
job.update(total=len(targets), status_line="Removing duplicate files")
|
||||
spellings = await self._run_in_executor(self._source_spellings)
|
||||
removed, failure, keepers = 0, None, {}
|
||||
totals = dict.fromkeys(("moved", "merged", "favorites", "tags", "review"), 0)
|
||||
for done, target in enumerate(targets, start=1):
|
||||
if failure is not None:
|
||||
self._record_target(job, "kept", target, NOT_ATTEMPTED)
|
||||
continue
|
||||
outcome, detail, counts = await self._run_in_executor(
|
||||
self._remove_one_duplicate, db, target, keepers, spellings
|
||||
)
|
||||
for key, value in (counts or {}).items():
|
||||
totals[key] += value
|
||||
if outcome == "removed":
|
||||
removed += 1
|
||||
text = f"A copy of {target['keeper_rel']}"
|
||||
self._record_target(job, "removed", target, text, "-")
|
||||
elif outcome == "kept":
|
||||
self._record_target(job, "kept", target, ROW_DETAILS[detail])
|
||||
else:
|
||||
failure = detail
|
||||
self._record_target(job, "errors", target, detail, "!")
|
||||
job.update(processed=done, progress=int(done / len(targets) * 100))
|
||||
if removed:
|
||||
self.invalidate_gallery_cache()
|
||||
self._log_moved_records(job, totals)
|
||||
line = f"Removed {_plural(removed, 'duplicate file')}"
|
||||
if removed < len(targets):
|
||||
line += f", {len(targets) - removed:,} not removed"
|
||||
job.log_line(line)
|
||||
if failure is not None:
|
||||
line = f"Stopped after an error · {line}"
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
if failure is not None:
|
||||
job.publish({"type": "error", "code": "delete-failed", "message": line})
|
||||
job.finish("failed")
|
||||
|
||||
@staticmethod
|
||||
def _log_moved_records(job, totals):
|
||||
links = totals["moved"] + totals["merged"]
|
||||
if links:
|
||||
job.log_line(f"Moved {_plural(links, 'database link')} to the kept copies")
|
||||
if totals["favorites"]:
|
||||
job.log_line(
|
||||
f"Moved {_plural(totals['favorites'], 'favorite')} to the kept copies"
|
||||
)
|
||||
if totals["tags"]:
|
||||
job.log_line(
|
||||
f"Added {_plural(totals['tags'], 'image tag')} to the kept copies"
|
||||
)
|
||||
if totals["review"]:
|
||||
job.log_line(
|
||||
f"Settled {_plural(totals['review'], 'Needs review item')} "
|
||||
"of removed copies"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _record_target(job, group, target, detail, mark="="):
|
||||
row = {
|
||||
"path": target["rel"],
|
||||
"root": target["root_label"],
|
||||
"root_index": target["root_index"],
|
||||
"detail": detail,
|
||||
}
|
||||
job.record(
|
||||
group, row, f"{mark} {target['root_label']}/{target['rel']} · {detail}"
|
||||
)
|
||||
|
||||
def _remove_one_duplicate(self, db, target, keepers, spellings):
|
||||
"""``(outcome, detail, counts)`` for one reviewed removal (blocking).
|
||||
|
||||
The kept copy (once per group; cached in ``keepers``) and the file
|
||||
are checked again, hashes included, and the file must not be its kept
|
||||
copy. Then the file's records move to the kept copy in one short
|
||||
transaction, and only then is the file unlinked: a crash or a failed
|
||||
unlink leaves its records on the identical kept copy, never on a
|
||||
deleted file. A failed move leaves the file and its records alone.
|
||||
"""
|
||||
hasher = self._calculate_file_hash
|
||||
group_id = target["group_id"]
|
||||
if group_id not in keepers:
|
||||
keepers[group_id] = _kept_copy_state(target, hasher)
|
||||
if keepers[group_id] is None:
|
||||
return "kept", KEEPER, None
|
||||
problem = file_problem(
|
||||
target["abs"],
|
||||
target["root"],
|
||||
target["hash"],
|
||||
target["size"],
|
||||
hasher,
|
||||
keeper=target["keeper_abs"],
|
||||
keeper_root=target["keeper_root"],
|
||||
)
|
||||
if problem:
|
||||
return "kept", problem, None
|
||||
try:
|
||||
# Held open until the unlink: on Windows, deleting the kept copy
|
||||
# itself through another path then fails instead of removing it.
|
||||
hold = open(target["keeper_abs"], "rb")
|
||||
except OSError:
|
||||
return "kept", KEEPER, None
|
||||
with hold:
|
||||
return self._move_then_unlink(
|
||||
db, target, keepers[group_id], spellings, hold
|
||||
)
|
||||
|
||||
def _move_then_unlink(self, db, target, kept_stat, spellings, hold):
|
||||
try:
|
||||
counts = db.move_copy_records(copy_move(target, kept_stat, spellings))
|
||||
except Exception:
|
||||
self.logger.exception(f"Could not move the records of {target['rel']}")
|
||||
return "error", RECORDS_FAILED, None
|
||||
if not still_kept(target, hold):
|
||||
# The kept copy went away or changed while the records moved:
|
||||
# keep this file and give its records back to it.
|
||||
return "kept", KEEPER_GONE, self._move_back(db, target, spellings)
|
||||
try:
|
||||
_unlink(target["abs"])
|
||||
except FileNotFoundError:
|
||||
return "removed", None, counts
|
||||
except PermissionError:
|
||||
self.logger.warning(
|
||||
f"Duplicate file is locked: {target['rel']}; "
|
||||
"its records now name the kept copy"
|
||||
)
|
||||
return "error", LOCKED_FILE, counts
|
||||
except OSError:
|
||||
self.logger.warning(
|
||||
f"Could not delete duplicate file {target['rel']}; "
|
||||
"its records now name the kept copy",
|
||||
exc_info=True,
|
||||
)
|
||||
return "error", DELETE_FAILED, counts
|
||||
self.logger.info(f"Deleted duplicate file: {target['rel']}")
|
||||
_remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
|
||||
return "removed", None, counts
|
||||
|
||||
def _move_back(self, db, target, spellings):
|
||||
"""Give the kept copy's records to the file that stays (blocking)."""
|
||||
swapped = {
|
||||
**target,
|
||||
"abs": target["keeper_abs"],
|
||||
"root": target["keeper_root"],
|
||||
"rel": target["keeper_rel"],
|
||||
"locator": target["keeper_locator"],
|
||||
"keeper_abs": target["abs"],
|
||||
"keeper_root": target["root"],
|
||||
"keeper_rel": target["rel"],
|
||||
"keeper_locator": target["locator"],
|
||||
}
|
||||
try:
|
||||
return db.move_copy_records(
|
||||
copy_move(swapped, os.stat(target["abs"]), spellings)
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception(f"Could not give the records back to {target['rel']}")
|
||||
return None
|
||||
|
||||
|
||||
def _bound_problem(record, group, keeper=None):
|
||||
"""``file_problem`` without reading the file: size, mtime and inode.
|
||||
|
||||
``keeper`` is the kept copy's record, never offered as a copy of itself.
|
||||
"""
|
||||
return file_problem(
|
||||
record["abs"],
|
||||
record["root"],
|
||||
group["hash"],
|
||||
record["size"],
|
||||
hash_check=False,
|
||||
mtime_ns=record.get("mtime_ns"),
|
||||
inode=record.get("inode"),
|
||||
keeper=keeper["abs"] if keeper else None,
|
||||
keeper_root=keeper["root"] if keeper else None,
|
||||
)
|
||||
|
||||
|
||||
def _kept_copy_state(target, hasher):
|
||||
"""The kept copy's ``os.stat`` when it is still the reviewed bytes, else None."""
|
||||
problem = file_problem(
|
||||
target["keeper_abs"],
|
||||
target["keeper_root"],
|
||||
target["hash"],
|
||||
target["keeper_size"],
|
||||
hasher,
|
||||
)
|
||||
if problem:
|
||||
return None
|
||||
try:
|
||||
return os.stat(target["keeper_abs"])
|
||||
except OSError:
|
||||
return None
|
||||
@@ -0,0 +1,451 @@
|
||||
"""Reviewing duplicate image files: the checks and the plan (#249 S6, T17).
|
||||
|
||||
Pure helpers of ``py/api/duplicate_files.py``: the keep-list parsing, the
|
||||
per-file check ``file_problem`` (containment decided as text before any
|
||||
filesystem access, then a regular file of the found size and hash) and the
|
||||
public ``duplicate_files`` plan.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat as stat_module
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from .admin import DELETABLE_MEDIA_EXTENSIONS
|
||||
from .maintenance import DUPLICATE_FILES
|
||||
|
||||
# Keep entries one review takes; with paths of a few hundred characters the
|
||||
# list stays inside the preview body cap (DUPLICATE_PREVIEW_BODY_MAX, 1 MiB).
|
||||
MAX_KEEP = 2000
|
||||
MAX_DETAILS = 50
|
||||
MAX_JOB_ID = 64
|
||||
HASH_CHUNK = 1024 * 1024
|
||||
|
||||
# What ``file_problem`` finds, and how a plan counts it as protected.
|
||||
OUTSIDE_SOURCE = "outside"
|
||||
LINK = "link"
|
||||
CHANGED_CONTENT = "changed"
|
||||
GONE = "gone"
|
||||
UNREADABLE = "unreadable"
|
||||
KEEPER = "keeper"
|
||||
SAME = "same"
|
||||
MAYBE_SAME = "maybe-same"
|
||||
NO_FILE_ID = "no-file-id"
|
||||
# Two mounts of one share may round mtimes differently (SMB keeps 100 ns,
|
||||
# FAT 2 s): within this, equal sizes on another device may be one file.
|
||||
MOUNT_MTIME_SLACK_NS = 2_000_000_000
|
||||
PROTECTED_REASONS = {
|
||||
MAYBE_SAME: (
|
||||
"file may be the same file reached through another mount or share",
|
||||
"files may be the same file reached through another mount or share",
|
||||
),
|
||||
NO_FILE_ID: (
|
||||
"file: can't confirm this is a separate file on this drive",
|
||||
"files: can't confirm these are separate files on this drive",
|
||||
),
|
||||
SAME: (
|
||||
"file is the same file as its kept copy",
|
||||
"files are the same file as their kept copy",
|
||||
),
|
||||
OUTSIDE_SOURCE: (
|
||||
"file is not inside a configured source",
|
||||
"files are not inside a configured source",
|
||||
),
|
||||
LINK: ("file is a link, not a regular file", "files are links, not regular files"),
|
||||
CHANGED_CONTENT: (
|
||||
"file changed since the search",
|
||||
"files changed since the search",
|
||||
),
|
||||
GONE: ("file no longer exists", "files no longer exist"),
|
||||
UNREADABLE: ("file could not be read", "files could not be read"),
|
||||
KEEPER: (
|
||||
"file kept: its kept copy changed or is gone",
|
||||
"files kept: their kept copy changed or is gone",
|
||||
),
|
||||
}
|
||||
# Job outcome rows (never an OS error text).
|
||||
CHANGED = "The file changed since the review; it was kept."
|
||||
KEEPER_CHANGED = "The kept copy changed or is gone; this file was kept."
|
||||
SAME_FILE = "This is the same file as its kept copy; it was kept."
|
||||
MAYBE_SAME_FILE = (
|
||||
"May be the same file reached through another mount or share; it was kept."
|
||||
)
|
||||
NO_FILE_ID_ROW = "Can't confirm this is a separate file on this drive; it was kept."
|
||||
KEEPER_GONE = "keeper-gone"
|
||||
KEEPER_WENT = (
|
||||
"The kept copy changed or went away during the removal; this file was kept "
|
||||
"and its links stay with it."
|
||||
)
|
||||
ROW_DETAILS = {
|
||||
KEEPER_GONE: None, # filled below
|
||||
MAYBE_SAME: MAYBE_SAME_FILE,
|
||||
NO_FILE_ID: NO_FILE_ID_ROW,
|
||||
SAME: SAME_FILE,
|
||||
OUTSIDE_SOURCE: "The file is no longer inside its source; it was kept.",
|
||||
LINK: "The file is now a link, not a regular file; it was kept.",
|
||||
CHANGED_CONTENT: CHANGED,
|
||||
GONE: "The file was already gone.",
|
||||
UNREADABLE: "The file could not be read; it was kept.",
|
||||
KEEPER: KEEPER_CHANGED,
|
||||
}
|
||||
# An unlink fails only after the file's records moved to its kept copy.
|
||||
LOCKED_FILE = (
|
||||
"The file is open in another program or read-only; it was not deleted, "
|
||||
"and its links now name the kept copy."
|
||||
)
|
||||
DELETE_FAILED = "The file could not be deleted; its links now name the kept copy."
|
||||
NOT_ATTEMPTED = "Not attempted: the job stopped after an error."
|
||||
KEPT_COPY = "Kept copy"
|
||||
SOURCES_CHANGED = (
|
||||
"The sources changed since the review; nothing was deleted. Review again."
|
||||
)
|
||||
RECORDS_FAILED = (
|
||||
"The database could not be updated, so the file was not deleted. "
|
||||
"Its links and tags are unchanged."
|
||||
)
|
||||
FIND_FAILED = "The duplicate search failed."
|
||||
FIND_ERROR_EVENT = "An internal error occurred. Check server logs for details."
|
||||
FIND_CANCELLED = "Duplicate scan cancelled"
|
||||
|
||||
|
||||
class SourcesChanged(Exception):
|
||||
"""A source a review or removal names is no longer listed at its index."""
|
||||
|
||||
|
||||
ROW_DETAILS[KEEPER_GONE] = KEEPER_WENT
|
||||
|
||||
|
||||
def still_kept(target, hold):
|
||||
"""Right before an unlink: the held kept copy is still the file at its
|
||||
path, and the file is still not the kept copy itself (``keeper_problem``).
|
||||
"""
|
||||
try:
|
||||
if not os.path.samestat(os.fstat(hold.fileno()), os.stat(target["keeper_abs"])):
|
||||
return False
|
||||
info = os.lstat(target["abs"])
|
||||
problem = keeper_problem(
|
||||
target["abs"],
|
||||
target["root"],
|
||||
info,
|
||||
target["keeper_abs"],
|
||||
target["keeper_root"],
|
||||
)
|
||||
return problem is None
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _plural(count, one, many=None):
|
||||
return f"{count:,} {one if count == 1 else (many or one + 's')}"
|
||||
|
||||
|
||||
def human_bytes(size):
|
||||
"""``10 B``, ``1.5 KB``, ``2.0 MB``…"""
|
||||
if size < 1024:
|
||||
return f"{size:,} B"
|
||||
for unit in ("KB", "MB", "GB", "TB"):
|
||||
size /= 1024
|
||||
if size < 1024 or unit == "TB":
|
||||
return f"{size:,.1f} {unit}"
|
||||
return f"{size:,.1f} TB" # pragma: no cover - unreachable
|
||||
|
||||
|
||||
def _hash_file(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(HASH_CHUNK), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def file_problem(
|
||||
path,
|
||||
root,
|
||||
digest,
|
||||
size,
|
||||
hasher=None,
|
||||
*,
|
||||
hash_check=True,
|
||||
mtime_ns=None,
|
||||
inode=None,
|
||||
keeper=None,
|
||||
keeper_root=None,
|
||||
):
|
||||
"""Why ``path`` may not be treated as a copy in ``root``, or None (blocking).
|
||||
|
||||
Containment is decided as text first, with no filesystem access: the
|
||||
normalized path must lie below the normalized root, outside
|
||||
``thumbnails``, with a media extension. Only then is the file looked at:
|
||||
a regular file, not a link, whose resolved path is still inside the
|
||||
resolved root and that is not ``keeper`` itself (``keeper_problem``: a
|
||||
hardlink, one file reached through two spellings or two mounts, or a
|
||||
volume that cannot tell files apart), of ``size`` bytes.
|
||||
``mtime_ns`` and ``inode`` (``[st_dev, st_ino]``), when given, bind it to
|
||||
what the search saw; ``hash_check`` compares its content hash with
|
||||
``digest`` (the removal job, right before an unlink). Every handle is
|
||||
closed when this returns.
|
||||
"""
|
||||
# py.config imports ComfyUI's server module: only when it is in use.
|
||||
from ..config import _canonical_path, _lexical_path, _lexically_inside
|
||||
|
||||
lexical = _lexical_path(str(path))
|
||||
base = _lexical_path(str(root))
|
||||
if lexical == base or not _lexically_inside(lexical, [base]):
|
||||
return OUTSIDE_SOURCE
|
||||
parts = Path(os.path.relpath(lexical, base)).parts
|
||||
if "thumbnails" in parts or ".." in parts:
|
||||
return OUTSIDE_SOURCE
|
||||
if os.path.splitext(lexical)[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||||
return OUTSIDE_SOURCE
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
return GONE
|
||||
except OSError:
|
||||
return UNREADABLE
|
||||
if stat_module.S_ISLNK(info.st_mode):
|
||||
return LINK
|
||||
if not stat_module.S_ISREG(info.st_mode):
|
||||
return LINK
|
||||
try:
|
||||
if not _lexically_inside(_canonical_path(path), [_canonical_path(root)]):
|
||||
return OUTSIDE_SOURCE # a linked folder on the way
|
||||
if keeper is not None:
|
||||
same = keeper_problem(path, root, info, keeper, keeper_root)
|
||||
if same:
|
||||
return same
|
||||
if info.st_size != size:
|
||||
return CHANGED_CONTENT
|
||||
if mtime_ns is not None and info.st_mtime_ns != mtime_ns:
|
||||
return CHANGED_CONTENT
|
||||
if inode is not None and [info.st_dev, info.st_ino] != list(inode):
|
||||
return CHANGED_CONTENT
|
||||
if hash_check and (hasher or _hash_file)(path) != digest:
|
||||
return CHANGED_CONTENT
|
||||
except FileNotFoundError:
|
||||
return GONE
|
||||
except OSError:
|
||||
return UNREADABLE
|
||||
return None
|
||||
|
||||
|
||||
def keeper_problem(path, root, info, keeper, keeper_root=None):
|
||||
"""Why ``path`` (its ``lstat`` is ``info``) may be ``keeper`` itself, or None.
|
||||
|
||||
- ``NO_FILE_ID``: a volume reports no file id (``st_ino`` 0), so the two
|
||||
cannot be told apart;
|
||||
- ``SAME``: ``os.path.samestat`` (a hardlink, or one file reached through
|
||||
two spellings of one mount);
|
||||
- ``MAYBE_SAME``: another device, the same size, mtimes within
|
||||
``MOUNT_MTIME_SLACK_NS``, and the same ``st_ino`` (NFS and SMB report
|
||||
the server's file id) or the same path within their sources, or one
|
||||
ending with the other on a folder name (one share mounted twice, a
|
||||
FUSE pool over a listed disk, sources listed at different depths).
|
||||
Protect, never guess: a true copy on another disk at the same relative
|
||||
path is kept too.
|
||||
|
||||
Raises OSError when the kept copy cannot be read (the caller's GONE or
|
||||
UNREADABLE).
|
||||
"""
|
||||
kept = os.stat(keeper)
|
||||
if not info.st_ino or not kept.st_ino:
|
||||
return NO_FILE_ID
|
||||
if os.path.samestat(info, kept):
|
||||
return SAME
|
||||
if info.st_dev == kept.st_dev or info.st_size != kept.st_size:
|
||||
return None
|
||||
if abs(info.st_mtime_ns - kept.st_mtime_ns) > MOUNT_MTIME_SLACK_NS:
|
||||
return None
|
||||
if info.st_ino == kept.st_ino:
|
||||
return MAYBE_SAME
|
||||
if keeper_root is not None and _same_tail(
|
||||
_relative_key(path, root), _relative_key(keeper, keeper_root)
|
||||
):
|
||||
return MAYBE_SAME
|
||||
return None
|
||||
|
||||
|
||||
def _same_tail(one, other):
|
||||
"""Whether one relative path is the other, or ends with it on a folder name.
|
||||
|
||||
Two mounts of one tree may be listed at different depths (``2024/x.png``
|
||||
in one source, ``x.png`` in the other); ``ax.png`` never matches ``x.png``.
|
||||
"""
|
||||
if one is None or other is None:
|
||||
return False
|
||||
return one == other or one.endswith("/" + other) or other.endswith("/" + one)
|
||||
|
||||
|
||||
def _relative_key(path, root):
|
||||
try:
|
||||
relative = os.path.relpath(str(path), str(root))
|
||||
except ValueError: # another drive (Windows)
|
||||
return None
|
||||
return os.path.normcase(relative).replace("\\", "/")
|
||||
|
||||
|
||||
@dataclass
|
||||
class HeldFind:
|
||||
"""One find: where it looked and, once it finished, what it found."""
|
||||
|
||||
origin: str # "workspace" or "gallery"
|
||||
job_id: str = ""
|
||||
roots: list = field(default_factory=list) # [(source index or None, root)]
|
||||
legacy_dir: str = None
|
||||
groups: list = None # [{group_id, hash, size, files:[record]}]
|
||||
|
||||
def covers(self, legacy_dir):
|
||||
"""Whether its events answer the legacy gallery for ``legacy_dir``."""
|
||||
if self.origin == "gallery":
|
||||
return True
|
||||
if not legacy_dir:
|
||||
return False
|
||||
wanted = _identity(legacy_dir)
|
||||
return any(_identity(root) == wanted for _index, root in self.roots)
|
||||
|
||||
|
||||
def _identity(path):
|
||||
return os.path.normcase(os.path.realpath(str(path)))
|
||||
|
||||
|
||||
def _group_order(group):
|
||||
return (-(len(group["files"]) - 1) * group["size"], group["hash"])
|
||||
|
||||
|
||||
def _keep_entry(entry):
|
||||
"""``((group_id, root_index, path), None)`` or ``(None, message)``."""
|
||||
if not isinstance(entry, dict) or set(entry) != {"group_id", "root_index", "path"}:
|
||||
return None, "Each keep entry is {group_id, root_index, path}"
|
||||
group_id, index, path = entry["group_id"], entry["root_index"], entry["path"]
|
||||
if not isinstance(group_id, str) or not 0 < len(group_id) <= 128:
|
||||
return None, "group_id must be a group's id"
|
||||
if index is not None and (
|
||||
isinstance(index, bool) or not isinstance(index, int) or index < 0
|
||||
):
|
||||
return None, "root_index must be a source index"
|
||||
if not isinstance(path, str) or not 0 < len(path) <= 4096:
|
||||
return None, "path must be a file's path in its group"
|
||||
return (group_id, index, path), None
|
||||
|
||||
|
||||
def _parse_options(options):
|
||||
"""``(job_id, keep, None)`` for duplicate_files options, else a message."""
|
||||
if not isinstance(options, dict) or set(options) != {"job_id", "keep"}:
|
||||
return None, None, "duplicate_files takes options {job_id, keep}"
|
||||
job_id, keep = options["job_id"], options["keep"]
|
||||
if not isinstance(job_id, str) or not 0 < len(job_id) <= MAX_JOB_ID:
|
||||
return None, None, "job_id must be a duplicate search's job id"
|
||||
if not isinstance(keep, list) or not 0 < len(keep) <= MAX_KEEP:
|
||||
return None, None, f"keep must list 1 to {MAX_KEEP:,} files to keep"
|
||||
parsed = []
|
||||
for entry in keep:
|
||||
key, message = _keep_entry(entry)
|
||||
if message:
|
||||
return None, None, message
|
||||
parsed.append(key)
|
||||
return job_id, parsed, None
|
||||
|
||||
|
||||
def _select(held, keep):
|
||||
"""``({group_id: (group, [keeper records])}, None)`` or ``(None, message)``."""
|
||||
groups = {group["group_id"]: group for group in held.groups}
|
||||
selection = {}
|
||||
for group_id, index, path in keep:
|
||||
group = groups.get(group_id)
|
||||
if group is None:
|
||||
return None, "A kept file names a group this search did not find"
|
||||
match = [
|
||||
record
|
||||
for record in group["files"]
|
||||
if record["root_index"] == index and record["rel"] == path
|
||||
]
|
||||
if not match:
|
||||
return None, "A kept file is not in its group"
|
||||
_group, keepers = selection.setdefault(group_id, (group, []))
|
||||
if match[0] not in keepers:
|
||||
keepers.append(match[0])
|
||||
return selection, None
|
||||
|
||||
|
||||
SOURCES_CHANGED_REVIEW = (
|
||||
"The sources changed since the search. Search for duplicates again."
|
||||
)
|
||||
|
||||
|
||||
def _fingerprint(targets):
|
||||
digest = hashlib.sha256()
|
||||
for target in sorted(targets, key=lambda t: _identity(t["abs"])):
|
||||
part = [
|
||||
os.path.normcase(target["abs"]),
|
||||
target["hash"],
|
||||
target["size"],
|
||||
os.path.normcase(target["keeper_abs"]),
|
||||
]
|
||||
digest.update(json.dumps(part).encode("utf-8") + b"\n")
|
||||
return "sha256:" + digest.hexdigest()
|
||||
|
||||
|
||||
def _record_parts(records, target):
|
||||
"""What moves with one removal: its links, tags and Needs review items."""
|
||||
found = records.get(target["abs"], {})
|
||||
links, tags, review = (found.get(key, 0) for key in ("links", "tags", "review"))
|
||||
parts = [
|
||||
(
|
||||
(
|
||||
f"{_plural(links, 'database link')} "
|
||||
f"{'moves' if links == 1 else 'move'} to the kept copy"
|
||||
)
|
||||
if links
|
||||
else "no database links"
|
||||
)
|
||||
]
|
||||
if tags:
|
||||
parts.append("its image tags join the kept copy's")
|
||||
if review:
|
||||
item = "Needs review item is" if review == 1 else "Needs review items are"
|
||||
parts.append(f"{review:,} {item} settled")
|
||||
return parts
|
||||
|
||||
|
||||
def _public_plan(targets, protected, records):
|
||||
count = len(targets)
|
||||
groups = len({t["group_id"] for t in targets})
|
||||
size = sum(t["size"] for t in targets)
|
||||
reasons = [
|
||||
f"{n:,} {PROTECTED_REASONS[code][0 if n == 1 else 1]}"
|
||||
for code, n in sorted(protected.items())
|
||||
]
|
||||
details = [
|
||||
{
|
||||
"label": target["rel"],
|
||||
"detail": f"{target['root_label']} · a copy of {target['keeper_rel']}"
|
||||
+ "".join(f" · {part}" for part in _record_parts(records, target)),
|
||||
}
|
||||
for target in targets[:MAX_DETAILS]
|
||||
]
|
||||
label = f"{_plural(count, 'duplicate file')} in {_plural(groups, 'group')}"
|
||||
media = (
|
||||
f"Deletes {_plural(count, 'file')} ({human_bytes(size)}) from disk; the "
|
||||
"kept copies stay. The safety backup covers the database, not these files."
|
||||
)
|
||||
tagged = sum(1 for t in targets if records.get(t["abs"], {}).get("tags"))
|
||||
review = sum(records.get(t["abs"], {}).get("review", 0) for t in targets)
|
||||
if tagged:
|
||||
media += (
|
||||
f" The image tags of {_plural(tagged, 'removed copy', 'removed copies')}"
|
||||
" join their kept copy's."
|
||||
)
|
||||
if review:
|
||||
media += (
|
||||
f" {_plural(review, 'pending Needs review item')} of the removed "
|
||||
f"{'copy is' if review == 1 else 'copies are'} settled."
|
||||
)
|
||||
return {
|
||||
"operation": DUPLICATE_FILES,
|
||||
"scope": {"count": count, "label": label},
|
||||
"protected": {"count": sum(protected.values()), "reasons": reasons},
|
||||
"media": {"deletes_files": True, "label": media},
|
||||
"details": details,
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Every path a duplicate copy is recorded under (#249 System S6, Task 17).
|
||||
|
||||
The search walks each source's real path, but a record names a file the way
|
||||
it was seen when it was written: under the folder the image monitor watches,
|
||||
the configured spelling, ComfyUI's output folder, a symlink to it, or a
|
||||
parent of a source listed later. ``Spellings`` finds them by containment:
|
||||
for each known folder spelling ``S`` (resolving to the folder ``R``) and a
|
||||
file ``P`` inside ``R``, the file is also recorded as ``S + relpath(P, R)``.
|
||||
|
||||
Folders are compared by ``os.stat`` identity (``st_dev``, ``st_ino``), not
|
||||
by text, so a case variant on a case-insensitive filesystem (macOS, Windows)
|
||||
or a symlinked spelling matches. A volume that reports no file id falls
|
||||
back to ``normcase(realpath)``. Favorites are keyed by a listed root and the
|
||||
path below it, so ``favorites`` lists every listed root that holds a file.
|
||||
Blocking: each lookup stats the file's folders once (cached).
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from .images import _favorite_relative_key, _favorite_root_key
|
||||
|
||||
|
||||
def _identity(path):
|
||||
try:
|
||||
info = os.stat(path)
|
||||
except OSError:
|
||||
return None
|
||||
if info.st_ino:
|
||||
return ("inode", info.st_dev, info.st_ino)
|
||||
return ("path", os.path.normcase(os.path.realpath(str(path))))
|
||||
|
||||
|
||||
class Spellings:
|
||||
"""Folder spellings and listed roots, matched to files by containment."""
|
||||
|
||||
def __init__(self, folders, listed_roots):
|
||||
self._cache = {}
|
||||
self._folders = []
|
||||
for spelled in dict.fromkeys(str(f) for f in folders if f):
|
||||
identity = self._identity(spelled)
|
||||
if identity is not None:
|
||||
self._folders.append((spelled, identity))
|
||||
self._listed = []
|
||||
for root in dict.fromkeys(str(r) for r in listed_roots if r):
|
||||
identity = self._identity(root)
|
||||
if identity is not None:
|
||||
self._listed.append((root, identity))
|
||||
|
||||
def _identity(self, path):
|
||||
if path not in self._cache:
|
||||
self._cache[path] = _identity(path)
|
||||
return self._cache[path]
|
||||
|
||||
def _ancestors(self, path):
|
||||
"""``{identity: folder}`` of every folder above ``path``, nearest first."""
|
||||
found = {}
|
||||
folder = os.path.dirname(os.path.abspath(str(path)))
|
||||
while True:
|
||||
identity = self._identity(folder)
|
||||
if identity is not None:
|
||||
found.setdefault(identity, folder)
|
||||
parent = os.path.dirname(folder)
|
||||
if parent == folder:
|
||||
return found
|
||||
folder = parent
|
||||
|
||||
def of(self, path):
|
||||
"""``{folder spelling: path spelled under it}``; ``""`` is ``path`` itself."""
|
||||
path = os.path.abspath(str(path))
|
||||
spelled = {"": path}
|
||||
ancestors = self._ancestors(path)
|
||||
for folder, identity in self._folders:
|
||||
inside = ancestors.get(identity)
|
||||
if inside is not None:
|
||||
spelled.setdefault(
|
||||
folder, os.path.join(folder, os.path.relpath(path, inside))
|
||||
)
|
||||
return spelled
|
||||
|
||||
def favorites(self, path):
|
||||
"""``{listed root: path below it (POSIX)}`` for each listed root holding it."""
|
||||
path = os.path.abspath(str(path))
|
||||
ancestors = self._ancestors(path)
|
||||
held = {}
|
||||
for root, identity in self._listed:
|
||||
inside = ancestors.get(identity)
|
||||
if inside is not None:
|
||||
held[root] = Path(os.path.relpath(path, inside)).as_posix()
|
||||
return held
|
||||
|
||||
|
||||
def copy_move(target, kept_stat, spellings):
|
||||
"""What ``move_copy_records`` moves for one removal.
|
||||
|
||||
A record found under a folder spelling moves to the kept copy under
|
||||
the same spelling when that folder holds it too, else to its real
|
||||
path. Its favorite moves under every listed root holding the file.
|
||||
"""
|
||||
removed = spellings.of(target["abs"])
|
||||
kept = spellings.of(target["keeper_abs"])
|
||||
targets = {
|
||||
spelled: kept.get(folder, target["keeper_abs"])
|
||||
for folder, spelled in removed.items()
|
||||
}
|
||||
kept_favorites = spellings.favorites(target["keeper_abs"])
|
||||
favorites = []
|
||||
for root, relative in spellings.favorites(target["abs"]).items():
|
||||
kept_root, kept_rel = (
|
||||
(root, kept_favorites[root])
|
||||
if root in kept_favorites
|
||||
else (target["keeper_root"], target["keeper_rel"])
|
||||
)
|
||||
favorites.append(
|
||||
(
|
||||
_favorite_root_key(root),
|
||||
_favorite_relative_key(relative),
|
||||
_favorite_root_key(kept_root),
|
||||
_favorite_relative_key(kept_rel),
|
||||
)
|
||||
)
|
||||
return {
|
||||
"removed": list(removed.values()),
|
||||
"kept": list(kept.values()),
|
||||
"targets": targets,
|
||||
"kept_path": target["keeper_abs"],
|
||||
"favorites": favorites,
|
||||
"removed_locator": target["locator"],
|
||||
"kept_locator": target["keeper_locator"],
|
||||
"sha256": target["hash"],
|
||||
"kept_size": kept_stat.st_size,
|
||||
"kept_mtime_ns": kept_stat.st_mtime_ns,
|
||||
}
|
||||
+42
-20
@@ -734,7 +734,10 @@ class HistoryRoutesMixin:
|
||||
raise HistoryReadError(422, "response-limit") from None
|
||||
else:
|
||||
prepared = _prepare_current_media(
|
||||
resolved, scope, thumbnail=kind == "thumbnail"
|
||||
resolved,
|
||||
scope,
|
||||
thumbnail=kind == "thumbnail",
|
||||
**media_options,
|
||||
)
|
||||
_check_current(scope, self, fence, epoch, generation)
|
||||
return prepared
|
||||
@@ -761,25 +764,44 @@ class HistoryRoutesMixin:
|
||||
_check_current(scope, self, fence, epoch, generation)
|
||||
raise
|
||||
|
||||
pending = asyncio.create_task(self._run_in_executor(project))
|
||||
try:
|
||||
prepared = await asyncio.shield(pending)
|
||||
except asyncio.CancelledError:
|
||||
pending.add_done_callback(_close_prepared_result)
|
||||
raise
|
||||
except (
|
||||
HistoryReadError,
|
||||
HistoryEpochChanged,
|
||||
sqlite3.Error,
|
||||
OSError,
|
||||
ValueError,
|
||||
):
|
||||
raise
|
||||
except Exception as error:
|
||||
self.logger.warning(
|
||||
"Current history media unavailable (%s)", type(error).__name__
|
||||
)
|
||||
raise _current_unavailable("temporary_unavailable") from None
|
||||
async def execute():
|
||||
pending = asyncio.create_task(self._run_in_executor(project))
|
||||
try:
|
||||
return await asyncio.shield(pending)
|
||||
except asyncio.CancelledError:
|
||||
pending.add_done_callback(_close_prepared_result)
|
||||
raise
|
||||
except (
|
||||
HistoryReadError,
|
||||
HistoryEpochChanged,
|
||||
sqlite3.Error,
|
||||
OSError,
|
||||
ValueError,
|
||||
):
|
||||
raise
|
||||
except Exception as error:
|
||||
self.logger.warning(
|
||||
"Current history media unavailable (%s)", type(error).__name__
|
||||
)
|
||||
raise _current_unavailable("temporary_unavailable") from None
|
||||
|
||||
# A missing preview (#249) is noted on the first read, made through
|
||||
# the serve route's admission and single flight, then read again. A
|
||||
# video whose preview cannot be made is the placeholder frame.
|
||||
needed = []
|
||||
media_options = {}
|
||||
if kind == "thumbnail":
|
||||
media_options = {
|
||||
"ensure_preview": lambda *preview: needed.append(preview),
|
||||
"placeholder": self._video_placeholder_bytes(),
|
||||
}
|
||||
prepared = await execute()
|
||||
if needed:
|
||||
if not isinstance(prepared, bytes):
|
||||
prepared.close()
|
||||
await self._admit_preview(*needed[0])
|
||||
media_options["ensure_preview"] = None
|
||||
prepared = await execute()
|
||||
if isinstance(prepared, bytes):
|
||||
return _success_response(prepared)
|
||||
return await _current_media_response(request, prepared)
|
||||
|
||||
@@ -5,6 +5,7 @@ from dataclasses import dataclass
|
||||
import errno
|
||||
import hashlib
|
||||
import hmac
|
||||
import io
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
@@ -309,17 +310,35 @@ class _OpenedCurrentMedia:
|
||||
self.handle.close()
|
||||
|
||||
|
||||
def _preview_writable(root: Path, preview: Path) -> bool:
|
||||
"""A preview may be written: ``thumbnails`` and every link segment of the
|
||||
preview's folder resolve inside *root* (the serve route's rule, #249)."""
|
||||
folder = root / "thumbnails"
|
||||
return _canonical(folder).is_relative_to(_canonical(root)) and _canonical(
|
||||
preview
|
||||
).is_relative_to(_canonical(folder))
|
||||
|
||||
|
||||
def _prepare_current_media(
|
||||
resolved: ResolvedCurrentCandidate, scope: CurrentOutputScope, *, thumbnail: bool
|
||||
resolved: ResolvedCurrentCandidate,
|
||||
scope: CurrentOutputScope,
|
||||
*,
|
||||
thumbnail: bool,
|
||||
ensure_preview=None,
|
||||
placeholder: bytes | None = None,
|
||||
) -> _OpenedCurrentMedia:
|
||||
"""Open the current file, or its preview for ``thumbnail``.
|
||||
|
||||
A missing preview that may be written is handed to
|
||||
``ensure_preview(src, dst, is_video)`` (the route makes it and reads
|
||||
again, #249). Without one an image's original is opened and a video is
|
||||
the ``placeholder`` JPEG, never the video inside an ``<img>``.
|
||||
"""
|
||||
target = resolved.canonical_target
|
||||
if thumbnail:
|
||||
relative = target.relative_to(resolved.gallery_root)
|
||||
extension = (
|
||||
".jpg"
|
||||
if target.suffix.lower() in VIDEO_EXTENSIONS
|
||||
else target.suffix.lower()
|
||||
)
|
||||
is_video = target.suffix.lower() in VIDEO_EXTENSIONS
|
||||
extension = ".jpg" if is_video else target.suffix.lower()
|
||||
preview = (
|
||||
resolved.gallery_root
|
||||
/ "thumbnails"
|
||||
@@ -328,7 +347,14 @@ def _prepare_current_media(
|
||||
try:
|
||||
preview.stat()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
if ensure_preview is not None and _preview_writable(
|
||||
resolved.gallery_root, preview
|
||||
):
|
||||
ensure_preview(target, preview, is_video)
|
||||
if is_video and placeholder is not None:
|
||||
return _OpenedCurrentMedia(
|
||||
io.BytesIO(placeholder), len(placeholder), "image/jpeg"
|
||||
)
|
||||
else:
|
||||
target = _canonical(preview)
|
||||
handle = _open_current_target(target, resolved.namespace, scope)
|
||||
|
||||
@@ -0,0 +1,509 @@
|
||||
"""Bulk image tagging as an Activity job: Add tags and Auto-tag (#249 D1).
|
||||
|
||||
``POST /prompt_manager/jobs/image-tags`` starts one ``image-tags`` job over
|
||||
1..500 listed images, named by the listing identity ``{root_index,
|
||||
root_scope, path}``. The job is record-changing (it writes image tag sets and
|
||||
new vocabulary names), so it never runs beside the metadata scan.
|
||||
|
||||
- **Add tags** (``mode: "add"``) hashes each current file and unions a fixed
|
||||
tag set into the record of exactly those bytes.
|
||||
- **Auto-tag** (``mode: "auto"``) copies each still image into a private
|
||||
capture, asks one legacy engine for its top N tags on the capture, re-hashes
|
||||
the current file and unions the suggestions only when the bytes are still
|
||||
the captured ones. It holds the shared model gate for the whole job, loads
|
||||
the model once and never downloads one.
|
||||
|
||||
Every step reuses the Tag dialog's pieces: the capture store, the model gate,
|
||||
the suggestion worker and the storage call ``add_image_tags``. Files are
|
||||
handled one at a time and the job stops at a file boundary when cancelled,
|
||||
keeping what it already wrote. Rows and log lines carry root-relative paths
|
||||
and public root labels only.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .. import model_gate
|
||||
from .admin import _plural, _time_left
|
||||
from .annotation_suggest import (
|
||||
OptionError,
|
||||
unavailable_message,
|
||||
validate_options,
|
||||
)
|
||||
from .annotations import (
|
||||
_MESSAGES,
|
||||
MAX_TARGET_PATH,
|
||||
_coded,
|
||||
_error,
|
||||
_pinned_suggestions,
|
||||
_settled,
|
||||
_submit,
|
||||
_target,
|
||||
)
|
||||
from .autotag_routes import model_busy_response
|
||||
from .images import _public_path
|
||||
from .job_routes import busy_response
|
||||
from .jobs import IMAGE_TAG_GROUP_NAMES, IMAGE_TAG_JOB_KIND, JobBusy
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
from ...database.annotations import ImageTagLimitError, normalize_image_tags
|
||||
from ...database.library_commands import LibraryCommandError
|
||||
from ...utils.media_capture import (
|
||||
CaptureCapacity,
|
||||
CaptureChanged,
|
||||
CaptureDenied,
|
||||
CaptureError,
|
||||
CaptureTooLarge,
|
||||
)
|
||||
from ...utils.video_metadata import is_video_path
|
||||
except ImportError:
|
||||
from database.annotations import ImageTagLimitError, normalize_image_tags
|
||||
from database.library_commands import LibraryCommandError
|
||||
from utils.media_capture import (
|
||||
CaptureCapacity,
|
||||
CaptureChanged,
|
||||
CaptureDenied,
|
||||
CaptureError,
|
||||
CaptureTooLarge,
|
||||
)
|
||||
from utils.video_metadata import is_video_path
|
||||
|
||||
MAX_JOB_TARGETS = 500
|
||||
# 500 maximal targets plus the options: a path of MAX_TARGET_PATH code points
|
||||
# takes at most 6 bytes each on the wire (a \uXXXX escape; raw UTF-8 is at most
|
||||
# 4), and the root index and scope fit in 256. The annotation routes' 64 KiB
|
||||
# bound is for one target, too small for a full selection.
|
||||
MAX_JOB_BODY_BYTES = MAX_JOB_TARGETS * (MAX_TARGET_PATH * 6 + 256) + 64 * 1024
|
||||
# A busy capture worker (a Tag dialog in another tab) is waited for this long,
|
||||
# try by try, before the file is left for a rerun.
|
||||
CAPACITY_BACKOFF = (0.1, 0.2, 0.4, 0.8)
|
||||
|
||||
ADD_TITLE = "Add tags to images"
|
||||
AUTO_TITLE = "Auto-tag images"
|
||||
GATE_OWNER_KIND = "image-tag-job"
|
||||
|
||||
_FIELDS = {
|
||||
"add": frozenset({"mode", "targets", "tags"}),
|
||||
"auto": frozenset({"mode", "targets", "model_type", "options"}),
|
||||
}
|
||||
_VERBS = {"add": "Add tags", "auto": "Auto-tag"}
|
||||
_LOG_PREFIXES = {"tagged": "+", "unchanged": "=", "skipped": "-", "errors": "!"}
|
||||
|
||||
NO_NEW_TAGS = "No new tags"
|
||||
VIDEO_SKIPPED = "Video: manual tags only"
|
||||
TOO_LARGE = "Too large to verify"
|
||||
FILE_CHANGED = "File changed"
|
||||
FILE_NOT_FOUND = "File not found"
|
||||
NOT_A_STILL = "Not a supported still image"
|
||||
INFERENCE_FAILED = "Auto-tagging failed. Check server logs for details."
|
||||
SAVE_FAILED = "The tags could not be saved. Check server logs for details."
|
||||
READ_FAILED = "The file could not be read."
|
||||
SOURCE_CHANGED = "The source changed. Refresh the image collection."
|
||||
STORAGE_RESTORED = "Prompt storage was restored; start the job again."
|
||||
BUSY_RETRY = "The file could not be read now. Run the job again for it."
|
||||
|
||||
_monotonic = time.monotonic # the job's clock (a test can fake it)
|
||||
|
||||
|
||||
class _JobStopped(Exception):
|
||||
"""No later file can be written (prompt storage was restored)."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Run:
|
||||
"""One running job: its request, the gate owner and the resolved roots.
|
||||
|
||||
``roots`` maps each target's ``(root_index, root_scope)`` to ``(root,
|
||||
label)``, read once per job off the event loop; ``root`` is None when the
|
||||
scope no longer names a current root.
|
||||
"""
|
||||
|
||||
job: object
|
||||
spec: object
|
||||
epoch: str
|
||||
owner: object
|
||||
service: object = None
|
||||
roots: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Spec:
|
||||
"""A validated start request."""
|
||||
|
||||
mode: str
|
||||
targets: tuple
|
||||
tags: tuple = ()
|
||||
model_type: str | None = None
|
||||
options: dict | None = None
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return ADD_TITLE if self.mode == "add" else AUTO_TITLE
|
||||
|
||||
def scope(self) -> dict:
|
||||
scope = {"mode": self.mode, "images": len(self.targets)}
|
||||
if self.mode == "auto":
|
||||
scope["model_type"] = self.model_type
|
||||
return scope
|
||||
|
||||
|
||||
def _parse_targets(raw):
|
||||
"""Unique validated targets in request order, or ``(None, error)``."""
|
||||
if not isinstance(raw, list) or not 1 <= len(raw) <= MAX_JOB_TARGETS:
|
||||
return None, bad_request(f"targets must list 1 to {MAX_JOB_TARGETS} images")
|
||||
seen, targets = set(), []
|
||||
for item in raw:
|
||||
target, error = _target(item)
|
||||
if error is not None:
|
||||
return None, error
|
||||
if target not in seen:
|
||||
seen.add(target)
|
||||
targets.append(target)
|
||||
return tuple(targets), None
|
||||
|
||||
|
||||
def _parse_add(targets, data):
|
||||
try:
|
||||
tags = normalize_image_tags(data["tags"])
|
||||
except ImageTagLimitError as exc:
|
||||
return None, _error(400, "tag-limit", str(exc))
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
if not tags:
|
||||
return None, bad_request("tags must name at least one tag")
|
||||
return _Spec("add", targets, tags=tuple(tags)), None
|
||||
|
||||
|
||||
def _parse_auto(targets, data):
|
||||
from ..config import MAX_SUGGESTION_LIMIT, AutotagConfig
|
||||
|
||||
try:
|
||||
model_type, options = validate_options(
|
||||
data["model_type"],
|
||||
data["options"],
|
||||
default_limit=AutotagConfig.suggestion_limit(),
|
||||
max_limit=MAX_SUGGESTION_LIMIT,
|
||||
)
|
||||
except OptionError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
return _Spec("auto", targets, model_type=model_type, options=options), None
|
||||
|
||||
|
||||
def parse_image_tag_request(data: dict):
|
||||
"""A :class:`_Spec` for a start body, or ``(None, error response)``.
|
||||
|
||||
``{mode:"add", targets, tags}`` or ``{mode:"auto", targets, model_type,
|
||||
options}`` exactly; options follow the Tag dialog's Generate rules.
|
||||
"""
|
||||
fields = _FIELDS.get(data.get("mode"))
|
||||
if fields is None:
|
||||
return None, bad_request('mode must be "add" or "auto"')
|
||||
if set(data) != fields:
|
||||
return None, bad_request(
|
||||
"Request must contain exactly " + ", ".join(sorted(fields))
|
||||
)
|
||||
targets, error = _parse_targets(data["targets"])
|
||||
if error is not None:
|
||||
return None, error
|
||||
if data["mode"] == "add":
|
||||
return _parse_add(targets, data)
|
||||
return _parse_auto(targets, data)
|
||||
|
||||
|
||||
def _capture_outcome(exc: CaptureError, root, relative: str):
|
||||
"""The (group, detail) of a file the capture store refused to read."""
|
||||
if isinstance(exc, CaptureTooLarge):
|
||||
return "skipped", TOO_LARGE
|
||||
if isinstance(exc, CaptureDenied):
|
||||
return "skipped", FILE_NOT_FOUND
|
||||
if isinstance(exc, CaptureChanged):
|
||||
path = os.path.join(root, *relative.split("/"))
|
||||
return "skipped", FILE_CHANGED if os.path.lexists(path) else FILE_NOT_FOUND
|
||||
if isinstance(exc, CaptureCapacity):
|
||||
return "skipped", BUSY_RETRY
|
||||
return "errors", READ_FAILED
|
||||
|
||||
|
||||
async def _capacity_retried(job, attempt):
|
||||
"""``await attempt()``, tried again while the capture store is at capacity.
|
||||
|
||||
Raises the last ``CaptureCapacity`` once ``CAPACITY_BACKOFF`` is spent or the
|
||||
job was cancelled meanwhile.
|
||||
"""
|
||||
for delay in CAPACITY_BACKOFF:
|
||||
try:
|
||||
return await attempt()
|
||||
except CaptureCapacity:
|
||||
if job.cancelled:
|
||||
raise
|
||||
await asyncio.sleep(delay)
|
||||
return await attempt()
|
||||
|
||||
|
||||
def _load_model(service, model_type: str, use_gpu: bool) -> None:
|
||||
"""Worker thread: load (or swap to) *model_type*; never downloads."""
|
||||
if service.is_model_loaded() and service.get_loaded_model_type() == model_type:
|
||||
return
|
||||
if service.load_model(model_type, use_gpu) is False:
|
||||
raise RuntimeError(f"{model_type} could not be loaded")
|
||||
|
||||
|
||||
class ImageTagJobsMixin:
|
||||
"""The ``image-tags`` job kind: start route and runner."""
|
||||
|
||||
async def start_image_tag_job(self, request: web.Request) -> web.Response:
|
||||
"""Start bulk tagging (202), or 409 ``busy`` / ``model-busy``.
|
||||
|
||||
Every target's root scope is checked before the job starts
|
||||
(``source-changed``). Auto-tag takes the model gate here, so a start
|
||||
while anything else holds it is refused at once.
|
||||
"""
|
||||
data, error = await self._read_json_body(request, max_bytes=MAX_JOB_BODY_BYTES)
|
||||
if error is not None:
|
||||
return error
|
||||
spec, error = parse_image_tag_request(data)
|
||||
if error is not None:
|
||||
return error
|
||||
roots = await self._run_in_executor(self._job_roots, spec.targets)
|
||||
if any(root is None for root, _label in roots.values()):
|
||||
return _coded("source-changed")
|
||||
epoch = await self._run_in_executor(self.db.get_execution_epoch)
|
||||
# A running job in the way answers first: it says which job to wait for.
|
||||
blocker = self._jobs.blocker(IMAGE_TAG_JOB_KIND, record_changing=True)
|
||||
if blocker is not None:
|
||||
return busy_response(JobBusy(blocker))
|
||||
owner = None
|
||||
if spec.mode == "auto":
|
||||
owner = model_gate.new_owner(GATE_OWNER_KIND)
|
||||
if not model_gate.try_acquire(owner):
|
||||
return model_busy_response()
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
IMAGE_TAG_JOB_KIND,
|
||||
spec.title,
|
||||
record_changing=True,
|
||||
scope=spec.scope(),
|
||||
runner=functools.partial(self._run_image_tag_job, spec, epoch, owner),
|
||||
groups=IMAGE_TAG_GROUP_NAMES,
|
||||
)
|
||||
except JobBusy as busy:
|
||||
if owner is not None:
|
||||
model_gate.release(owner)
|
||||
return busy_response(busy)
|
||||
if owner is not None:
|
||||
# Also when the task is cancelled before its runner ever starts.
|
||||
job.task.add_done_callback(lambda _task: model_gate.release(owner))
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
# -- runner ----------------------------------------------------------------
|
||||
|
||||
def _job_roots(self, targets) -> dict:
|
||||
"""Worker thread: ``{(index, scope): (root or None, label)}`` (see _Run)."""
|
||||
roots = list(self._get_all_output_dirs())
|
||||
resolved = {}
|
||||
for index, scope in dict.fromkeys((t[0], t[1]) for t in targets):
|
||||
label = _public_path(roots[index]) if index < len(roots) else ""
|
||||
error = self._gallery_scope_error(
|
||||
{"root_scope": scope, "root": str(index)}, roots
|
||||
)
|
||||
resolved[(index, scope)] = (roots[index] if error is None else None, label)
|
||||
return resolved
|
||||
|
||||
async def _run_image_tag_job(self, spec: _Spec, epoch: str, owner, job) -> None:
|
||||
verb = _VERBS[spec.mode]
|
||||
total = len(spec.targets)
|
||||
job.update(total=total, status_line=f"Processing file 0 of {total:,}")
|
||||
started = f"{verb} started · {_plural(total, 'image')}"
|
||||
if spec.mode == "auto":
|
||||
started += f" · {spec.model_type}"
|
||||
job.log_line(started)
|
||||
run = _Run(job, spec, epoch, owner)
|
||||
try:
|
||||
if spec.mode == "auto":
|
||||
run.service = await self._prepare_tagging_model(job, spec, owner)
|
||||
if run.service is None:
|
||||
job.finish("failed")
|
||||
return
|
||||
if not await self._tag_each_file(run):
|
||||
job.log_line(f"{verb} stopped · {STORAGE_RESTORED}")
|
||||
job.finish("failed")
|
||||
return
|
||||
state = "cancelled" if job.cancelled else "finished"
|
||||
job.log_line(f"{verb} {state} · {_plural(job.processed, 'file')} processed")
|
||||
job.publish({"type": state, "progress": job.progress})
|
||||
job.finish(state)
|
||||
finally:
|
||||
if owner is not None:
|
||||
model_gate.release(owner)
|
||||
|
||||
async def _tag_each_file(self, run: _Run) -> bool:
|
||||
"""Tag the targets in order; False when the job had to stop."""
|
||||
job, total = run.job, len(run.spec.targets)
|
||||
run.roots = await self._run_in_executor(self._job_roots, run.spec.targets)
|
||||
started = _monotonic()
|
||||
for number, target in enumerate(run.spec.targets, 1):
|
||||
if job.cancelled:
|
||||
break
|
||||
# The scan's estimate (D56), with each finished file as one batch.
|
||||
left = _time_left(number - 1, total, number - 1, _monotonic() - started)
|
||||
job.update(status_line=f"Processing file {number:,} of {total:,}{left}")
|
||||
job.publish({"type": "progress", "progress": job.progress})
|
||||
stopped = False
|
||||
try:
|
||||
group, detail = await self._tag_one_file(run, target)
|
||||
except _JobStopped:
|
||||
stopped, group, detail = True, "errors", STORAGE_RESTORED
|
||||
except Exception:
|
||||
self.logger.exception("Image tagging failed for one file")
|
||||
group, detail = "errors", SAVE_FAILED
|
||||
self._record_tag_outcome(run, target, group, detail)
|
||||
job.update(processed=number, progress=number * 100 // total)
|
||||
if stopped:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _prepare_tagging_model(self, job, spec: _Spec, owner):
|
||||
"""The loaded service, or None (logged) when the job cannot run."""
|
||||
from ..autotag import MODELS, get_autotag_service
|
||||
|
||||
service = get_autotag_service()
|
||||
status = await self._run_in_executor(service.get_models_status)
|
||||
if not status.get(spec.model_type, {}).get("downloaded"):
|
||||
name = MODELS.get(spec.model_type, {}).get("name", spec.model_type)
|
||||
message = unavailable_message(name)
|
||||
job.update(status_line=message)
|
||||
job.log_line(f"! {message}")
|
||||
return None
|
||||
try:
|
||||
await model_gate.run(
|
||||
owner,
|
||||
_load_model,
|
||||
service,
|
||||
spec.model_type,
|
||||
spec.options["use_gpu"],
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("The auto-tag job could not load its model")
|
||||
job.update(status_line=INFERENCE_FAILED)
|
||||
job.log_line(f"! {INFERENCE_FAILED}")
|
||||
return None
|
||||
return service
|
||||
|
||||
async def _verified(self, run: _Run, store, root, relative):
|
||||
"""The current file's facts, waiting out a busy capture store."""
|
||||
return await _capacity_retried(
|
||||
run.job, lambda: self._run_in_executor(store.verify, root, relative)
|
||||
)
|
||||
|
||||
async def _tag_one_file(self, run: _Run, target):
|
||||
"""The (group, detail) of one target; raises _JobStopped on a restore."""
|
||||
index, scope, relative = target
|
||||
root, _label = run.roots[(index, scope)]
|
||||
if root is None:
|
||||
return "errors", SOURCE_CHANGED
|
||||
store = self._capture_store()
|
||||
if run.spec.mode == "add":
|
||||
try:
|
||||
facts = await self._verified(run, store, root, relative)
|
||||
except CaptureError as exc:
|
||||
return _capture_outcome(exc, root, relative)
|
||||
return await self._union_image_tags(facts, list(run.spec.tags), run.epoch)
|
||||
if is_video_path(relative):
|
||||
return "skipped", VIDEO_SKIPPED
|
||||
try:
|
||||
capture = await _capacity_retried(
|
||||
run.job,
|
||||
lambda: _settled(
|
||||
_submit(
|
||||
store.capture,
|
||||
root,
|
||||
relative,
|
||||
epoch=run.epoch,
|
||||
scope=scope,
|
||||
root_index=index,
|
||||
),
|
||||
lambda made: made is not None and store.release(made.token),
|
||||
),
|
||||
)
|
||||
except CaptureError as exc:
|
||||
return _capture_outcome(exc, root, relative)
|
||||
try:
|
||||
return await self._auto_tag_capture(run, store, capture, root, relative)
|
||||
finally:
|
||||
store.release(capture.token)
|
||||
|
||||
async def _auto_tag_capture(self, run: _Run, store, capture, root, relative):
|
||||
"""Suggest on the capture, then union only onto the same current bytes."""
|
||||
if not capture.inferable:
|
||||
return "skipped", NOT_A_STILL
|
||||
if store.acquire(capture.token) is None:
|
||||
raise _JobStopped # every capture was invalidated (restore)
|
||||
spec = run.spec
|
||||
try:
|
||||
pairs = await model_gate.run(
|
||||
run.owner,
|
||||
_pinned_suggestions,
|
||||
store,
|
||||
capture.token,
|
||||
run.service,
|
||||
capture,
|
||||
spec.model_type,
|
||||
spec.options,
|
||||
on_skipped=functools.partial(store.release_use, capture.token),
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Auto-tagging failed for one file")
|
||||
return "errors", INFERENCE_FAILED
|
||||
try:
|
||||
facts = await self._verified(run, store, root, relative)
|
||||
except CaptureError as exc:
|
||||
return _capture_outcome(exc, root, relative)
|
||||
if facts.locator != capture.locator or facts.sha256 != capture.sha256:
|
||||
return "skipped", FILE_CHANGED
|
||||
names = [str(name) for name, _score in pairs]
|
||||
if not names:
|
||||
return "unchanged", NO_NEW_TAGS
|
||||
return await self._union_image_tags(facts, names, run.epoch)
|
||||
|
||||
async def _union_image_tags(self, facts, tags: list, epoch: str):
|
||||
"""``add_image_tags`` on the verified bytes, as a (group, detail)."""
|
||||
try:
|
||||
receipt = await self._run_in_executor(
|
||||
self.db.add_image_tags,
|
||||
locator=facts.locator,
|
||||
sha256=facts.sha256,
|
||||
size=facts.size,
|
||||
mtime_ns=facts.mtime_ns,
|
||||
expected_epoch=epoch,
|
||||
tags=tags,
|
||||
)
|
||||
except LibraryCommandError as exc:
|
||||
if exc.code == "storage-changed":
|
||||
raise _JobStopped from None
|
||||
if exc.code == "tag-limit":
|
||||
return "errors", _MESSAGES["tag-limit"]
|
||||
self.logger.warning(f"Image tags not saved ({exc.code})")
|
||||
return "errors", SAVE_FAILED
|
||||
except ValueError:
|
||||
self.logger.exception("Image tags were refused")
|
||||
return "errors", SAVE_FAILED
|
||||
added = receipt.get("added", 0)
|
||||
if added:
|
||||
return "tagged", f"Added {_plural(added, 'tag')}"
|
||||
return "unchanged", NO_NEW_TAGS
|
||||
|
||||
def _record_tag_outcome(self, run: _Run, target, group: str, detail: str) -> None:
|
||||
index, scope, relative = target
|
||||
_root, label = run.roots[(index, scope)]
|
||||
where = f"{label}/{relative}" if label else relative
|
||||
run.job.record(
|
||||
group,
|
||||
{"path": relative, "root": label, "root_index": index, "detail": detail},
|
||||
f"{_LOG_PREFIXES[group]} {where} · {detail}",
|
||||
)
|
||||
+643
-303
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,7 @@
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
try:
|
||||
from ...utils.prompt_inspector import (
|
||||
@@ -93,8 +94,19 @@ def _in_host_temp(path: Path) -> bool:
|
||||
|
||||
|
||||
def inspect_current_file(
|
||||
path: Path, *, source_label: str, saved: dict | None, library: dict
|
||||
path: Path,
|
||||
*,
|
||||
source_label: str,
|
||||
saved: dict | None,
|
||||
library: dict,
|
||||
image_tags: Callable[[int, int], dict] | None = None,
|
||||
) -> dict:
|
||||
"""Current-file evidence read between two equal stats of *path*.
|
||||
|
||||
*image_tags*, when given, is called once with that stable
|
||||
``(st_size, st_mtime_ns)`` and its facet is returned as ``image_tags``
|
||||
(spec amendment S1: tags of a byte record verified at exactly that stat).
|
||||
"""
|
||||
before = _current_file_facts(path)
|
||||
try:
|
||||
inspection = read_inspection_file(path)
|
||||
@@ -104,6 +116,9 @@ def inspect_current_file(
|
||||
fields = ("st_dev", "st_ino", "st_mode", "st_size", "st_mtime_ns", "st_ctime_ns")
|
||||
if any(getattr(before, key) != getattr(after, key) for key in fields):
|
||||
raise CurrentFileChanged
|
||||
facets = {}
|
||||
if image_tags is not None:
|
||||
facets["image_tags"] = image_tags(before.st_size, before.st_mtime_ns)
|
||||
if inspection is None:
|
||||
evidence = unreadable_evidence()
|
||||
else:
|
||||
@@ -132,4 +147,5 @@ def inspect_current_file(
|
||||
**evidence,
|
||||
"inspection": inspection,
|
||||
"library": library,
|
||||
**facets,
|
||||
}
|
||||
|
||||
+160
-17
@@ -1,8 +1,10 @@
|
||||
"""Session job routes (Activity): list, inspect, follow and cancel jobs.
|
||||
|
||||
The registry itself is ``py/api/jobs.py``; the scan job's runner is in
|
||||
``py/api/admin.py``. Every response leaves out absolute server paths: job
|
||||
rows already carry root-relative paths and public root labels.
|
||||
The registry itself is ``py/api/jobs.py``. The scan job's runner is in
|
||||
``py/api/admin.py``, the image tagging job's in ``py/api/image_tag_jobs.py``
|
||||
and the thumbnail job's in ``py/api/thumbnail_jobs.py``.
|
||||
Every response leaves out absolute server paths: job rows already carry
|
||||
root-relative paths and public root labels.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
@@ -10,13 +12,97 @@ import json
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .jobs import JobBusy
|
||||
from .jobs import DEFAULT_SCAN_SCOPE, SCAN_SCOPES, JobBusy
|
||||
|
||||
|
||||
def _not_found():
|
||||
return web.json_response({"success": False, "error": "Job not found"}, status=404)
|
||||
|
||||
|
||||
MAX_SCAN_ROOTS = 64
|
||||
_SCOPE_ERROR = 'The scan scope must be "new" or "all"'
|
||||
_ROOTS_ERROR = (
|
||||
f"roots must be a non-empty list of up to {MAX_SCAN_ROOTS} source indexes, "
|
||||
"with paths naming the public path of each"
|
||||
)
|
||||
SOURCES_CHANGED_ERROR = "The sources changed since they were loaded. Reload them."
|
||||
|
||||
|
||||
def _bad_request(message, code=None):
|
||||
body = {"success": False, "error": message}
|
||||
if code is not None:
|
||||
body["code"] = code
|
||||
return web.json_response(body, status=400)
|
||||
|
||||
|
||||
def _scan_roots(body):
|
||||
"""``(roots, error)``: ``{index: public path}`` of the selected sources, or None.
|
||||
|
||||
Absent ``roots`` means every source. When present it is a non-empty list
|
||||
of at most MAX_SCAN_ROOTS non-negative ints (never bools or floats), and
|
||||
``paths`` the same-length list of the public path the caller read for
|
||||
each (GET /settings), so a stale index never scans another folder.
|
||||
"""
|
||||
if "roots" not in body:
|
||||
return None, None
|
||||
raw = body["roots"]
|
||||
paths = body.get("paths")
|
||||
if not isinstance(raw, list) or not raw or len(raw) > MAX_SCAN_ROOTS:
|
||||
return None, _ROOTS_ERROR
|
||||
if not isinstance(paths, list) or len(paths) != len(raw):
|
||||
return None, _ROOTS_ERROR
|
||||
selected = {}
|
||||
for value, path in zip(raw, paths):
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||
return None, _ROOTS_ERROR
|
||||
if not isinstance(path, str) or selected.get(value, path) != path:
|
||||
return None, _ROOTS_ERROR
|
||||
selected[value] = path
|
||||
return dict(sorted(selected.items())), None
|
||||
|
||||
|
||||
async def _scan_request(request):
|
||||
"""``({files, roots}, None)`` for a scan start, or ``(None, message)``.
|
||||
|
||||
``files`` is the scope ("new" or "all"); ``roots`` the selected source
|
||||
indexes or None for every source. An empty body (also a chunked one with
|
||||
no bytes), or one without ``scope``, asks for the default scope.
|
||||
"""
|
||||
default = {"files": DEFAULT_SCAN_SCOPE, "roots": None}
|
||||
if not request.can_read_body:
|
||||
return default, None
|
||||
text = await request.text()
|
||||
if not text.strip(): # e.g. a chunked POST that sent no bytes
|
||||
return default, None
|
||||
try:
|
||||
body = json.loads(text)
|
||||
except ValueError:
|
||||
return None, _SCOPE_ERROR
|
||||
if not isinstance(body, dict):
|
||||
return None, _SCOPE_ERROR
|
||||
files = body.get("scope", DEFAULT_SCAN_SCOPE)
|
||||
if files not in SCAN_SCOPES:
|
||||
return None, _SCOPE_ERROR
|
||||
roots, error = _scan_roots(body)
|
||||
if error:
|
||||
return None, error
|
||||
return {"files": files, "roots": roots}, None
|
||||
|
||||
|
||||
def busy_response(busy: JobBusy):
|
||||
"""``409 busy``, naming the running job that holds the slot."""
|
||||
running = busy.running
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "busy",
|
||||
"error": str(busy),
|
||||
"running": {"id": running.id, "title": running.title},
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
|
||||
|
||||
def _summary_event(job):
|
||||
return f"data: {json.dumps(job.summary())}\n\n".encode("utf-8")
|
||||
|
||||
@@ -24,6 +110,13 @@ def _summary_event(job):
|
||||
class JobRoutesMixin:
|
||||
"""Mixin providing the /prompt_manager/jobs endpoints."""
|
||||
|
||||
def _roots_still_listed(self, selected, output_dirs):
|
||||
"""Whether each ``{index: public path}`` still names that source."""
|
||||
return all(
|
||||
index < len(output_dirs) and self._public_path(output_dirs[index]) == path
|
||||
for index, path in selected.items()
|
||||
)
|
||||
|
||||
def _register_job_routes(self, routes):
|
||||
@routes.get("/prompt_manager/jobs")
|
||||
async def list_jobs_route(request):
|
||||
@@ -33,10 +126,22 @@ class JobRoutesMixin:
|
||||
async def start_scan_job_route(request):
|
||||
return await self.start_scan_job(request)
|
||||
|
||||
@routes.post("/prompt_manager/jobs/image-tags")
|
||||
async def start_image_tag_job_route(request):
|
||||
return await self.start_image_tag_job(request)
|
||||
|
||||
@routes.post("/prompt_manager/jobs/thumbnails")
|
||||
async def start_thumbnail_job_route(request):
|
||||
return await self.start_thumbnail_job(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}")
|
||||
async def job_detail_route(request):
|
||||
return await self.job_detail(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}/report")
|
||||
async def job_report_route(request):
|
||||
return await self.job_report(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}/events")
|
||||
async def job_events_route(request):
|
||||
return await self.job_events(request)
|
||||
@@ -56,6 +161,22 @@ class JobRoutesMixin:
|
||||
return _not_found()
|
||||
return web.json_response({"success": True, "job": job.detail()})
|
||||
|
||||
async def job_report(self, request):
|
||||
"""The job's whole record as a JSON attachment (no absolute paths)."""
|
||||
job = self._jobs.get(request.match_info["job_id"])
|
||||
if job is None:
|
||||
return _not_found()
|
||||
body = json.dumps(job.report(), indent=2)
|
||||
disposition = f'attachment; filename="{job.report_filename()}"'
|
||||
return web.Response(
|
||||
text=body,
|
||||
content_type="application/json",
|
||||
headers={
|
||||
"Content-Disposition": disposition,
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
)
|
||||
|
||||
async def cancel_job(self, request):
|
||||
"""Stop a job at its next file boundary; a finished job is a no-op."""
|
||||
job_id = request.match_info["job_id"]
|
||||
@@ -65,20 +186,42 @@ class JobRoutesMixin:
|
||||
return web.json_response({"success": True, "cancellation_requested": requested})
|
||||
|
||||
async def start_scan_job(self, request):
|
||||
"""Start the metadata scan as a job (202), or 409 naming the busy job."""
|
||||
try:
|
||||
job = self._start_scan_job()
|
||||
except JobBusy as busy:
|
||||
running = busy.running
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "busy",
|
||||
"error": str(busy),
|
||||
"running": {"id": running.id, "title": running.title},
|
||||
},
|
||||
status=409,
|
||||
"""Start the metadata scan as a job (202), or 409 naming the busy job.
|
||||
|
||||
The body's ``scope`` is "new" (the default: new and unresolved files)
|
||||
or "all". ``roots``, when given, selects sources by their index in
|
||||
the effective sources (``_get_all_output_dirs``), and ``paths`` names
|
||||
the public path read for each. An index past them, or one whose
|
||||
source now has another public path (the list changed since it was
|
||||
read), is a 409 ``sources-changed`` and starts nothing. The job walks
|
||||
the sources resolved here, so a later source change cannot move it.
|
||||
"""
|
||||
scan, error = await _scan_request(request)
|
||||
if error:
|
||||
return _bad_request(error)
|
||||
output_dirs = None
|
||||
roots = None
|
||||
if scan["roots"] is not None:
|
||||
output_dirs = await self._run_in_executor(self._get_all_output_dirs)
|
||||
listed = await self._run_in_executor(
|
||||
self._roots_still_listed, scan["roots"], output_dirs
|
||||
)
|
||||
if not listed:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "sources-changed",
|
||||
"error": SOURCES_CHANGED_ERROR,
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
roots = list(scan["roots"])
|
||||
try:
|
||||
job = self._start_scan_job(
|
||||
scan["files"], roots=roots, output_dirs=output_dirs
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
async def job_events(self, request):
|
||||
|
||||
+274
-29
@@ -9,8 +9,25 @@ as a snapshot and then every event published after it, until the job ends.
|
||||
|
||||
Jobs live only in server memory and are gone after a restart. The registry
|
||||
keeps the running jobs and this session's last ``MAX_FINISHED_JOBS`` finished
|
||||
ones. Record-changing jobs (the metadata scan, and later maintenance, cleanup
|
||||
and restore) never run together, and one job of a kind runs at a time.
|
||||
ones. One job of a kind runs at a time, and each job has an admission class
|
||||
(``ADMISSIONS``) that decides what may run beside it:
|
||||
|
||||
- ``records`` changes library records or files (the metadata scan, bulk image
|
||||
tagging, and the legacy restore, maintenance, cleanup, duplicate delete,
|
||||
LoRA import and batch auto-tag routes). It is refused while any
|
||||
``records`` or ``guarded`` job runs.
|
||||
- ``guarded`` must not overlap record changes but may overlap other guarded
|
||||
work (thumbnail generation and clearing, later backup). It is refused while
|
||||
a ``records`` job runs.
|
||||
- ``read`` only reads; it is refused only by a running job of its own kind
|
||||
or an ``exclusive`` one.
|
||||
- ``exclusive`` runs alone (Optimize's VACUUM, which rewrites the whole
|
||||
database under the write lock): it is refused while any job runs, and every
|
||||
job is refused while it runs.
|
||||
|
||||
A short legacy-page operation takes a ``hold``: a running, non-cancellable
|
||||
job that lasts as long as the request, so the Activity page lists it and a
|
||||
workspace job is refused while it runs (and the other way round).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -35,18 +52,78 @@ MAX_FINISHED_JOBS = 20 # finished jobs this session keeps
|
||||
# The metadata scan's outcome groups, in counter-tile order of the spec's
|
||||
# mapping, and the per-file scan status each group collects. Files without
|
||||
# usable metadata have no status and go to "no_metadata".
|
||||
# Which files a scan reads (#249 Activity-2): "new" skips the files the Library
|
||||
# already links with a resolved link (get_known_resolved_paths) before reading
|
||||
# them, so new, unlinked, unresolved and ambiguous files are read; "all" reads
|
||||
# every file again. The workspace job defaults to "new"; legacy /scan is "all".
|
||||
SCAN_SCOPES = ("new", "all")
|
||||
DEFAULT_SCAN_SCOPE = "new"
|
||||
SCAN_GROUP_NAMES = ("added", "known", "review", "no_metadata", "errors")
|
||||
SCAN_GROUPS = {
|
||||
"added": "added",
|
||||
"linked": "added",
|
||||
"enriched": "added",
|
||||
"unchanged": "known",
|
||||
"skipped": "known", # "new" scope: known and resolved, not read again
|
||||
"ambiguous": "review",
|
||||
"conflict": "review",
|
||||
"unresolved": "review",
|
||||
"failed": "errors",
|
||||
}
|
||||
|
||||
# Bulk image tagging (#249 D1): Add tags and Auto-tag over selected images.
|
||||
# Record-changing: it writes image tag sets and new vocabulary names.
|
||||
IMAGE_TAG_JOB_KIND = "image-tags"
|
||||
IMAGE_TAG_GROUP_NAMES = ("tagged", "unchanged", "skipped", "errors")
|
||||
|
||||
# Thumbnail generation (#249 System S5): previews written, already fresh, failed.
|
||||
THUMBNAIL_GROUP_NAMES = ("generated", "skipped", "errors")
|
||||
|
||||
ADMISSIONS = ("read", "guarded", "records", "exclusive")
|
||||
# What each admission class is refused beside (besides its own kind).
|
||||
# ``exclusive`` (Optimize's VACUUM) runs alone: beside no job of any class.
|
||||
_REFUSED_BESIDE = {
|
||||
"read": frozenset({"exclusive"}),
|
||||
"guarded": frozenset({"records", "exclusive"}),
|
||||
"records": frozenset({"records", "guarded", "exclusive"}),
|
||||
"exclusive": frozenset(ADMISSIONS),
|
||||
}
|
||||
|
||||
# Kinds of the legacy-page operations that hold the registry (D80; titles in
|
||||
# their handlers end in "(admin page)", "(gallery page)" or "(legacy page)").
|
||||
RESTORE_JOB_KIND = "restore"
|
||||
MAINTENANCE_JOB_KIND = "maintenance"
|
||||
DUPLICATE_CLEANUP_JOB_KIND = "duplicate_cleanup"
|
||||
THUMBNAILS_JOB_KIND = "thumbnails"
|
||||
LORA_IMPORT_JOB_KIND = "lora_import"
|
||||
AUTOTAG_BATCH_JOB_KIND = "autotag"
|
||||
# Duplicate image files (#249 System S6, py/api/duplicate_files.py): the
|
||||
# find is a cancellable ``read`` job; the reviewed removal is ``records``.
|
||||
DUPLICATES_JOB_KIND = "duplicates"
|
||||
DUPLICATE_GROUP_NAMES = ("groups_found",)
|
||||
DUPLICATE_CLEANUP_GROUP_NAMES = ("removed", "kept", "errors")
|
||||
# The System health check (py/api/system.py): a read-only ``read`` job with
|
||||
# no file outcomes, so it may run beside a scan.
|
||||
HEALTH_JOB_KIND = "health"
|
||||
|
||||
|
||||
def resolve_admission(admission=None, record_changing=None) -> str:
|
||||
"""The admission class for a start call; ``record_changing`` is an alias.
|
||||
|
||||
``record_changing=True`` means ``records`` and ``False`` means ``read``;
|
||||
with neither given the class is ``read``. Raises ValueError for an
|
||||
unknown class or one that contradicts ``record_changing``.
|
||||
"""
|
||||
if admission is None:
|
||||
return "records" if record_changing else "read"
|
||||
if admission not in ADMISSIONS:
|
||||
raise ValueError(f"Unknown job admission {admission!r}")
|
||||
if record_changing is not None and record_changing != (admission == "records"):
|
||||
raise ValueError(
|
||||
f"record_changing={record_changing!r} contradicts admission {admission!r}"
|
||||
)
|
||||
return admission
|
||||
|
||||
|
||||
def _relative_path(path) -> str:
|
||||
"""``path`` as a root-relative POSIX path; an absolute one keeps its name."""
|
||||
@@ -72,13 +149,23 @@ class Job:
|
||||
"""One tracked operation: state, counters, outcomes, log and subscribers."""
|
||||
|
||||
def __init__(
|
||||
self, kind, title, *, record_changing=False, scope=None, groups=(), clock=None
|
||||
self,
|
||||
kind,
|
||||
title,
|
||||
*,
|
||||
admission=None,
|
||||
record_changing=None,
|
||||
cancellable=True,
|
||||
scope=None,
|
||||
groups=(),
|
||||
clock=None,
|
||||
):
|
||||
self._clock = clock or time.time
|
||||
self.id = uuid.uuid4().hex
|
||||
self.kind = kind
|
||||
self.title = title
|
||||
self.record_changing = record_changing
|
||||
self.admission = resolve_admission(admission, record_changing)
|
||||
self.cancellable = bool(cancellable)
|
||||
self.scope = dict(scope or {})
|
||||
self.state = "running"
|
||||
self.started_at = self._clock()
|
||||
@@ -97,6 +184,10 @@ class Job:
|
||||
self.task = None
|
||||
self._subscribers = set()
|
||||
|
||||
@property
|
||||
def record_changing(self):
|
||||
return self.admission == "records"
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self.state == "running"
|
||||
@@ -170,6 +261,32 @@ class Job:
|
||||
)
|
||||
self.log_line(log_line)
|
||||
|
||||
def record_counted(self, group, count, rows=()):
|
||||
"""Count ``count`` outcomes in ``group`` at once, listing ``rows``.
|
||||
|
||||
For outcomes that need no log line each (a thumbnail job's already
|
||||
fresh previews): the counter is exact, ``rows`` (``record``'s row
|
||||
shape) are listed up to MAX_OUTCOME_ROWS, and nothing is logged.
|
||||
"""
|
||||
if group not in self.counters:
|
||||
raise ValueError(f"Unknown outcome group {group!r} for {self.kind}")
|
||||
self.counters[group] += max(0, int(count))
|
||||
listed = self.outcomes[group]
|
||||
for row in rows:
|
||||
if len(listed) >= MAX_OUTCOME_ROWS:
|
||||
break
|
||||
index = row.get("root_index")
|
||||
if isinstance(index, bool) or not isinstance(index, int) or index < 0:
|
||||
index = None
|
||||
listed.append(
|
||||
{
|
||||
"path": _relative_path(row.get("path")),
|
||||
"root": str(row.get("root") or ""),
|
||||
"root_index": index,
|
||||
"detail": str(row.get("detail") or ""),
|
||||
}
|
||||
)
|
||||
|
||||
def finish(self, state="finished"):
|
||||
"""End the job in ``state`` and release every subscriber.
|
||||
|
||||
@@ -186,18 +303,24 @@ class Job:
|
||||
for queue in list(self._subscribers):
|
||||
queue.put_nowait(None)
|
||||
|
||||
async def events(self):
|
||||
async def events(self, lead=None):
|
||||
"""Yield the latest event, then each later one, until the job finishes.
|
||||
|
||||
``lead``, when given, is yielded first, after the subscription is in
|
||||
place, so nothing published while the consumer sends it is missed.
|
||||
Consume it under ``contextlib.aclosing`` so a consumer that stops early
|
||||
(a closed socket) drops its queue at once instead of at garbage collection.
|
||||
"""
|
||||
queue = asyncio.Queue()
|
||||
self._subscribers.add(queue)
|
||||
# As of the subscription: a later event or the end reaches the queue.
|
||||
snapshot, ended = self.last_event, self.finished
|
||||
try:
|
||||
if self.last_event is not None:
|
||||
yield self.last_event
|
||||
if self.finished:
|
||||
if lead is not None:
|
||||
yield lead
|
||||
if snapshot is not None:
|
||||
yield snapshot
|
||||
if ended:
|
||||
return
|
||||
while True:
|
||||
payload = await queue.get()
|
||||
@@ -227,6 +350,9 @@ class Job:
|
||||
# What detail() lists, so a view re-reads it only when it changed.
|
||||
"listed": {group: len(rows) for group, rows in self.outcomes.items()},
|
||||
"log_length": len(self.log),
|
||||
# A view offers Cancel only when the server can stop the job.
|
||||
"cancellable": self.cancellable,
|
||||
"admission": self.admission,
|
||||
}
|
||||
|
||||
def detail(self):
|
||||
@@ -240,6 +366,36 @@ class Job:
|
||||
"log": list(self.log),
|
||||
}
|
||||
|
||||
def report(self):
|
||||
"""The downloadable report: the summary, every outcome group and the log.
|
||||
|
||||
Each group carries its exact count, its listed rows (at most
|
||||
MAX_OUTCOME_ROWS) and whether the list is cut short. Rows and log lines
|
||||
already hold root-relative paths only.
|
||||
"""
|
||||
summary = self.summary()
|
||||
summary.pop("listed", None)
|
||||
summary.pop("log_length", None)
|
||||
return {
|
||||
"report_version": 1,
|
||||
"generated_at": self._clock(),
|
||||
"job": summary,
|
||||
"outcomes": {
|
||||
group: {
|
||||
"count": self.counters.get(group, len(rows)),
|
||||
"rows": [dict(row) for row in rows],
|
||||
"truncated": self.counters.get(group, len(rows)) > len(rows),
|
||||
}
|
||||
for group, rows in self.outcomes.items()
|
||||
},
|
||||
"log": list(self.log),
|
||||
}
|
||||
|
||||
def report_filename(self):
|
||||
"""The download name: prompt-manager-KIND-YYYYMMDD-HHMM.json (local time)."""
|
||||
stamp = time.strftime("%Y%m%d-%H%M", time.localtime(self._clock()))
|
||||
return f"prompt-manager-{self.kind}-{stamp}.json"
|
||||
|
||||
|
||||
class JobRegistry:
|
||||
"""Every job of this server session, running and recently finished."""
|
||||
@@ -248,27 +404,93 @@ class JobRegistry:
|
||||
self._logger = logger_ or logger
|
||||
self._jobs = {}
|
||||
|
||||
def start(self, kind, title, *, record_changing, scope, runner, groups=()) -> Job:
|
||||
def start(
|
||||
self,
|
||||
kind,
|
||||
title,
|
||||
*,
|
||||
admission=None,
|
||||
record_changing=None,
|
||||
cancellable=True,
|
||||
scope=None,
|
||||
runner,
|
||||
groups=(),
|
||||
) -> Job:
|
||||
"""Create a job and run ``runner(job)`` as a task.
|
||||
|
||||
Raises JobBusy, naming the job in the way, when a job of ``kind`` is
|
||||
running or when ``record_changing`` and another record-changing job
|
||||
runs. Must be called from the event loop.
|
||||
``admission`` is one of ``ADMISSIONS``; ``record_changing=True`` is
|
||||
the older spelling of ``records`` and ``False`` of ``read``. One of
|
||||
the two is required (TypeError otherwise), so no new job runs as
|
||||
``read`` beside a scan by omission. Raises JobBusy, naming the job in
|
||||
the way, when the admission class refuses it (see the module
|
||||
docstring). Must be called from the event loop.
|
||||
"""
|
||||
if admission is None and record_changing is None:
|
||||
raise TypeError(
|
||||
f"JobRegistry.start({kind!r}) needs an admission class "
|
||||
"(admission= or record_changing=)"
|
||||
)
|
||||
job = self._admit(
|
||||
kind,
|
||||
title,
|
||||
admission=admission,
|
||||
record_changing=record_changing,
|
||||
cancellable=cancellable,
|
||||
scope=scope,
|
||||
groups=groups,
|
||||
)
|
||||
job.task = asyncio.ensure_future(self._run(job, runner))
|
||||
return job
|
||||
|
||||
def hold(self, kind, title, *, admission, scope=None):
|
||||
"""A running, non-cancellable job for a legacy-page operation.
|
||||
|
||||
Returns an async context manager. Entering it registers the job and
|
||||
yields it, or raises JobBusy at once, without waiting, when the
|
||||
admission class refuses it; nothing is registered until then, so a
|
||||
hold that is never entered leaves no job behind. The job ends when
|
||||
the block exits: ``finished``, or ``failed`` when it raised (unless
|
||||
the body already ended it). A caller that answers a refusal in its
|
||||
own way checks ``blocker(kind, admission=...)`` first, with no await
|
||||
before the ``async with``.
|
||||
"""
|
||||
resolve_admission(admission) # an unknown class fails at the call
|
||||
return _Hold(
|
||||
lambda: self._admit(
|
||||
kind, title, admission=admission, cancellable=False, scope=scope
|
||||
)
|
||||
)
|
||||
|
||||
def _admit(self, kind, title, *, admission, record_changing=None, **job_args):
|
||||
resolved = resolve_admission(admission, record_changing)
|
||||
blocker = self.blocker(kind, admission=resolved)
|
||||
if blocker is not None:
|
||||
raise JobBusy(blocker)
|
||||
self._prune()
|
||||
job = Job(kind, title, admission=resolved, **job_args)
|
||||
self._jobs[job.id] = job
|
||||
return job
|
||||
|
||||
def blocker(self, kind, *, admission=None, record_changing=None, compatible=()):
|
||||
"""The running job a new ``kind`` job would be refused for, or None.
|
||||
|
||||
The same kind refuses first; then the oldest running job whose class
|
||||
the new job's admission class may not run beside. Running jobs of a
|
||||
kind in ``compatible`` never refuse it: the caller knows that work
|
||||
cannot conflict with its own (the monitor's Needs review recording
|
||||
beside thumbnail work, which writes previews and no records).
|
||||
"""
|
||||
same_kind = self.active(kind)
|
||||
if same_kind is not None:
|
||||
raise JobBusy(same_kind)
|
||||
if record_changing:
|
||||
for job in self._jobs.values():
|
||||
if job.running and job.record_changing:
|
||||
raise JobBusy(job)
|
||||
self._prune()
|
||||
job = Job(
|
||||
kind, title, record_changing=record_changing, scope=scope, groups=groups
|
||||
)
|
||||
self._jobs[job.id] = job
|
||||
job.task = asyncio.ensure_future(self._run(job, runner))
|
||||
return job
|
||||
return same_kind
|
||||
refused = _REFUSED_BESIDE[resolve_admission(admission, record_changing)]
|
||||
# A snapshot: the monitor's worker thread asks too (Needs review).
|
||||
running = [job for job in tuple(self._jobs.values()) if job.running]
|
||||
running.sort(key=lambda job: job._started_seq)
|
||||
for job in running:
|
||||
if job.admission in refused and job.kind not in compatible:
|
||||
return job
|
||||
return None
|
||||
|
||||
async def _run(self, job, runner):
|
||||
try:
|
||||
@@ -293,9 +515,15 @@ class JobRegistry:
|
||||
def get(self, job_id):
|
||||
return self._jobs.get(job_id)
|
||||
|
||||
def running(self):
|
||||
"""The running jobs, oldest first (a snapshot, safe off the event loop)."""
|
||||
jobs = [job for job in tuple(self._jobs.values()) if job.running]
|
||||
jobs.sort(key=lambda job: job._started_seq)
|
||||
return jobs
|
||||
|
||||
def active(self, kind):
|
||||
"""The running job of ``kind``, or None."""
|
||||
for job in self._jobs.values():
|
||||
"""The running job of ``kind``, or None (safe off the event loop)."""
|
||||
for job in tuple(self._jobs.values()):
|
||||
if job.kind == kind and job.running:
|
||||
return job
|
||||
return None
|
||||
@@ -312,11 +540,28 @@ class JobRegistry:
|
||||
def cancel(self, job_id) -> bool:
|
||||
"""Ask a running job to stop at its next safe boundary.
|
||||
|
||||
Returns whether a cancellation was requested: False for a finished or
|
||||
unknown job, so a cancel after the end is a no-op.
|
||||
Returns whether a cancellation was requested: False for a finished,
|
||||
unknown or non-cancellable job, so a cancel after the end is a no-op.
|
||||
"""
|
||||
job = self._jobs.get(job_id)
|
||||
if job is None or job.finished:
|
||||
if job is None or job.finished or not job.cancellable:
|
||||
return False
|
||||
job.cancelled = True
|
||||
return True
|
||||
|
||||
|
||||
class _Hold:
|
||||
"""The async context manager ``JobRegistry.hold`` returns."""
|
||||
|
||||
def __init__(self, admit):
|
||||
self._admit = admit
|
||||
self._job = None
|
||||
|
||||
async def __aenter__(self):
|
||||
self._job = self._admit()
|
||||
return self._job
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
if self._job.running:
|
||||
self._job.finish("finished" if exc_type is None else "failed")
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,420 @@
|
||||
"""Library location routes (#249 S6b): where the library lives, Move, Keep.
|
||||
|
||||
- ``GET /prompt_manager/system/library`` → ``{success, location}``: the
|
||||
database's path (relative to ComfyUI, as the statistics show it), why it is
|
||||
there (``source``: ``default``, ``environment``, ``config`` or
|
||||
``node_folder``), whether it can move and where to (``target``), the
|
||||
"Keep it here" choice (``keep``), whether to offer the move (``offer``:
|
||||
movable and not kept), a ``problem`` code and the database's size. The
|
||||
System Overview carries the same object as ``location``.
|
||||
- ``POST /prompt_manager/system/library/move`` → 202 ``{job}``: the
|
||||
``library_move`` job, ``exclusive`` (it runs alone and every other job,
|
||||
legacy-page operations included, is refused while it runs), not
|
||||
cancellable. It copies, verifies and switches the live library
|
||||
(``database/library_move.py``) and ends with one truthful line; a failure
|
||||
leaves the library in the node folder, unchanged. Refused before any job
|
||||
starts: 400 ``not-movable``/``same-folder``/``target-invalid``, 409
|
||||
``target-has-library``/``target-conflict``/``busy``, 507
|
||||
``insufficient-space``.
|
||||
- ``POST /prompt_manager/system/library/keep {keep: bool}`` persists
|
||||
``library.keep_in_node_folder`` through the config store and answers the
|
||||
new location.
|
||||
|
||||
Errors are ``{success:false, code, error}`` and never name a path.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import os
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .job_routes import busy_response
|
||||
from .jobs import JobBusy
|
||||
|
||||
try:
|
||||
from ...database import library_move as lm
|
||||
from ...database.operations import resolve_library_location
|
||||
from ...utils import library_location as ll
|
||||
except ImportError:
|
||||
from database import library_move as lm
|
||||
from database.operations import resolve_library_location
|
||||
from utils import library_location as ll
|
||||
|
||||
LIBRARY_MOVE_JOB_KIND = "library_move"
|
||||
LIBRARY_MOVE_TITLE = "Move library"
|
||||
LIBRARY_MOVE_GROUPS = ("files", "errors")
|
||||
KEEP_BODY_MAX = 1024
|
||||
|
||||
_REFUSAL_STATUS = {
|
||||
"not-movable": 400,
|
||||
"same-folder": 400,
|
||||
"target-invalid": 400,
|
||||
"target-has-library": 409,
|
||||
"target-conflict": 409,
|
||||
"insufficient-space": 507,
|
||||
}
|
||||
STILL_THERE = "The library is still in the node folder, unchanged."
|
||||
NOT_MOVED = "The library was not moved."
|
||||
OLD_LEFT = {
|
||||
"old-in-use": (
|
||||
"The old database is still open elsewhere, so it was left in the node "
|
||||
"folder; it is no longer used."
|
||||
),
|
||||
"old-locked": (
|
||||
"The old files could not be renamed, so they were left in the node "
|
||||
"folder; they are no longer used."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _failure(message, status, code):
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": message}, status=status
|
||||
)
|
||||
|
||||
|
||||
def _plural(count, one, many=None):
|
||||
return f"{count:,} {one if count == 1 else (many or one + 's')}"
|
||||
|
||||
|
||||
def _same_file(first, second):
|
||||
try:
|
||||
return os.path.samefile(first, second)
|
||||
except OSError:
|
||||
return os.path.normcase(os.path.realpath(first)) == os.path.normcase(
|
||||
os.path.realpath(second)
|
||||
)
|
||||
|
||||
|
||||
CUSTOM_NODES = "custom_nodes"
|
||||
|
||||
|
||||
def _inside(path, folder):
|
||||
"""``path`` relative to ``folder`` by text (normalised spellings), or None."""
|
||||
child, parent = os.path.normcase(path), os.path.normcase(folder)
|
||||
if child == parent:
|
||||
return ""
|
||||
prefix = parent.rstrip(os.sep) + os.sep
|
||||
return path[len(prefix) :] if child.startswith(prefix) else None
|
||||
|
||||
|
||||
def _comfyui_custom_nodes():
|
||||
"""ComfyUI's custom_nodes folders as it lists them (unresolved), or []."""
|
||||
try:
|
||||
from ..config import _import_folder_paths
|
||||
|
||||
getter = getattr(_import_folder_paths(), "get_folder_paths", None)
|
||||
folders = getter(CUSTOM_NODES) if getter else []
|
||||
return [folder for folder in folders or [] if isinstance(folder, str)]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def node_folder_public_path(path, extension_root, custom_nodes):
|
||||
"""``custom_nodes/<node folder>/...`` for a path in the node folder, or None.
|
||||
|
||||
The node folder is matched at its own unresolved place in a custom_nodes
|
||||
folder (one of ``custom_nodes``, or the parent of ``extension_root`` when
|
||||
that is named custom_nodes), so a node folder that is a symlink to a
|
||||
checkout outside ComfyUI still reads relative to ComfyUI. ``path`` may be
|
||||
spelled through the link or resolved. Never absolute, never ``..``:
|
||||
anything else is None and the caller uses ``_public_path``.
|
||||
"""
|
||||
if not path or not extension_root:
|
||||
return None
|
||||
try:
|
||||
root = os.path.normpath(os.path.abspath(extension_root))
|
||||
name = os.path.basename(root)
|
||||
folders = [os.path.normpath(os.path.abspath(f)) for f in custom_nodes]
|
||||
parent = os.path.dirname(root)
|
||||
if os.path.normcase(os.path.basename(parent)) == CUSTOM_NODES:
|
||||
folders.append(parent)
|
||||
spellings = [os.path.normpath(os.path.abspath(path))]
|
||||
spellings.append(os.path.realpath(spellings[0]))
|
||||
except (OSError, ValueError, TypeError):
|
||||
return None
|
||||
for folder in folders:
|
||||
node = os.path.join(folder, name)
|
||||
for spelling, node_spelling in zip(spellings, (node, os.path.realpath(node))):
|
||||
rest = _inside(spelling, node_spelling)
|
||||
if rest is None:
|
||||
continue
|
||||
parts = [os.path.basename(folder), name] + (
|
||||
rest.split(os.sep) if rest else []
|
||||
)
|
||||
if all(part and part not in (".", "..") for part in parts):
|
||||
return "/".join(parts)
|
||||
return None
|
||||
|
||||
|
||||
class LibraryLocationRoutesMixin:
|
||||
"""Mixin for the /prompt_manager/system/library endpoints."""
|
||||
|
||||
# Replaced in tests with a resolver over a temp tree.
|
||||
_library_resolver = staticmethod(resolve_library_location)
|
||||
|
||||
def _register_library_location_routes(self, routes):
|
||||
@routes.get("/prompt_manager/system/library")
|
||||
async def library_location_route(request):
|
||||
return await self.get_library_location(request)
|
||||
|
||||
@routes.post("/prompt_manager/system/library/move")
|
||||
async def library_move_route(request):
|
||||
return await self.start_library_move(request)
|
||||
|
||||
@routes.post("/prompt_manager/system/library/keep")
|
||||
async def library_keep_route(request):
|
||||
return await self.keep_library(request)
|
||||
|
||||
# ── location ──────────────────────────────────────────────────────
|
||||
|
||||
def _library_public_path(self, path):
|
||||
"""The library's path for display: through the node folder's place in
|
||||
custom_nodes when it is there (a symlinked node folder included), else
|
||||
``_public_path``. Only the library location uses this; sources and
|
||||
jobs keep ``_public_path``, which their requests round-trip."""
|
||||
shown = node_folder_public_path(
|
||||
path, self._extension_root, _comfyui_custom_nodes()
|
||||
)
|
||||
return shown or self._public_path(path)
|
||||
|
||||
def _live_location(self):
|
||||
"""The resolved location, unless the live database is another file.
|
||||
|
||||
A database the API was handed (tests, the browser fixture server) or
|
||||
one the resolver no longer names is reported as configured and is
|
||||
never moved.
|
||||
"""
|
||||
location = self._library_resolver()
|
||||
live = self.db.model.db_path
|
||||
if _same_file(location.db_path, live):
|
||||
return location
|
||||
return ll.LibraryLocation(
|
||||
ll.SOURCE_CONFIG,
|
||||
live,
|
||||
location.config_path,
|
||||
os.path.dirname(live),
|
||||
config_explicit=location.config_explicit,
|
||||
)
|
||||
|
||||
def _location_payload(self, location):
|
||||
"""The public view of ``location`` (blocking: stats the database)."""
|
||||
from ..config import PromptManagerConfig
|
||||
|
||||
keep = bool(PromptManagerConfig.KEEP_LIBRARY_IN_NODE_FOLDER)
|
||||
size = 0
|
||||
try:
|
||||
size = os.path.getsize(location.db_path)
|
||||
except OSError:
|
||||
pass
|
||||
return {
|
||||
"path": self._library_public_path(location.db_path),
|
||||
# Where Move library would put it, and whether config.json goes too.
|
||||
"target_source": location.target_source if location.movable else None,
|
||||
"config_moves": location.movable and not location.config_explicit,
|
||||
"source": location.source,
|
||||
"movable": location.movable,
|
||||
"keep": keep,
|
||||
"offer": location.movable and not keep,
|
||||
"target": (
|
||||
self._library_public_path(location.target_dir)
|
||||
if location.movable
|
||||
else None
|
||||
),
|
||||
"problem": location.problem or self._node_folder_risk(location),
|
||||
"size_bytes": size,
|
||||
}
|
||||
|
||||
# The node folder (tests point it at a temp tree).
|
||||
_extension_root = ll.EXTENSION_ROOT
|
||||
|
||||
def _node_folder_risk(self, location):
|
||||
"""``inside-node-folder`` for a library that stays in the node folder.
|
||||
|
||||
A configured database path, or a config.json the resolver still reads
|
||||
from the node folder, is deleted with the node by a reinstall, and
|
||||
it is never offered a move.
|
||||
"""
|
||||
if location.source == ll.SOURCE_NODE_FOLDER:
|
||||
return None
|
||||
paths = [location.db_path] + (
|
||||
[] if location.config_explicit else [location.config_path]
|
||||
)
|
||||
root = self._extension_root
|
||||
for path in paths:
|
||||
parent = os.path.dirname(os.path.abspath(path))
|
||||
if ll.same_folder(parent, root) or os.path.normcase(
|
||||
os.path.realpath(parent)
|
||||
).startswith(os.path.normcase(os.path.realpath(root)) + os.sep):
|
||||
return "inside-node-folder"
|
||||
return None
|
||||
|
||||
def _library_payload(self):
|
||||
return self._location_payload(self._live_location())
|
||||
|
||||
async def get_library_location(self, request):
|
||||
try:
|
||||
payload = await self._run_in_executor(self._library_payload)
|
||||
except Exception:
|
||||
self.logger.exception("Library location could not be read")
|
||||
return _failure(
|
||||
"The library location could not be read.", 500, "location-failed"
|
||||
)
|
||||
return web.json_response(
|
||||
{"success": True, "location": payload},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
# ── move ──────────────────────────────────────────────────────────
|
||||
|
||||
def _plan_library_move(self):
|
||||
"""The checked plan (blocking); raises MoveFailed."""
|
||||
plan = lm.plan_move(self._live_location())
|
||||
if lm.space_problem(plan) is not None:
|
||||
raise lm.MoveFailed("insufficient-space")
|
||||
return plan
|
||||
|
||||
async def start_library_move(self, request):
|
||||
blocker = self._jobs.blocker(LIBRARY_MOVE_JOB_KIND, admission="exclusive")
|
||||
if blocker is not None:
|
||||
return busy_response(JobBusy(blocker))
|
||||
try:
|
||||
await self._run_in_executor(self._plan_library_move)
|
||||
except lm.MoveFailed as refused:
|
||||
status = _REFUSAL_STATUS.get(refused.code, 400)
|
||||
return _failure(refused.message, status, refused.code)
|
||||
except Exception:
|
||||
self.logger.exception("Move library could not be planned")
|
||||
return _failure(NOT_MOVED, 500, "move-failed")
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
LIBRARY_MOVE_JOB_KIND,
|
||||
LIBRARY_MOVE_TITLE,
|
||||
admission="exclusive",
|
||||
cancellable=False,
|
||||
scope={},
|
||||
groups=LIBRARY_MOVE_GROUPS,
|
||||
runner=self._run_library_move,
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
def _after_library_switch(self):
|
||||
"""Forget what described the old file (best effort, after the switch).
|
||||
|
||||
Plans, the health report and the history read fence named the old
|
||||
file; the fence is closed before it is dropped.
|
||||
"""
|
||||
self._database_replaced()
|
||||
fence, self._history_read_fence = self._history_read_fence, None
|
||||
if fence is not None:
|
||||
fence.close()
|
||||
|
||||
def finish_interrupted_library_move(self):
|
||||
"""Startup: rename old files a crash left after a move (blocking)."""
|
||||
try:
|
||||
location = self._library_resolver()
|
||||
if _same_file(location.db_path, self.db.model.db_path):
|
||||
return lm.finish_interrupted_move(location)
|
||||
except Exception:
|
||||
self.logger.exception("Could not finish an interrupted library move")
|
||||
return []
|
||||
|
||||
async def _run_library_move(self, job):
|
||||
from ..config_store import _COMMIT_LOCK
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def progress(line):
|
||||
def show():
|
||||
job.log_line(line)
|
||||
job.update(status_line=line)
|
||||
|
||||
loop.call_soon_threadsafe(show)
|
||||
|
||||
job.update(status_line="Checking the new folder")
|
||||
try:
|
||||
plan = await self._run_in_executor(self._plan_library_move)
|
||||
target = self._library_public_path(plan.target_dir)
|
||||
job.log_line(f"Moving the library to {target}")
|
||||
result = await self._run_in_executor(
|
||||
functools.partial(
|
||||
lm.move_library,
|
||||
plan,
|
||||
after_switch=self._after_library_switch,
|
||||
config_lock=_COMMIT_LOCK,
|
||||
progress=progress,
|
||||
)
|
||||
)
|
||||
except lm.MoveFailed as failed:
|
||||
self.logger.error(f"Move library failed: {failed.code}")
|
||||
return self._fail_library_move(
|
||||
job, failed.code, f"{failed.message} {STILL_THERE}"
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Move library failed")
|
||||
return self._fail_library_move(job, None, f"{NOT_MOVED} {STILL_THERE}")
|
||||
await asyncio.sleep(0) # the progress lines come first
|
||||
job.record_counted("files", len(result.files))
|
||||
line = (
|
||||
f"Moved the library to {target}: "
|
||||
f"{_plural(result.counts.get('prompts', 0), 'prompt')} and "
|
||||
f"{_plural(result.counts.get('generated_images', 0), 'image')} verified"
|
||||
)
|
||||
if result.retire_problem:
|
||||
line += ". " + OLD_LEFT.get(result.retire_problem, OLD_LEFT["old-locked"])
|
||||
elif result.renamed:
|
||||
line += (
|
||||
f". The old files were renamed {', '.join(result.renamed)}; "
|
||||
"nothing was deleted"
|
||||
)
|
||||
line += ". Logs move at the next restart."
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=line)
|
||||
|
||||
@staticmethod
|
||||
def _fail_library_move(job, code, message):
|
||||
from .maintenance import _fail_job
|
||||
|
||||
_fail_job(job, code, message)
|
||||
|
||||
# ── keep it here ─────────────────────────────────────────────────
|
||||
|
||||
async def keep_library(self, request):
|
||||
from ..config import PromptManagerConfig
|
||||
from ..config_store import get_config_store
|
||||
|
||||
try:
|
||||
if (request.content_length or 0) > KEEP_BODY_MAX:
|
||||
raise ValueError("too large")
|
||||
data = await request.json()
|
||||
except Exception:
|
||||
data = None
|
||||
if not isinstance(data, dict) or not isinstance(data.get("keep"), bool):
|
||||
return _failure("keep must be true or false.", 400, "invalid-request")
|
||||
keep = data["keep"]
|
||||
|
||||
def mutate(config):
|
||||
library = config.get("library")
|
||||
library = dict(library) if isinstance(library, dict) else {}
|
||||
library["keep_in_node_folder"] = keep
|
||||
config["library"] = library
|
||||
return config
|
||||
|
||||
def publish(_config):
|
||||
PromptManagerConfig.KEEP_LIBRARY_IN_NODE_FOLDER = keep
|
||||
|
||||
try:
|
||||
await self._run_in_executor(
|
||||
functools.partial(
|
||||
get_config_store().commit,
|
||||
mutate,
|
||||
expected_revision=None,
|
||||
publish=publish,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
return self._config_commit_failure(exc)
|
||||
return await self.get_library_location(request)
|
||||
+104
-20
@@ -5,12 +5,47 @@ from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from ..config_store import ConfigUnreadable, RevisionConflict, get_config_store
|
||||
from .jobs import LORA_IMPORT_JOB_KIND
|
||||
|
||||
# Flags of POST /lora/enable; each must be a JSON boolean when present.
|
||||
_LORA_FLAGS = ("enabled", "trigger_words_enabled", "clear_civitai_api_key")
|
||||
|
||||
_LORA_PATH_ERROR = (
|
||||
"LoraManager path must be a directory under ComfyUI's custom_nodes "
|
||||
"whose name contains 'lora'"
|
||||
)
|
||||
|
||||
|
||||
def _lora_settings_mutation(values):
|
||||
"""The config commit for a LoRA save: only ``integrations.lora_manager``.
|
||||
|
||||
The submitted keys replace their values; other LoRA keys, other
|
||||
integrations and every other section stay as they are on disk.
|
||||
"""
|
||||
|
||||
def mutate(config):
|
||||
integrations = config.get("integrations")
|
||||
integrations = dict(integrations) if isinstance(integrations, dict) else {}
|
||||
lora = integrations.get("lora_manager")
|
||||
lora = dict(lora) if isinstance(lora, dict) else {}
|
||||
lora.update(values)
|
||||
integrations["lora_manager"] = lora
|
||||
config["integrations"] = integrations
|
||||
return config
|
||||
|
||||
return mutate
|
||||
|
||||
|
||||
def _lora_error_frame(message):
|
||||
"""An import stream's error event, as the admin page's modal reads it.
|
||||
|
||||
The modal shows ``status`` and ``progress`` from every frame and closes
|
||||
on ``type: error``, naming ``message``.
|
||||
"""
|
||||
return {"type": "error", "status": message, "progress": 0, "message": message}
|
||||
|
||||
|
||||
class LoraIntegrationMixin:
|
||||
"""Mixin providing LoraManager detection, scanning, and trigger word endpoints."""
|
||||
|
||||
@@ -131,14 +166,25 @@ class LoraIntegrationMixin:
|
||||
return allowed, None
|
||||
|
||||
async def lora_enable(self, request):
|
||||
"""Enable or disable LoraManager integration and save to config.json."""
|
||||
"""Enable or disable LoraManager integration and save it to config.json.
|
||||
|
||||
The values are committed through the config store first (#220) and
|
||||
published at runtime only once config.json holds them; a save that
|
||||
cannot be written reports ``persisted: false`` and changes nothing.
|
||||
The CivitAI key is never echoed back.
|
||||
"""
|
||||
try:
|
||||
data = await request.json()
|
||||
data, error = await self._read_json_body(request)
|
||||
if error is not None:
|
||||
return error
|
||||
for flag in _LORA_FLAGS:
|
||||
if flag in data and not isinstance(data[flag], bool):
|
||||
return self._bad_lora_request(f"{flag} must be true or false")
|
||||
enabled = data.get("enabled", False)
|
||||
path = data.get("path", "")
|
||||
trigger_words = data.get("trigger_words_enabled", False)
|
||||
|
||||
from ..config import IntegrationConfig, PromptManagerConfig
|
||||
from ..config import IntegrationConfig
|
||||
from ..lora_utils import get_trigger_cache
|
||||
|
||||
# The key is never echoed back to the UI, so a blank or absent
|
||||
@@ -161,20 +207,40 @@ class LoraIntegrationMixin:
|
||||
if error is not None:
|
||||
return error
|
||||
|
||||
# Update in-memory config
|
||||
IntegrationConfig.LORA_MANAGER_ENABLED = enabled
|
||||
IntegrationConfig.LORA_MANAGER_PATH = path
|
||||
IntegrationConfig.LORA_TRIGGER_WORDS_ENABLED = trigger_words
|
||||
IntegrationConfig.CIVITAI_API_KEY = civitai_key
|
||||
values = {
|
||||
"enabled": enabled,
|
||||
"path": path,
|
||||
"trigger_words_enabled": trigger_words,
|
||||
"civitai_api_key": civitai_key,
|
||||
}
|
||||
|
||||
# Persist to the configured config.json
|
||||
PromptManagerConfig.save_to_file()
|
||||
def publish(_config):
|
||||
# Under the commit lock, so the last save written is the one
|
||||
# the runtime keeps.
|
||||
IntegrationConfig.LORA_MANAGER_ENABLED = enabled
|
||||
IntegrationConfig.LORA_MANAGER_PATH = path
|
||||
IntegrationConfig.LORA_TRIGGER_WORDS_ENABLED = trigger_words
|
||||
IntegrationConfig.CIVITAI_API_KEY = civitai_key
|
||||
|
||||
try:
|
||||
_saved, revision = await self._run_in_executor(
|
||||
get_config_store().commit,
|
||||
_lora_settings_mutation(values),
|
||||
expected_revision=None,
|
||||
publish=publish,
|
||||
)
|
||||
except (ConfigUnreadable, RevisionConflict, OSError) as exc:
|
||||
return self._config_commit_failure(exc)
|
||||
|
||||
# Load trigger word cache if enabling
|
||||
cache = get_trigger_cache()
|
||||
if enabled and trigger_words and path:
|
||||
count = await self._run_in_executor(cache.load, path)
|
||||
self.logger.info(f"Trigger word cache loaded: {count} LoRAs")
|
||||
try:
|
||||
count = await self._run_in_executor(cache.load, path)
|
||||
self.logger.info(f"Trigger word cache loaded: {count} LoRAs")
|
||||
except Exception as e:
|
||||
# Saved and enabled; only the trigger words are missing.
|
||||
self.logger.warning(f"Trigger word cache could not be loaded: {e}")
|
||||
cache.clear()
|
||||
elif not enabled:
|
||||
cache.clear()
|
||||
|
||||
@@ -184,6 +250,10 @@ class LoraIntegrationMixin:
|
||||
"enabled": enabled,
|
||||
"path": self._public_path(path),
|
||||
"trigger_words_enabled": trigger_words,
|
||||
"persisted": True,
|
||||
"revision": revision,
|
||||
"runtime_applied": True,
|
||||
"cache_loaded": cache.is_loaded,
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
@@ -218,19 +288,33 @@ class LoraIntegrationMixin:
|
||||
reason="OK",
|
||||
headers={"Content-Type": "text/event-stream"},
|
||||
)
|
||||
await response.prepare(request)
|
||||
|
||||
async def send_progress(data):
|
||||
line = f"data: {json.dumps(data)}\n\n"
|
||||
await response.write(line.encode("utf-8"))
|
||||
|
||||
try:
|
||||
await self._lora_scan_stream(allowed, send_progress)
|
||||
except Exception as e:
|
||||
self.logger.exception("LoRA scan failed")
|
||||
await send_progress({"type": "error", "message": self._public_error(e)})
|
||||
# The import replaces prompts and images: a records hold, so it
|
||||
# never runs beside a scan, restore or tagging job.
|
||||
busy = self._jobs.blocker(LORA_IMPORT_JOB_KIND, admission="records")
|
||||
if busy is not None:
|
||||
await response.prepare(request)
|
||||
message = f"{busy.title} already running"
|
||||
await send_progress(_lora_error_frame(message))
|
||||
await response.write_eof()
|
||||
return response
|
||||
|
||||
await response.write_eof()
|
||||
async with self._jobs.hold(
|
||||
LORA_IMPORT_JOB_KIND, "LoRA import (admin page)", admission="records"
|
||||
) as job:
|
||||
await response.prepare(request)
|
||||
try:
|
||||
await self._lora_scan_stream(allowed, send_progress)
|
||||
except Exception as e:
|
||||
self.logger.exception("LoRA scan failed")
|
||||
job.finish("failed")
|
||||
await send_progress(_lora_error_frame(self._public_error(e)))
|
||||
|
||||
await response.write_eof()
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -0,0 +1,815 @@
|
||||
"""Maintenance routes: reviewed, read-only plans (#249 System S6).
|
||||
|
||||
``POST /prompt_manager/maintenance/preview {operation, options:{}}`` reviews
|
||||
one operation (``duplicate_prompts``, ``optimize``, ``unused_components`` or
|
||||
``missing_links``; ``database/maintenance.py``) off the event loop and
|
||||
answers a ``Plan``: ``{success, token, operation, expires_at, scope:{count,
|
||||
label}, protected:{count, reasons}, backup:{policy, destination_label},
|
||||
media:{deletes_files, label}, details:[≤50 {label, detail}]}``. The preview
|
||||
writes nothing. The plan's fingerprint, its targets and the sources it judged
|
||||
stay in the server's ``PlanStore`` under the token, bound to the database the
|
||||
preview read (a restore that replaces it invalidates the plan).
|
||||
|
||||
The operation enum is strict, ``options`` must be an empty object and the
|
||||
body is capped at ``PREVIEW_BODY_MAX`` bytes. Errors are ``{success:false,
|
||||
code, error}`` with ``invalid-request`` (400), ``too-large`` (413: the body,
|
||||
or a plan over the store's budget), ``storage-changed`` (409: the database
|
||||
was replaced while the preview read it) or a null code (500); none carries a
|
||||
server path or exception text.
|
||||
|
||||
``POST /prompt_manager/maintenance/apply {token}`` applies a reviewed plan as
|
||||
an Activity job (202 ``{job}``): kind ``maintenance``, titled after the
|
||||
operation, admission ``records`` (refused while a scan, restore, LoRA import
|
||||
or other record-changing job runs, and refusing them meanwhile), not
|
||||
cancellable, with outcome groups ``changed``, ``protected`` and ``errors``.
|
||||
The token is consumed when the job starts (on the same loop turn, before its
|
||||
runner runs); a refused start keeps it. The runner:
|
||||
|
||||
1. takes a safety backup when the plan was reviewed with the policy on or the
|
||||
policy (``maintenance.safety_backup``, default on) is on now: SQLite's
|
||||
backup API into a temp file beside the database, ``verify_database_file``,
|
||||
fsync, ``0o600`` on POSIX, then ``os.replace`` onto a unique
|
||||
``<db>.backup_<YYYYmmdd_HHMMSS_ffffff>[_n]`` name. Any failure removes
|
||||
what it wrote and ends the job ``backup-failed`` with nothing changed;
|
||||
2. applies the plan, which first rechecks the reviewed fingerprint inside
|
||||
``BEGIN IMMEDIATE`` (``plan-changed`` writes nothing);
|
||||
3. counts the changed and protected records, and prunes old safety backups.
|
||||
|
||||
The job log names the backup by its file name only, and the job says "No
|
||||
safety backup" when none was taken. Apply errors: ``invalid-request`` (400),
|
||||
``plan-expired`` (410: unknown, expired or forgotten after a restart),
|
||||
``plan-used`` (409, with the job that applied it), ``storage-changed`` (409:
|
||||
a restore replaced the database since the review) and ``busy`` (409).
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from contextlib import closing
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .admin import _prune_safety_backups, _scan_location, _scan_roots
|
||||
from .job_routes import busy_response
|
||||
from .jobs import MAINTENANCE_JOB_KIND, JobBusy
|
||||
from .maintenance_plans import PlanExpired, PlanUsed, ScopeTooLarge
|
||||
|
||||
try:
|
||||
from ...database.workflow_storage import PRE_MIGRATION_LABEL, pre_migration_backups
|
||||
from ...database.health import readonly_uri
|
||||
from ...database.maintenance import MAINTENANCE_OPERATIONS, PlanChanged
|
||||
except ImportError:
|
||||
from database.workflow_storage import PRE_MIGRATION_LABEL, pre_migration_backups
|
||||
from database.health import readonly_uri
|
||||
from database.maintenance import MAINTENANCE_OPERATIONS, PlanChanged
|
||||
|
||||
PREVIEW_BODY_MAX = 4096
|
||||
# Duplicate-file previews carry the reviewed keepers (py/api/duplicate_files.py).
|
||||
DUPLICATE_PREVIEW_BODY_MAX = 1024 * 1024
|
||||
DUPLICATE_FILES = "duplicate_files"
|
||||
WORKFLOW_STORAGE = "workflow_storage" # applied by py/api/workflow_storage.py
|
||||
PREVIEW_OPERATIONS = MAINTENANCE_OPERATIONS + (DUPLICATE_FILES,)
|
||||
# Room left over beyond what a step writes, so a disk is never filled to 0.
|
||||
DISK_MARGIN_BYTES = 16 * 2**20
|
||||
# The concept's Maintenance row titles, which name each operation's job.
|
||||
MAINTENANCE_TITLES = {
|
||||
"duplicate_prompts": "Duplicate prompt text",
|
||||
"optimize": "Optimize database",
|
||||
"unused_components": "Unused prompt components",
|
||||
"missing_links": "Missing-file links",
|
||||
"workflow_storage": "Store each workflow once",
|
||||
}
|
||||
_OPERATION_ERROR = "operation must be one of: " + ", ".join(PREVIEW_OPERATIONS)
|
||||
MAINTENANCE_GROUPS = ("changed", "protected", "errors")
|
||||
MAX_TOKEN_LENGTH = 128
|
||||
BACKUP_FAILED = (
|
||||
"The safety backup could not be taken and verified; nothing was changed."
|
||||
)
|
||||
PLAN_CHANGED = (
|
||||
"The library changed since the review; nothing was changed. "
|
||||
"Review the plan again."
|
||||
)
|
||||
APPLY_FAILED = "Maintenance failed; nothing was changed."
|
||||
NOTHING_TO_DO = "Every image record already stores its workflow once; nothing to do."
|
||||
NO_BACKUP = "No safety backup"
|
||||
_NAME_ATTEMPTS = 1000
|
||||
|
||||
|
||||
class SafetyBackupFailed(Exception):
|
||||
"""The safety backup could not be written, verified or put in place.
|
||||
|
||||
``reason`` is the verify reason when the copy did not verify.
|
||||
"""
|
||||
|
||||
def __init__(self, message, reason=None):
|
||||
super().__init__(message)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def backup_failed_message(error):
|
||||
"""The job line for a failed backup, naming a damaged workflow document."""
|
||||
reason = getattr(error, "reason", None) or ""
|
||||
if reason.startswith("Workflow document "):
|
||||
return f"{BACKUP_FAILED} {reason}"
|
||||
return BACKUP_FAILED
|
||||
|
||||
|
||||
def _now():
|
||||
return datetime.datetime.now()
|
||||
|
||||
|
||||
def _free_bytes(path):
|
||||
return shutil.disk_usage(path).free
|
||||
|
||||
|
||||
def _device(path):
|
||||
try:
|
||||
return os.stat(path).st_dev
|
||||
except OSError:
|
||||
return path
|
||||
|
||||
|
||||
def sqlite_temp_dir(environ=None, *, windows=None, usable=None):
|
||||
"""The folder SQLite builds its temporary files in (VACUUM's copy).
|
||||
|
||||
Mirrors SQLite's own search, which differs from ``tempfile``'s. On Unix
|
||||
(``unixTempFileDir``): ``SQLITE_TMPDIR``, ``TMPDIR``, then ``/var/tmp``,
|
||||
``/usr/tmp`` and ``/tmp``, the first that is a writable directory, else
|
||||
the current directory. On Windows (``GetTempPathW``): ``TMP``, ``TEMP``,
|
||||
then ``USERPROFILE``, else the current directory.
|
||||
"""
|
||||
environ = os.environ if environ is None else environ
|
||||
if windows is None:
|
||||
windows = os.name == "nt"
|
||||
if windows:
|
||||
for name in ("TMP", "TEMP", "USERPROFILE"):
|
||||
if environ.get(name):
|
||||
return environ[name]
|
||||
return os.path.abspath(".")
|
||||
if usable is None:
|
||||
|
||||
def usable(path):
|
||||
return os.path.isdir(path) and os.access(path, os.W_OK | os.X_OK)
|
||||
|
||||
candidates = [environ.get("SQLITE_TMPDIR"), environ.get("TMPDIR")]
|
||||
candidates += ["/var/tmp", "/usr/tmp", "/tmp"]
|
||||
for candidate in candidates:
|
||||
if candidate and usable(candidate):
|
||||
return candidate
|
||||
return os.path.abspath(".")
|
||||
|
||||
|
||||
def database_bytes(db_path):
|
||||
"""The size a safety backup of the database will be (blocking).
|
||||
|
||||
SQLite's backup API copies every page, committed WAL content included, so
|
||||
the copy is ``page_count * page_size`` read on one read-only snapshot.
|
||||
"""
|
||||
conn = sqlite3.connect(readonly_uri(db_path), uri=True)
|
||||
try:
|
||||
page_size = conn.execute("PRAGMA page_size").fetchone()[0]
|
||||
return page_size * conn.execute("PRAGMA page_count").fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _megabytes(size):
|
||||
return f"{size / 2**20:,.1f} MB"
|
||||
|
||||
|
||||
def disk_plan(db_path, *, backup, vacuum, vacuum_bytes=None):
|
||||
"""Whether the disk has room for an apply's writes (blocking).
|
||||
|
||||
``backup``: the safety backup, one database-sized file beside the
|
||||
database. ``vacuum``: VACUUM writes the rebuilt database through the WAL
|
||||
beside it and builds it in a temporary copy in SQLite's temporary folder
|
||||
(``sqlite_temp_dir``; SQLite's documentation: up to twice the database
|
||||
in free space). Needs
|
||||
on one device add up. ``vacuum_bytes``, when given, is what VACUUM
|
||||
copies (the live pages, for Reclaim space) instead of the whole file.
|
||||
Returns ``(public, short)``: ``public`` is
|
||||
``{free_bytes, required_bytes, sufficient}`` for the database folder,
|
||||
``short`` None or ``(place, what, required, free)`` of the first place
|
||||
that is short, and ``size`` the database's size (0 when nothing is
|
||||
written).
|
||||
"""
|
||||
db_path = os.path.abspath(db_path)
|
||||
folder = os.path.dirname(db_path)
|
||||
size = database_bytes(db_path) if backup or vacuum else 0
|
||||
needs = [] # (place, folder, bytes, what)
|
||||
if backup:
|
||||
needs.append(("database folder", folder, size, "the safety backup"))
|
||||
if vacuum:
|
||||
copy = size if vacuum_bytes is None else vacuum_bytes
|
||||
what = "Optimize" if vacuum_bytes is None else "Reclaim space"
|
||||
needs.append(("database folder", folder, copy, what))
|
||||
needs.append(("temporary folder", sqlite_temp_dir(), copy, what))
|
||||
devices = {}
|
||||
for place, path, needed, what in needs:
|
||||
entry = devices.setdefault(_device(path), {"place": place, "path": path})
|
||||
entry["bytes"] = entry.get("bytes", 0) + needed
|
||||
entry.setdefault("what", [])
|
||||
if what not in entry["what"]:
|
||||
entry["what"].append(what)
|
||||
short = None
|
||||
public = {"free_bytes": _free_bytes(folder), "required_bytes": 0}
|
||||
for device, entry in devices.items():
|
||||
required = entry["bytes"] + DISK_MARGIN_BYTES
|
||||
free = (
|
||||
public["free_bytes"]
|
||||
if entry["path"] == folder
|
||||
else _free_bytes(entry["path"])
|
||||
)
|
||||
if device == _device(folder):
|
||||
public["required_bytes"] = required
|
||||
if free < required and short is None:
|
||||
short = (entry["place"], " and ".join(entry["what"]), required, free)
|
||||
public["sufficient"] = short is None
|
||||
return public, short, size
|
||||
|
||||
|
||||
# A restore upload's hard ceiling; below it, free disk space decides.
|
||||
RESTORE_MAX_BYTES = 64 * 2**30
|
||||
|
||||
|
||||
def size_label(size):
|
||||
"""``1.6 GB`` or ``120.0 MB``: a backup's size for a message."""
|
||||
if size >= 2**30:
|
||||
return f"{size / 2**30:,.1f} GB"
|
||||
return _megabytes(size)
|
||||
|
||||
|
||||
def restore_room(db_path):
|
||||
"""The largest restore upload the disks have room for (blocking).
|
||||
|
||||
The upload is written to a temp file and staged once more in the
|
||||
temporary folder (2x), then copied over the live database beside it,
|
||||
whose current contents are first kept as a safety backup (1x the upload
|
||||
for the WAL, plus the current database). Needs on one device add up,
|
||||
each with ``DISK_MARGIN_BYTES``.
|
||||
"""
|
||||
db_path = os.path.abspath(db_path)
|
||||
folder = os.path.dirname(db_path)
|
||||
current = database_bytes(db_path) if os.path.exists(db_path) else 0
|
||||
devices = {}
|
||||
for path, per_upload, fixed in (
|
||||
(tempfile.gettempdir(), 2, 0),
|
||||
(folder, 1, current),
|
||||
):
|
||||
entry = devices.setdefault(_device(path), {"path": path, "per": 0, "fixed": 0})
|
||||
entry["per"] += per_upload
|
||||
entry["fixed"] += fixed
|
||||
room = min(
|
||||
(_free_bytes(e["path"]) - e["fixed"] - DISK_MARGIN_BYTES) // e["per"]
|
||||
for e in devices.values()
|
||||
)
|
||||
return max(0, room)
|
||||
|
||||
|
||||
def restore_space_message(size, room, *, known):
|
||||
"""The truthful refusal of a restore the disks have no room for."""
|
||||
if known:
|
||||
return (
|
||||
f"Not enough free disk space to restore a {size_label(size)} backup; "
|
||||
f"there is room for about {size_label(room)}. Nothing was changed."
|
||||
)
|
||||
return (
|
||||
"Not enough free disk space to restore this backup: it is larger than "
|
||||
f"the {size_label(room)} there is room for. Nothing was changed."
|
||||
)
|
||||
|
||||
|
||||
def short_space_message(short):
|
||||
"""The truthful refusal for ``disk_plan``'s ``short``."""
|
||||
place, what, required, free = short
|
||||
verb = "need" if " and " in what else "needs"
|
||||
return (
|
||||
f"Not enough free disk space: {what} {verb} about {_megabytes(required)} "
|
||||
f"in the {place} and {_megabytes(free)} is free. Nothing was changed."
|
||||
)
|
||||
|
||||
|
||||
def _changed_line(operation, count):
|
||||
if operation == "optimize":
|
||||
return "Optimized the database"
|
||||
if operation == "duplicate_prompts":
|
||||
noun = "duplicate prompt" if count == 1 else "duplicate prompts"
|
||||
return f"Merged {count:,} {noun}"
|
||||
if operation == "unused_components":
|
||||
noun = "unused prompt component" if count == 1 else "unused prompt components"
|
||||
return f"Removed {count:,} {noun}"
|
||||
noun = "link to a missing file" if count == 1 else "links to missing files"
|
||||
return f"Removed {count:,} {noun}"
|
||||
|
||||
|
||||
def _remove_quietly(*paths):
|
||||
for path in paths:
|
||||
try:
|
||||
if path and os.path.exists(path):
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _reserve_backup_name(db_path, label="backup"):
|
||||
"""Create and return an unused ``<db>.<label>_<stamp>[_n]`` (O_EXCL)."""
|
||||
for candidate in _backup_names(db_path, label):
|
||||
try:
|
||||
fd = os.open(candidate, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||
except FileExistsError:
|
||||
continue
|
||||
os.close(fd)
|
||||
return candidate
|
||||
raise SafetyBackupFailed("No free backup name")
|
||||
|
||||
|
||||
def _backup_names(db_path, label="backup"):
|
||||
base = f"{db_path}.{label}_{_now().strftime('%Y%m%d_%H%M%S_%f')}"
|
||||
for attempt in range(_NAME_ATTEMPTS):
|
||||
yield base if attempt == 0 else f"{base}_{attempt}"
|
||||
|
||||
|
||||
def _place_backup(temp, db_path, label="backup"):
|
||||
"""Give the verified ``temp`` its unique backup name; return that path.
|
||||
|
||||
``os.link`` is atomic and refuses an existing name, so no empty or half
|
||||
file ever carries a backup name, even across a crash. A filesystem
|
||||
without hard links falls back to reserving the name (``O_EXCL``) and
|
||||
``os.replace``. The temp file is removed by the caller.
|
||||
"""
|
||||
for candidate in _backup_names(db_path, label):
|
||||
try:
|
||||
os.link(temp, candidate)
|
||||
except FileExistsError:
|
||||
continue
|
||||
except (OSError, NotImplementedError):
|
||||
break # no hard links here
|
||||
else:
|
||||
return candidate
|
||||
else:
|
||||
raise SafetyBackupFailed("No free backup name")
|
||||
final = _reserve_backup_name(db_path, label)
|
||||
try:
|
||||
os.replace(temp, final)
|
||||
except OSError:
|
||||
_remove_quietly(final)
|
||||
raise
|
||||
return final
|
||||
|
||||
|
||||
SAFETY_TEMP_PREFIX = ".pm-safety-"
|
||||
|
||||
|
||||
def sweep_safety_temps(db_path):
|
||||
"""Remove the temp copies an interrupted safety backup left (blocking).
|
||||
|
||||
Called once at startup, before any job can take a backup. Returns the
|
||||
names removed.
|
||||
"""
|
||||
folder = os.path.dirname(os.path.abspath(db_path))
|
||||
removed = []
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
return removed
|
||||
for name in names:
|
||||
if not (name.startswith(SAFETY_TEMP_PREFIX) and ".tmp" in name):
|
||||
continue
|
||||
try:
|
||||
os.remove(os.path.join(folder, name))
|
||||
removed.append(name)
|
||||
except OSError:
|
||||
continue
|
||||
return removed
|
||||
|
||||
|
||||
def _standalone(path):
|
||||
"""Make the copy one self-contained file (rollback journal, not WAL).
|
||||
|
||||
The backup API copies the live database's WAL flag; a WAL file opened
|
||||
read-only later (a verify, a restore preview) would leave ``-wal`` and
|
||||
``-shm`` sidecars beside it.
|
||||
"""
|
||||
try:
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
conn.execute("PRAGMA journal_mode=DELETE").fetchone()
|
||||
except sqlite3.Error as exc:
|
||||
raise SafetyBackupFailed("The backup could not be finished") from exc
|
||||
|
||||
|
||||
def _fsync(path):
|
||||
with open(path, "rb+") as handle:
|
||||
os.fsync(handle.fileno())
|
||||
|
||||
|
||||
def take_safety_backup(model, label="backup"):
|
||||
"""Write and verify a safety backup; return its file name (blocking).
|
||||
|
||||
The copy is made in a temp file beside the database and only renamed
|
||||
onto its final, unique name ``<db>.<label>_<stamp>`` once it verified,
|
||||
so a backup file on disk is always a verified one. ``label`` "backup"
|
||||
joins the pruned pool; another label (the workflow storage migration's
|
||||
``pre-workflow-storage``) is kept until the user removes it. Raises
|
||||
SafetyBackupFailed, leaving no file.
|
||||
"""
|
||||
db_path = os.path.abspath(model.db_path)
|
||||
fd, temp = tempfile.mkstemp(
|
||||
dir=os.path.dirname(db_path), prefix=SAFETY_TEMP_PREFIX, suffix=".tmp"
|
||||
)
|
||||
os.close(fd)
|
||||
final = None
|
||||
try:
|
||||
if not model.backup_database(temp):
|
||||
raise SafetyBackupFailed("The backup could not be written")
|
||||
_standalone(temp)
|
||||
ok, reason = model.verify_database_file(temp)
|
||||
if not ok:
|
||||
raise SafetyBackupFailed("The backup did not verify", reason)
|
||||
_fsync(temp)
|
||||
if os.name != "nt":
|
||||
os.chmod(temp, 0o600)
|
||||
final = _place_backup(temp, db_path, label)
|
||||
return os.path.basename(final)
|
||||
except OSError as exc:
|
||||
_remove_quietly(final)
|
||||
raise SafetyBackupFailed("The backup could not be put in place") from exc
|
||||
except SafetyBackupFailed:
|
||||
_remove_quietly(final)
|
||||
raise
|
||||
finally:
|
||||
_remove_quietly(temp, temp + "-journal", temp + "-wal", temp + "-shm")
|
||||
|
||||
|
||||
def _failure(message, status, code=None, **extra):
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": message, **extra}, status=status
|
||||
)
|
||||
|
||||
|
||||
def _invalid(message):
|
||||
return _failure(message, 400, "invalid-request")
|
||||
|
||||
|
||||
def _iso(timestamp):
|
||||
return datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc).isoformat()
|
||||
|
||||
|
||||
class MaintenanceRoutesMixin:
|
||||
"""Mixin providing the /prompt_manager/maintenance plan endpoints."""
|
||||
|
||||
def _register_maintenance_routes(self, routes):
|
||||
@routes.post("/prompt_manager/maintenance/preview")
|
||||
async def maintenance_preview_route(request):
|
||||
return await self.preview_maintenance(request)
|
||||
|
||||
@routes.post("/prompt_manager/maintenance/apply")
|
||||
async def maintenance_apply_route(request):
|
||||
return await self.apply_maintenance_plan(request)
|
||||
|
||||
def _database_identity(self, db):
|
||||
"""What binds a plan to one database: the swap generation and its file."""
|
||||
path = os.path.normcase(os.path.realpath(db.model.db_path))
|
||||
return (self._db_generation, path)
|
||||
|
||||
@staticmethod
|
||||
def _safety_backup_policy():
|
||||
"""``on`` or ``off``: whether an apply takes a safety backup first."""
|
||||
from ..config import PromptManagerConfig
|
||||
|
||||
return "on" if PromptManagerConfig.SAFETY_BACKUP else "off"
|
||||
|
||||
async def _read_maintenance_body(self, request, allowed, cap=None):
|
||||
"""``(data, None)`` for a capped JSON object with only ``allowed`` keys.
|
||||
|
||||
``cap(data)``, when given, is the body's byte limit for what it asks
|
||||
(read up to ``DUPLICATE_PREVIEW_BODY_MAX``); else PREVIEW_BODY_MAX.
|
||||
"""
|
||||
limit = PREVIEW_BODY_MAX if cap is None else DUPLICATE_PREVIEW_BODY_MAX
|
||||
data, error = await self._read_json_body(request, max_bytes=limit)
|
||||
if error is not None:
|
||||
if error.status == 413:
|
||||
return None, _failure("Request body too large", 413, "too-large")
|
||||
return None, _invalid("Request body must be a JSON object")
|
||||
if cap is not None:
|
||||
size = len(json.dumps(data, separators=(",", ":")).encode("utf-8"))
|
||||
if size > cap(data):
|
||||
return None, _failure("Request body too large", 413, "too-large")
|
||||
unknown = sorted(set(data) - set(allowed))
|
||||
if unknown:
|
||||
return None, _invalid("Unknown field: " + ", ".join(unknown)[:200])
|
||||
return data, None
|
||||
|
||||
async def preview_maintenance(self, request):
|
||||
data, error = await self._read_maintenance_body(
|
||||
request,
|
||||
("operation", "options"),
|
||||
cap=lambda body: (
|
||||
DUPLICATE_PREVIEW_BODY_MAX
|
||||
if body.get("operation") == DUPLICATE_FILES
|
||||
else PREVIEW_BODY_MAX
|
||||
),
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
operation = data.get("operation")
|
||||
options = data.get("options", {})
|
||||
if not isinstance(operation, str) or operation not in PREVIEW_OPERATIONS:
|
||||
return _invalid(_OPERATION_ERROR)
|
||||
if operation == DUPLICATE_FILES:
|
||||
return await self.preview_duplicate_files(options)
|
||||
if not isinstance(options, dict) or options:
|
||||
return _invalid(f"{operation} takes no options")
|
||||
db = self.db
|
||||
identity = self._database_identity(db)
|
||||
try:
|
||||
roots = (
|
||||
await self._run_in_executor(self._health_roots)
|
||||
if operation == "missing_links"
|
||||
else []
|
||||
)
|
||||
draft = await self._run_in_executor(
|
||||
db.preview_maintenance, operation, roots
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception(f"Maintenance preview of {operation} failed")
|
||||
return _failure("Failed to review maintenance", 500)
|
||||
return await self._plan_response(
|
||||
db,
|
||||
identity,
|
||||
draft.operation,
|
||||
draft.fingerprint,
|
||||
draft.public(),
|
||||
{
|
||||
"targets": draft.targets,
|
||||
"roots": roots,
|
||||
"protected": draft.protected["count"],
|
||||
"rows": draft.scope["count"],
|
||||
},
|
||||
)
|
||||
|
||||
async def _plan_response(
|
||||
self, db, identity, operation, fingerprint, public, payload
|
||||
):
|
||||
"""Issue the reviewed plan's token and answer the ``Plan``.
|
||||
|
||||
Refused with ``storage-changed`` when the database was replaced while
|
||||
it was reviewed. The plan reports the safety backup's expected size
|
||||
and the disk space the apply needs (``disk_plan``).
|
||||
"""
|
||||
if identity != self._database_identity(self.db):
|
||||
return _failure(
|
||||
"The database was replaced while it was reviewed. Review again.",
|
||||
409,
|
||||
"storage-changed",
|
||||
)
|
||||
policy = self._safety_backup_policy()
|
||||
destination = "backup"
|
||||
if operation == WORKFLOW_STORAGE:
|
||||
# Always backed up first, whatever the policy (the way back);
|
||||
# one kept pre-migration backup ever, pooled ones after it.
|
||||
policy = "required"
|
||||
kept = await self._run_in_executor(pre_migration_backups, db.model.db_path)
|
||||
destination = "backup" if kept else PRE_MIGRATION_LABEL
|
||||
try:
|
||||
disk, _short, size = await self._run_in_executor(
|
||||
disk_plan,
|
||||
db.model.db_path,
|
||||
backup=policy != "off",
|
||||
vacuum=operation == "optimize",
|
||||
)
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Could not measure the disk space for a plan")
|
||||
return _failure("Failed to review maintenance", 500)
|
||||
store = self._maintenance_plans
|
||||
try:
|
||||
token = store.issue(
|
||||
operation, fingerprint, {**payload, "backup": policy}, identity
|
||||
)
|
||||
except ScopeTooLarge:
|
||||
return _failure(
|
||||
"This plan is too large to review at once.", 413, "too-large"
|
||||
)
|
||||
plan = store.peek(token, operation)
|
||||
name = os.path.basename(db.model.db_path)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"token": token,
|
||||
"expires_at": _iso(plan.expires_at),
|
||||
**public,
|
||||
"backup": {
|
||||
"policy": policy,
|
||||
"destination_label": f"{name}.{destination}_<date and time>",
|
||||
"expected_bytes": size if policy != "off" else None,
|
||||
},
|
||||
"disk": disk,
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
async def apply_maintenance_plan(self, request):
|
||||
data, error = await self._read_maintenance_body(request, ("token",))
|
||||
if error is not None:
|
||||
return error
|
||||
token = data.get("token")
|
||||
if not isinstance(token, str) or not 0 < len(token) <= MAX_TOKEN_LENGTH:
|
||||
return _invalid("token must be the plan's token")
|
||||
plan, refused = self._peek_plan(token)
|
||||
if refused is not None:
|
||||
return refused
|
||||
store = self._maintenance_plans
|
||||
db = self.db
|
||||
if plan.db_identity != self._database_identity(db):
|
||||
store.discard(token)
|
||||
return _failure(
|
||||
"The database was replaced since the review. Review again.",
|
||||
409,
|
||||
"storage-changed",
|
||||
)
|
||||
if plan.operation == WORKFLOW_STORAGE and not plan.payload.get("rows"):
|
||||
store.discard(token) # no job, no backup: nothing to rewrite
|
||||
return _failure(NOTHING_TO_DO, 409, "nothing-to-do")
|
||||
backup = self._backup_promised(plan)
|
||||
try:
|
||||
_disk, short, _size = await self._run_in_executor(
|
||||
disk_plan,
|
||||
db.model.db_path,
|
||||
backup=backup,
|
||||
vacuum=plan.operation == "optimize",
|
||||
)
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Could not measure the disk space for an apply")
|
||||
return _failure(APPLY_FAILED, 500)
|
||||
if short is not None:
|
||||
# Refused before any job starts: nothing fails halfway for space.
|
||||
return _failure(short_space_message(short), 507, "insufficient-space")
|
||||
# The token may have been used, expired or dropped while the disk was
|
||||
# measured: look again with no await before the job starts.
|
||||
plan, refused = self._peek_plan(token)
|
||||
if refused is not None:
|
||||
return refused
|
||||
try:
|
||||
if plan.operation == DUPLICATE_FILES:
|
||||
job = self._start_duplicate_cleanup(db, plan, backup)
|
||||
elif plan.operation == WORKFLOW_STORAGE:
|
||||
job = self._start_workflow_storage(db, backup)
|
||||
else:
|
||||
job = self._jobs.start(
|
||||
MAINTENANCE_JOB_KIND,
|
||||
MAINTENANCE_TITLES[plan.operation],
|
||||
# VACUUM rewrites the whole file under the write lock: it
|
||||
# runs alone, beside no other job of any class.
|
||||
admission=(
|
||||
"exclusive" if plan.operation == "optimize" else "records"
|
||||
),
|
||||
cancellable=False,
|
||||
scope={"operation": plan.operation},
|
||||
groups=MAINTENANCE_GROUPS,
|
||||
runner=lambda job: self._run_maintenance_plan(
|
||||
job, db, plan, backup
|
||||
),
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy) # the token stays usable
|
||||
# Same loop turn as the peek and the start: the runner has not run.
|
||||
store.consume(token, plan.operation, job_id=job.id)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
def _peek_plan(self, token):
|
||||
"""``(plan, None)`` for a usable token, else ``(None, refusal)``."""
|
||||
try:
|
||||
return self._maintenance_plans.peek(token), None
|
||||
except PlanUsed as used:
|
||||
job = self._jobs.get(used.job_id)
|
||||
return None, _failure(
|
||||
"This plan was already applied.",
|
||||
409,
|
||||
"plan-used",
|
||||
job={"id": used.job_id, "title": job.title if job else None},
|
||||
)
|
||||
except PlanExpired:
|
||||
return None, _failure(
|
||||
"This review expired. Review the plan again.", 410, "plan-expired"
|
||||
)
|
||||
|
||||
def _backup_promised(self, plan):
|
||||
"""A plan reviewed with the policy on keeps that promise; so does on now.
|
||||
|
||||
Decided once, when the apply starts (the disk check measured it), and
|
||||
handed to the job.
|
||||
"""
|
||||
if plan.operation == WORKFLOW_STORAGE:
|
||||
return True # never without its pre-migration backup
|
||||
return plan.payload["backup"] == "on" or self._safety_backup_policy() == "on"
|
||||
|
||||
async def _safety_backup_step(self, job, db, backup_on):
|
||||
"""``(ok, name)``: take the promised safety backup, or report none.
|
||||
|
||||
``name`` is the backup's file name, or None when no backup was
|
||||
promised. A failed backup ends the job ``backup-failed`` (``ok``
|
||||
False).
|
||||
"""
|
||||
if not backup_on:
|
||||
job.log_line("No safety backup (the policy is off)")
|
||||
return True, None
|
||||
job.update(status_line="Taking a safety backup")
|
||||
job.log_line("Taking a safety backup of the database")
|
||||
try:
|
||||
backup = await self._run_in_executor(take_safety_backup, db.model)
|
||||
except SafetyBackupFailed as error:
|
||||
self.logger.error("Safety backup failed", exc_info=True)
|
||||
_fail_job(job, "backup-failed", backup_failed_message(error))
|
||||
return False, None
|
||||
job.log_line(f"Safety backup {backup} verified")
|
||||
return True, backup
|
||||
|
||||
async def _prune_backup_pool(self, job, db):
|
||||
"""Keep the one ``<db>.backup_*`` pool (restore's too) at its cap."""
|
||||
removed = await self._run_in_executor(_prune_safety_backups, db.model.db_path)
|
||||
if removed:
|
||||
job.log_line(f"Removed {len(removed)} older safety backups")
|
||||
|
||||
async def _run_maintenance_plan(self, job, db, plan, backup_on):
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return
|
||||
try:
|
||||
await self._apply_maintenance_plan(job, db, plan, backup)
|
||||
finally:
|
||||
if backup:
|
||||
await self._prune_backup_pool(job, db)
|
||||
|
||||
async def _apply_maintenance_plan(self, job, db, plan, backup):
|
||||
payload = plan.payload
|
||||
job.update(status_line=f"Applying: {job.title}")
|
||||
try:
|
||||
result = await self._run_in_executor(
|
||||
db.apply_maintenance,
|
||||
plan.operation,
|
||||
plan.fingerprint,
|
||||
payload["targets"],
|
||||
payload["roots"],
|
||||
)
|
||||
except PlanChanged:
|
||||
return _fail_job(job, "plan-changed", PLAN_CHANGED)
|
||||
except Exception:
|
||||
self.logger.exception(f"Maintenance {plan.operation} failed")
|
||||
return _fail_job(job, None, APPLY_FAILED)
|
||||
rows = await self._run_in_executor(
|
||||
self._maintenance_rows, plan.operation, result["rows"]
|
||||
)
|
||||
job.record_counted("changed", result["changed"], rows)
|
||||
job.record_counted("protected", payload["protected"])
|
||||
line = _changed_line(plan.operation, result["changed"])
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
|
||||
def _maintenance_rows(self, operation, described):
|
||||
"""Outcome rows for the changed records (root-relative paths only)."""
|
||||
if operation == "missing_links":
|
||||
roots = _scan_roots(self._get_all_output_dirs())
|
||||
rows = []
|
||||
for item in described:
|
||||
path, label, index = _scan_location(item["image_path"], roots)
|
||||
rows.append(
|
||||
{
|
||||
"path": path,
|
||||
"root": label,
|
||||
"root_index": index,
|
||||
"detail": "Link removed; the file was already missing",
|
||||
}
|
||||
)
|
||||
return rows
|
||||
if operation == "duplicate_prompts":
|
||||
return [
|
||||
{
|
||||
"path": "",
|
||||
"detail": (
|
||||
f"Prompt {item['prompt_id']} merged into "
|
||||
f"prompt {item['kept_id']}"
|
||||
),
|
||||
}
|
||||
for item in described
|
||||
]
|
||||
return [
|
||||
{"path": "", "detail": f"Removed: {item.get('text') or item['prompt_id']}"}
|
||||
for item in described
|
||||
]
|
||||
|
||||
|
||||
def backup_note(backup):
|
||||
"""The status line's backup part: the file name, or that there is none."""
|
||||
return f"Safety backup {backup}" if backup else NO_BACKUP
|
||||
|
||||
|
||||
def _fail_job(job, code, message):
|
||||
"""End ``job`` failed with ``message`` (one error row) and its code."""
|
||||
job.record("errors", {"path": "", "detail": message}, f"! {message}")
|
||||
job.update(status_line=message)
|
||||
job.publish({"type": "error", "code": code, "message": message})
|
||||
job.finish("failed")
|
||||
@@ -0,0 +1,138 @@
|
||||
"""The reviewed-plan store behind maintenance previews (#249 System S6).
|
||||
|
||||
A preview issues a plan: its operation, the fingerprint of exactly what was
|
||||
reviewed, the payload an apply needs (target ids, the sources it judged) and
|
||||
the identity of the database it describes. Everything is held here, in
|
||||
server memory, under a single-use token (``secrets.token_urlsafe(32)``); the
|
||||
client only ever sees the token and the public plan. Plans expire after
|
||||
``ttl`` seconds and are gone after a restart.
|
||||
|
||||
The store keeps at most ``max_plans`` plans and ``max_bytes`` of payload
|
||||
(measured as compact JSON). A plan larger than ``max_bytes`` is refused with
|
||||
``ScopeTooLarge``: it is never cut down to fit, since an apply must do
|
||||
exactly what was reviewed. A new plan that fits makes room by dropping the
|
||||
oldest ones, which then read as expired.
|
||||
|
||||
A consumed token is remembered (the last ``MAX_USED``) with the job it
|
||||
started, so a second apply of it is told which job ran it.
|
||||
"""
|
||||
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
class PlanExpired(Exception):
|
||||
"""No live plan has the token: unknown, expired, dropped or another operation's."""
|
||||
|
||||
|
||||
class PlanUsed(Exception):
|
||||
"""The token was already applied; ``job_id`` is the job that applied it."""
|
||||
|
||||
def __init__(self, job_id):
|
||||
super().__init__("This plan was already applied")
|
||||
self.job_id = job_id
|
||||
|
||||
|
||||
class ScopeTooLarge(Exception):
|
||||
"""The plan's payload exceeds the store's byte budget."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class StoredPlan:
|
||||
"""One reviewed plan, as only the server sees it."""
|
||||
|
||||
operation: str
|
||||
fingerprint: str
|
||||
payload: dict
|
||||
db_identity: tuple
|
||||
expires_at: float # wall-clock seconds since the epoch, for the response
|
||||
deadline: float = field(repr=False) # on the store's monotonic clock
|
||||
size: int = field(repr=False)
|
||||
|
||||
|
||||
def _payload_size(payload):
|
||||
return len(json.dumps(payload, separators=(",", ":"), default=str))
|
||||
|
||||
|
||||
class PlanStore:
|
||||
"""Reviewed maintenance plans under single-use tokens."""
|
||||
|
||||
MAX_USED = 64 # consumed tokens remembered for plan-used answers
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ttl=900,
|
||||
max_plans=8,
|
||||
max_bytes=16 * 2**20,
|
||||
*,
|
||||
clock=time.monotonic,
|
||||
wall_clock=time.time,
|
||||
):
|
||||
self.ttl = ttl
|
||||
self.max_plans = max_plans
|
||||
self.max_bytes = max_bytes
|
||||
self._clock = clock
|
||||
self._wall_clock = wall_clock
|
||||
self._plans = OrderedDict() # token -> StoredPlan, oldest first
|
||||
self._used = OrderedDict() # token -> job id, oldest first
|
||||
|
||||
def issue(self, operation, fingerprint, payload, db_identity) -> str:
|
||||
"""Hold a plan and return its token; ScopeTooLarge when it cannot fit."""
|
||||
size = _payload_size(payload)
|
||||
if size > self.max_bytes:
|
||||
raise ScopeTooLarge(f"The plan needs {size} bytes")
|
||||
self._drop_expired()
|
||||
while self._plans and (
|
||||
len(self._plans) >= self.max_plans
|
||||
or self._held_bytes() + size > self.max_bytes
|
||||
):
|
||||
self._plans.popitem(last=False)
|
||||
token = secrets.token_urlsafe(32)
|
||||
self._plans[token] = StoredPlan(
|
||||
operation=operation,
|
||||
fingerprint=fingerprint,
|
||||
payload=payload,
|
||||
db_identity=tuple(db_identity),
|
||||
expires_at=self._wall_clock() + self.ttl,
|
||||
deadline=self._clock() + self.ttl,
|
||||
size=size,
|
||||
)
|
||||
return token
|
||||
|
||||
def peek(self, token, operation=None) -> StoredPlan:
|
||||
"""The live plan of ``token``, left in place; raises PlanUsed or PlanExpired."""
|
||||
if not isinstance(token, str):
|
||||
raise PlanExpired("No such plan")
|
||||
if token in self._used:
|
||||
raise PlanUsed(self._used[token])
|
||||
plan = self._plans.get(token)
|
||||
if plan is None or (operation is not None and plan.operation != operation):
|
||||
raise PlanExpired("No such plan")
|
||||
if self._clock() > plan.deadline:
|
||||
del self._plans[token]
|
||||
raise PlanExpired("The plan expired")
|
||||
return plan
|
||||
|
||||
def consume(self, token, operation=None, *, job_id=None) -> StoredPlan:
|
||||
"""Take the plan of ``token`` once; later calls raise PlanUsed(job_id)."""
|
||||
plan = self.peek(token, operation)
|
||||
del self._plans[token]
|
||||
self._used[token] = job_id
|
||||
while len(self._used) > self.MAX_USED:
|
||||
self._used.popitem(last=False)
|
||||
return plan
|
||||
|
||||
def discard(self, token):
|
||||
"""Forget a plan that can no longer apply (its database was replaced)."""
|
||||
self._plans.pop(token, None)
|
||||
|
||||
def _held_bytes(self):
|
||||
return sum(plan.size for plan in self._plans.values())
|
||||
|
||||
def _drop_expired(self):
|
||||
now = self._clock()
|
||||
for token in [t for t, plan in self._plans.items() if now > plan.deadline]:
|
||||
del self._plans[token]
|
||||
@@ -0,0 +1,713 @@
|
||||
"""Needs review routes (#249 E3): the pending queue, item detail, decisions.
|
||||
|
||||
``GET /prompt_manager/review/items`` lists pending items oldest first behind
|
||||
an HMAC keyset cursor bound to the storage epoch only (a restore makes an
|
||||
old cursor stale; a decision or a new item does not, because the
|
||||
``(created_at, id)`` keyset stays ordered when items leave or are appended);
|
||||
``total`` is the exact pending count. ``GET .../items/{id}`` adds the
|
||||
candidates and the dependency fingerprint. ``POST .../items/{id}/decision``
|
||||
records one explicit decision: Apply links the chosen sampler's pair only
|
||||
when the capture is the item's bytes, the current file still hashes to them,
|
||||
the evidence they yield is unchanged and the ingester's compatible-only rules
|
||||
allow it; Leave unresolved links nothing.
|
||||
|
||||
Targets are formatted from the recorded spelling against the current roots
|
||||
(no file access beyond the roots' real paths); a file under no current root
|
||||
is ``unavailable``. Responses carry no server path, no locator and no
|
||||
exception text.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib.parse
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .annotations import _HEX64, _TOKEN, _UUID, _capture_error, _error
|
||||
from .images import media_thumbnail_url
|
||||
from .inspection_service import VIDEO_EXTENSIONS
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
from ...database.library import display_title
|
||||
from ...database.library_commands import LibraryCommandError
|
||||
from ...database.review_decisions import ReviewCommandError
|
||||
from ...utils.media_capture import CaptureError
|
||||
from ...utils.review_capture import (
|
||||
captured_review,
|
||||
item_profile,
|
||||
review_evidence,
|
||||
)
|
||||
except ImportError:
|
||||
from database.library import display_title
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.review_decisions import ReviewCommandError
|
||||
from utils.media_capture import CaptureError
|
||||
from utils.review_capture import captured_review, item_profile, review_evidence
|
||||
|
||||
ITEMS_URL = "/prompt_manager/review/items"
|
||||
DEFAULT_LIMIT = 40
|
||||
MAX_LIMIT = 40
|
||||
MAX_CURSOR = 4096
|
||||
MAX_BODY_BYTES = 64 * 1024
|
||||
# The serialized detail stays under this; past it the detail is ``limited``.
|
||||
MAX_DETAIL_BYTES = 8 * 1024 * 1024
|
||||
UNRESOLVED = "unresolved"
|
||||
_DECISION_FIELDS = frozenset(
|
||||
{
|
||||
"expected_epoch",
|
||||
"expected_revision",
|
||||
"capture_token",
|
||||
"sha256",
|
||||
"dependency_fingerprint",
|
||||
"choice",
|
||||
}
|
||||
)
|
||||
_CURSOR_SECRET = os.urandom(32)
|
||||
# The real path of a spelling; module-level so a test can fake a junction.
|
||||
_realpath = os.path.realpath
|
||||
_LIMIT = re.compile(r"[0-9]{1,3}")
|
||||
_NO_STORE = {"Cache-Control": "no-store"}
|
||||
|
||||
_STATUS = {
|
||||
"review-missing": 404,
|
||||
"review-conflict": 409,
|
||||
"association-conflict": 409,
|
||||
"file-changed": 409,
|
||||
"storage-changed": 409,
|
||||
"source-changed": 409,
|
||||
"capture-expired": 409,
|
||||
}
|
||||
_MESSAGES = {
|
||||
"review-missing": "This review item no longer exists. Reload the queue.",
|
||||
"review-conflict": (
|
||||
"This item was decided or changed since it was opened. Reload the queue."
|
||||
),
|
||||
"association-conflict": (
|
||||
"This image is already linked to another prompt or negative, so this "
|
||||
"sampler path cannot be attached safely. The existing links were "
|
||||
"preserved."
|
||||
),
|
||||
"dependencies-changed": (
|
||||
"The image's saved links changed since this item was opened. Reopen it "
|
||||
"to see the current association."
|
||||
),
|
||||
"file-changed": (
|
||||
"The file changed since it was reviewed. A metadata scan adds its new "
|
||||
"bytes as a new item."
|
||||
),
|
||||
"storage-changed": "Prompt storage was restored; reload the review queue.",
|
||||
"source-changed": (
|
||||
"The source changed. Refresh the image collection before reviewing this "
|
||||
"file."
|
||||
),
|
||||
"capture-expired": "The preview expired. Open the item again.",
|
||||
"other-file": "The preview belongs to another file. Open the item again.",
|
||||
"evidence-changed": (
|
||||
"This file's metadata no longer gives the evidence that was reviewed. "
|
||||
"Reload the queue."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
_DEFAULT_MESSAGE = "The decision could not be saved. Reload the queue."
|
||||
|
||||
|
||||
def _coded(code: str, message_key: str | None = None, **extra) -> web.Response:
|
||||
message = _MESSAGES.get(message_key or code, _DEFAULT_MESSAGE)
|
||||
return _error(_STATUS.get(code, 409), code, message, **extra)
|
||||
|
||||
|
||||
def _failed() -> web.Response:
|
||||
return _error(500, "review-failed", "The review queue could not be read.")
|
||||
|
||||
|
||||
def format_review_target(spelled_path, locator, real_roots, *, pathmod=os.path):
|
||||
"""``(root_index, relative)`` of the first root that lists the file, or None.
|
||||
|
||||
``spelled_path`` is the file's recorded spelling, ``locator`` its folded
|
||||
identity and ``real_roots`` ``(index, realpath)`` pairs. The relative
|
||||
path keeps the recorded spelling (a Windows locator is case-folded, a
|
||||
listing target is not) and must fold back to the locator.
|
||||
"""
|
||||
sep = pathmod.sep
|
||||
for index, real_root in real_roots:
|
||||
head = spelled_path[: len(real_root)]
|
||||
if pathmod.normcase(head) != pathmod.normcase(real_root):
|
||||
continue
|
||||
rest = spelled_path[len(real_root) :]
|
||||
if not real_root.endswith(sep):
|
||||
if not rest.startswith(sep):
|
||||
continue
|
||||
rest = rest[1:]
|
||||
parts = rest.split(sep)
|
||||
if any(
|
||||
part in ("", ".", "..") or "/" in part or "\\" in part or ":" in part
|
||||
for part in parts
|
||||
):
|
||||
continue
|
||||
if pathmod.normcase(pathmod.join(real_root, *parts)) != locator:
|
||||
continue
|
||||
return index, "/".join(parts)
|
||||
return None
|
||||
|
||||
|
||||
class _StaleCursor(ValueError):
|
||||
"""A well-formed cursor this process cannot verify (e.g. after a restart)."""
|
||||
|
||||
|
||||
def _encode_cursor(epoch: str, key: tuple) -> str:
|
||||
payload = json.dumps(
|
||||
{"v": 2, "e": epoch, "k": list(key)},
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
).encode()
|
||||
signature = hmac.new(_CURSOR_SECRET, payload, hashlib.sha256).digest()
|
||||
return base64.urlsafe_b64encode(signature + payload).decode().rstrip("=")
|
||||
|
||||
|
||||
def _decode_cursor(token: str) -> tuple:
|
||||
"""``(epoch, (created_at, id))`` of a cursor.
|
||||
|
||||
``ValueError`` when malformed; :class:`_StaleCursor` when well-formed but
|
||||
signed by another secret (the per-process secret changes on every start,
|
||||
so an open page's cursor is stale after a restart, not forged).
|
||||
"""
|
||||
try:
|
||||
if len(token) > MAX_CURSOR:
|
||||
raise ValueError()
|
||||
raw = base64.b64decode(
|
||||
token + "=" * (-len(token) % 4), altchars=b"-_", validate=True
|
||||
)
|
||||
signature, payload = raw[:32], raw[32:]
|
||||
if len(signature) != 32 or not payload:
|
||||
raise ValueError()
|
||||
expected = hmac.new(_CURSOR_SECRET, payload, hashlib.sha256).digest()
|
||||
if not hmac.compare_digest(signature, expected):
|
||||
raise _StaleCursor("This page of the queue is out of date")
|
||||
data = json.loads(payload)
|
||||
key = data["k"]
|
||||
if (
|
||||
data["v"] != 2
|
||||
or not isinstance(data["e"], str)
|
||||
or not isinstance(key, list)
|
||||
or len(key) != 2
|
||||
or not all(isinstance(part, str) for part in key)
|
||||
):
|
||||
raise ValueError()
|
||||
return data["e"], tuple(key)
|
||||
except _StaleCursor:
|
||||
raise
|
||||
except (ValueError, KeyError, TypeError, UnicodeError, binascii.Error):
|
||||
raise ValueError("Invalid review cursor; reload the queue") from None
|
||||
|
||||
|
||||
def _limit(value):
|
||||
if value is None:
|
||||
return DEFAULT_LIMIT
|
||||
if _LIMIT.fullmatch(value) is None or not 1 <= int(value) <= MAX_LIMIT:
|
||||
return None
|
||||
return int(value)
|
||||
|
||||
|
||||
def _side_text(side) -> str | None:
|
||||
if isinstance(side, dict) and isinstance(side.get("text"), str):
|
||||
return side["text"]
|
||||
return None
|
||||
|
||||
|
||||
def _workable(candidate: dict) -> bool:
|
||||
return (
|
||||
candidate.get("state") == "resolved"
|
||||
and _side_text(candidate.get("positive")) is not None
|
||||
and _side_text(candidate.get("negative")) is not None
|
||||
)
|
||||
|
||||
|
||||
def _side_label(titles: dict, side, empty: str) -> str:
|
||||
"""The Library title of the side's saved prompt, else its first line."""
|
||||
text = _side_text(side)
|
||||
if text is None:
|
||||
return empty
|
||||
stored = side.get("storage_text")
|
||||
stored = stored if isinstance(stored, str) else text
|
||||
return display_title(titles.get(stored, ()), text) or empty
|
||||
|
||||
|
||||
class ReviewRoutesMixin:
|
||||
"""The Needs review queue over E1's schema and E2's recorded items."""
|
||||
|
||||
def _register_review_routes(self, routes: web.RouteTableDef) -> None:
|
||||
routes.get(ITEMS_URL)(self.list_review_items)
|
||||
routes.get(ITEMS_URL + "/{item_id}")(self.get_review_item)
|
||||
routes.post(ITEMS_URL + "/{item_id}/decision")(self.decide_review_item)
|
||||
|
||||
# -- shared ----------------------------------------------------------------
|
||||
|
||||
def _review_roots(self):
|
||||
"""``(roots, real_roots)``: the current gallery roots and their realpaths.
|
||||
|
||||
Blocking (it lists the roots and resolves them): call it on a worker.
|
||||
"""
|
||||
roots = list(self._get_all_output_dirs())
|
||||
return roots, [(i, _realpath(str(root))) for i, root in enumerate(roots)]
|
||||
|
||||
def _review_view(self, row: dict, roots, real_roots) -> dict:
|
||||
"""The public ReviewItem of one stored row (no path, no locator)."""
|
||||
found = format_review_target(row["spelled_path"], row["locator"], real_roots)
|
||||
target = url = thumbnail_url = None
|
||||
if found is not None:
|
||||
index, relative = found
|
||||
target = {
|
||||
"root_index": index,
|
||||
"root_scope": self._gallery_root_scope(roots[index], index),
|
||||
"path": relative,
|
||||
}
|
||||
# The Images listing's media URL for the same file (formatting only).
|
||||
url = (
|
||||
"/prompt_manager/images/serve/"
|
||||
f"{urllib.parse.quote(relative, safe='/')}?root={index}"
|
||||
f"&root_scope={target['root_scope']}"
|
||||
)
|
||||
# The Images listing's thumbnail, when one exists (one stat).
|
||||
thumbnail_url = media_thumbnail_url(
|
||||
roots[index], relative, index, root_scope=target["root_scope"]
|
||||
)
|
||||
filename = os.path.basename(row["spelled_path"])
|
||||
return {
|
||||
"id": row["id"],
|
||||
"state": row["state"],
|
||||
"revision": row["revision"],
|
||||
"created_at": row["created_at"],
|
||||
"filename": filename,
|
||||
"target": target,
|
||||
"unavailable": target is None,
|
||||
"candidate_count": row["candidate_count"],
|
||||
"limited": row["limited"],
|
||||
"url": url,
|
||||
"thumbnail_url": thumbnail_url,
|
||||
"is_video": os.path.splitext(filename)[1].lower() in VIDEO_EXTENSIONS,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _review_image_path(row: dict, roots, target) -> str:
|
||||
"""The path the scan and the monitor link this file under."""
|
||||
if target is None:
|
||||
return row["spelled_path"]
|
||||
root = roots[target["root_index"]]
|
||||
return os.path.join(str(root), *target["path"].split("/"))
|
||||
|
||||
# -- queue -----------------------------------------------------------------
|
||||
|
||||
async def list_review_items(self, request: web.Request) -> web.Response:
|
||||
"""GET ``?limit=1..40&cursor=``: one page of pending items."""
|
||||
limit = _limit(request.query.get("limit"))
|
||||
if limit is None:
|
||||
return bad_request(f"limit must be an integer from 1 to {MAX_LIMIT}")
|
||||
bound = None
|
||||
cursor = request.query.get("cursor")
|
||||
if cursor:
|
||||
try:
|
||||
bound = _decode_cursor(cursor)
|
||||
except _StaleCursor:
|
||||
return await self._stale_cursor()
|
||||
except ValueError as exc:
|
||||
return bad_request(str(exc))
|
||||
try:
|
||||
page, views = await self._run_in_executor(
|
||||
self._review_page, limit, bound[1] if bound else None
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Review queue read failed")
|
||||
return _failed()
|
||||
if bound is not None and bound[0] != page["epoch"]:
|
||||
return _coded("storage-changed", library_epoch=page["epoch"])
|
||||
items = page["items"]
|
||||
next_cursor = None
|
||||
if page["more"] and items:
|
||||
last = items[-1]
|
||||
next_cursor = _encode_cursor(
|
||||
page["epoch"], (last["created_at"], last["id"])
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"library_epoch": page["epoch"],
|
||||
# F3: the queue version the change stream's ``ready`` names.
|
||||
"review_revision": page["revision"],
|
||||
"total": page["total"],
|
||||
"items": views,
|
||||
"next_cursor": next_cursor,
|
||||
},
|
||||
headers=_NO_STORE,
|
||||
)
|
||||
|
||||
async def _stale_cursor(self) -> web.Response:
|
||||
"""409 ``review-conflict`` with the current total: reload the queue."""
|
||||
try:
|
||||
page = await self._run_in_executor(self.db.review_queue_page, limit=1)
|
||||
except Exception:
|
||||
self.logger.exception("Review queue read failed")
|
||||
return _failed()
|
||||
return _coded(
|
||||
"review-conflict", total=page["total"], library_epoch=page["epoch"]
|
||||
)
|
||||
|
||||
def _review_page(self, limit: int, after):
|
||||
"""Worker thread: one queue page and the public views of its items."""
|
||||
page = self.db.review_queue_page(limit=limit, after=after)
|
||||
roots, real_roots = self._review_roots()
|
||||
return page, [
|
||||
self._review_view(row, roots, real_roots) for row in page["items"]
|
||||
]
|
||||
|
||||
# -- detail ----------------------------------------------------------------
|
||||
|
||||
async def get_review_item(self, request: web.Request) -> web.Response:
|
||||
"""GET one item with its candidates and dependency fingerprint."""
|
||||
try:
|
||||
read = await self._run_in_executor(
|
||||
self._review_detail, request.match_info["item_id"]
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Review item read failed")
|
||||
return _failed()
|
||||
if read is None:
|
||||
return _coded("review-missing")
|
||||
item, view, fingerprint, candidates = read
|
||||
detail = {
|
||||
"success": True,
|
||||
"library_epoch": item["epoch"],
|
||||
**view,
|
||||
"candidates": candidates,
|
||||
"dependency_fingerprint": fingerprint,
|
||||
"sha256": item["sha256"],
|
||||
"total": item["total"],
|
||||
}
|
||||
if len(json.dumps(detail).encode("utf-8")) > MAX_DETAIL_BYTES:
|
||||
detail = {**detail, "candidates": [], "limited": True}
|
||||
return web.json_response(detail, headers=_NO_STORE)
|
||||
|
||||
def _review_detail(self, item_id: str):
|
||||
"""Worker thread: ``(item, view, fingerprint, candidates)`` or None."""
|
||||
item = self.db.review_item(item_id)
|
||||
if item is None:
|
||||
return None
|
||||
roots, real_roots = self._review_roots()
|
||||
view = self._review_view(item, roots, real_roots)
|
||||
image_path = self._review_image_path(item, roots, view["target"])
|
||||
aliases, _known = physical_spellings(self.db, image_path, item["locator"])
|
||||
fingerprint = self.db.review_dependency_fingerprint(image_path, aliases)
|
||||
return item, view, fingerprint, self._review_candidates(item)
|
||||
|
||||
def _review_candidates(self, item: dict) -> list:
|
||||
"""The detail's candidates: role texts, a label and selectability."""
|
||||
if item["limited"]:
|
||||
return []
|
||||
candidates = [
|
||||
c for c in item["evidence"].get("candidates", []) if isinstance(c, dict)
|
||||
]
|
||||
texts = set()
|
||||
for candidate in candidates:
|
||||
for role in ("positive", "negative"):
|
||||
side = candidate.get(role)
|
||||
if isinstance(side, dict):
|
||||
for key in ("text", "storage_text"):
|
||||
if isinstance(side.get(key), str):
|
||||
texts.add(side[key])
|
||||
titles = self.db.stored_prompt_titles(texts)
|
||||
rows = []
|
||||
for candidate in candidates:
|
||||
positive = _side_text(candidate.get("positive"))
|
||||
negative = _side_text(candidate.get("negative"))
|
||||
label = (
|
||||
_side_label(titles, candidate.get("positive"), "No positive")
|
||||
+ " / "
|
||||
+ _side_label(titles, candidate.get("negative"), "No negative")
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"hash": candidate.get("hash"),
|
||||
"sampler_id": candidate.get("sampler_id"),
|
||||
"state": "resolved" if _workable(candidate) else "unresolved",
|
||||
"reason": candidate.get("reason") or "",
|
||||
"label": label,
|
||||
"positive": {"text": positive} if positive is not None else None,
|
||||
"negative": {"text": negative} if negative is not None else None,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
# -- decision --------------------------------------------------------------
|
||||
|
||||
async def decide_review_item(self, request: web.Request) -> web.Response:
|
||||
"""POST one explicit decision on one pending item."""
|
||||
data, error = await self._read_json_body(request, max_bytes=MAX_BODY_BYTES)
|
||||
if error is not None:
|
||||
return error
|
||||
guards, error = _decision_guards(data)
|
||||
if error is not None:
|
||||
return error
|
||||
try:
|
||||
item = await self._run_in_executor(
|
||||
self.db.review_item, request.match_info["item_id"]
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Review item read failed")
|
||||
return _failed()
|
||||
if item is None:
|
||||
return _coded("review-missing")
|
||||
choice = guards["choice"]
|
||||
candidate = None
|
||||
if choice != UNRESOLVED:
|
||||
candidate = next(
|
||||
(
|
||||
c
|
||||
for c in item["evidence"].get("candidates", [])
|
||||
if isinstance(c, dict) and c.get("hash") == choice
|
||||
),
|
||||
None,
|
||||
)
|
||||
if candidate is None or item["limited"]:
|
||||
return bad_request("choice is not one of this item's candidates")
|
||||
if not _workable(candidate):
|
||||
return bad_request("This sampler path cannot be applied")
|
||||
if guards["capture_token"] is None:
|
||||
return bad_request("Apply needs a capture of the file")
|
||||
verified = None
|
||||
if guards["capture_token"] is not None:
|
||||
verified, error = await self._verified_review_capture(
|
||||
item, guards, need_metadata=candidate is not None
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
elif guards["sha256"] != item["sha256"]:
|
||||
return bad_request("sha256 does not match the item")
|
||||
return await self._record_decision(item, guards, candidate, verified)
|
||||
|
||||
async def _verified_review_capture(self, item, guards, *, need_metadata):
|
||||
"""``((capture, facts, image_path, metadata), None)`` or ``(None, error)``.
|
||||
|
||||
The capture must be the item's file and bytes, under the expected
|
||||
epoch and a current root; the current file must re-hash to the same
|
||||
bytes; for Apply, the capture's metadata must still yield exactly the
|
||||
reviewed evidence.
|
||||
"""
|
||||
store = self._captures
|
||||
capture = store.get(guards["capture_token"]) if store is not None else None
|
||||
if capture is None:
|
||||
return None, _coded("capture-expired")
|
||||
if capture.locator != item["locator"]:
|
||||
return None, _coded("capture-expired", "other-file")
|
||||
if guards["sha256"] != capture.sha256:
|
||||
return None, bad_request("sha256 does not match the preview")
|
||||
if capture.sha256 != item["sha256"]:
|
||||
return None, _coded("file-changed")
|
||||
if guards["expected_epoch"] != capture.epoch:
|
||||
current = await self._run_in_executor(self.db.get_execution_epoch)
|
||||
return None, _coded("storage-changed", library_epoch=current)
|
||||
try:
|
||||
root, error = await self._run_in_executor(
|
||||
self._scoped_root, capture.root_index, capture.scope
|
||||
)
|
||||
if error is not None:
|
||||
return None, error
|
||||
read = await self._run_in_executor(
|
||||
_verify_and_read, store, capture, item, root, need_metadata
|
||||
)
|
||||
except CaptureError as exc:
|
||||
return None, _capture_error(exc)
|
||||
if read is None:
|
||||
return None, _coded("source-changed")
|
||||
relative, facts, captured = read
|
||||
if facts.locator != item["locator"]:
|
||||
return None, _coded("capture-expired", "other-file")
|
||||
if facts.sha256 != item["sha256"]:
|
||||
return None, _coded("file-changed")
|
||||
metadata = None
|
||||
if need_metadata:
|
||||
if captured is None or not _candidate_still_offered(
|
||||
captured[1], guards["choice"]
|
||||
):
|
||||
return None, _coded("review-conflict", "evidence-changed")
|
||||
metadata = captured[0]
|
||||
image_path = os.path.join(str(root), *relative.split("/"))
|
||||
return (capture, facts, image_path, metadata), None
|
||||
|
||||
def _decide_sync(self, item, guards, candidate, verified):
|
||||
"""Worker thread: resolve the file's spellings, decide, build the view.
|
||||
|
||||
Returns ``(result, view)``; a refusal raises with ``exc.view`` set.
|
||||
"""
|
||||
_capture, facts, image_path, metadata = verified or (None, None, None, None)
|
||||
aliases, known = (), None
|
||||
if candidate is not None:
|
||||
aliases, known = physical_spellings(self.db, image_path, item["locator"])
|
||||
try:
|
||||
result = self.db.decide_review_item(
|
||||
item_id=item["id"],
|
||||
expected_epoch=guards["expected_epoch"],
|
||||
expected_revision=guards["expected_revision"],
|
||||
# The bytes the re-hash proved, or the body's for Leave unresolved.
|
||||
sha256=facts.sha256 if facts is not None else guards["sha256"],
|
||||
outcome="selected" if candidate is not None else UNRESOLVED,
|
||||
candidate=candidate,
|
||||
image_path=image_path,
|
||||
metadata=metadata,
|
||||
expected_fingerprint=guards["dependency_fingerprint"],
|
||||
aliases=aliases,
|
||||
known_rows=known,
|
||||
)
|
||||
except ReviewCommandError as exc:
|
||||
if exc.item is not None:
|
||||
roots, real_roots = self._review_roots()
|
||||
exc.view = self._review_view(exc.item, roots, real_roots)
|
||||
raise
|
||||
roots, real_roots = self._review_roots()
|
||||
return result, self._review_view(result["item"], roots, real_roots)
|
||||
|
||||
async def _record_decision(self, item, guards, candidate, verified):
|
||||
capture = verified[0] if verified else None
|
||||
try:
|
||||
result, view = await self._run_in_executor(
|
||||
self._decide_sync, item, guards, candidate, verified
|
||||
)
|
||||
except ReviewCommandError as exc:
|
||||
return self._decision_refused(exc)
|
||||
except LibraryCommandError as exc:
|
||||
return _coded(exc.code, library_epoch=exc.epoch)
|
||||
except Exception:
|
||||
self.logger.exception("Review decision failed")
|
||||
return _error(500, "decision-failed", "The decision could not be saved.")
|
||||
if capture is not None:
|
||||
# The decision used the capture; the page returns to Images.
|
||||
self._captures.release(capture.token)
|
||||
body = {
|
||||
"success": True,
|
||||
"library_epoch": guards["expected_epoch"],
|
||||
"item": view,
|
||||
"total": result["total"],
|
||||
}
|
||||
if result["pair"] is not None:
|
||||
body["pair"] = result["pair"]
|
||||
return web.json_response(body, headers=_NO_STORE)
|
||||
|
||||
def _decision_refused(self, exc: ReviewCommandError) -> web.Response:
|
||||
extra = {"library_epoch": exc.epoch}
|
||||
if exc.total is not None:
|
||||
extra["total"] = exc.total
|
||||
if exc.view is not None:
|
||||
extra["item"] = exc.view
|
||||
message = None
|
||||
if exc.fingerprint is not None:
|
||||
extra["dependency_fingerprint"] = exc.fingerprint
|
||||
if exc.code == "association-conflict" and exc.changed:
|
||||
message = "dependencies-changed"
|
||||
return _coded(exc.code, message, **extra)
|
||||
|
||||
|
||||
def _decision_guards(data: dict):
|
||||
"""The validated decision fields, or ``(None, error)``."""
|
||||
if set(data) != _DECISION_FIELDS:
|
||||
return None, bad_request(
|
||||
"Request must contain exactly " + ", ".join(sorted(_DECISION_FIELDS))
|
||||
)
|
||||
epoch, revision = data["expected_epoch"], data["expected_revision"]
|
||||
if not isinstance(epoch, str) or _UUID.fullmatch(epoch) is None:
|
||||
return None, bad_request("expected_epoch must be a canonical UUID")
|
||||
if type(revision) is not int or revision < 0:
|
||||
return None, bad_request("expected_revision must be a non-negative integer")
|
||||
token = data["capture_token"]
|
||||
if token is not None and (
|
||||
not isinstance(token, str) or _TOKEN.fullmatch(token) is None
|
||||
):
|
||||
return None, bad_request("Invalid capture token")
|
||||
for name in ("sha256", "dependency_fingerprint"):
|
||||
value = data[name]
|
||||
if not isinstance(value, str) or _HEX64.fullmatch(value) is None:
|
||||
return None, bad_request(
|
||||
f"{name} must be 64 lowercase hexadecimal characters"
|
||||
)
|
||||
choice = data["choice"]
|
||||
if not isinstance(choice, str) or (
|
||||
choice != UNRESOLVED and _HEX64.fullmatch(choice) is None
|
||||
):
|
||||
return None, bad_request("choice must be a candidate hash or unresolved")
|
||||
return {**data}, None
|
||||
|
||||
|
||||
def _verify_and_read(store, capture, item, root, need_metadata: bool):
|
||||
"""Worker thread: ``(relative, facts, captured)``, or None off the root.
|
||||
|
||||
Formats the item's target under the capture's root (resolving the root's
|
||||
real path), re-hashes the current file, then reads the capture and
|
||||
resolves it with the profile the item's evidence was recorded with.
|
||||
"""
|
||||
found = format_review_target(
|
||||
item["spelled_path"],
|
||||
item["locator"],
|
||||
[(capture.root_index, _realpath(str(root)))],
|
||||
)
|
||||
if found is None:
|
||||
return None
|
||||
relative = found[1]
|
||||
facts = store.verify(root, relative)
|
||||
captured = None
|
||||
if need_metadata:
|
||||
captured = captured_review(store, capture, item_profile(item["evidence"]))
|
||||
return relative, facts, captured
|
||||
|
||||
|
||||
def _same_file(spelled: str, image_path: str, locator: str) -> bool:
|
||||
"""True when ``spelled`` reaches the physical file at ``locator``."""
|
||||
try:
|
||||
if os.path.normcase(_realpath(spelled)) == locator:
|
||||
return True
|
||||
return os.path.samefile(spelled, image_path)
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def physical_spellings(db, image_path: str, locator: str):
|
||||
"""``(aliases, known_rows)`` for the legacy rows of one physical file.
|
||||
|
||||
The roots are real paths, while the monitor links a file under the
|
||||
directory it watches, so a symlinked or junctioned root, a ``subst`` or a
|
||||
mapped drive spells the same file otherwise. Every row whose spelling
|
||||
ends with the file's name (the link summaries' fallback lookup) is a
|
||||
candidate; ``aliases`` are the candidates that resolve to ``locator``
|
||||
(``normcase(realpath)``, as the capture forms it) or are the same file,
|
||||
and ``known_rows`` every candidate id, so a decision can refuse a
|
||||
same-name row it did not classify. Blocking: call it on a worker.
|
||||
"""
|
||||
rows = db.review_same_name_rows(os.path.basename(image_path))
|
||||
aliases = tuple(
|
||||
spelled
|
||||
for _row_id, spelled in rows
|
||||
if isinstance(spelled, str)
|
||||
and spelled != image_path
|
||||
and _same_file(spelled, image_path, locator)
|
||||
)
|
||||
return tuple(dict.fromkeys(aliases)), frozenset(row_id for row_id, _ in rows)
|
||||
|
||||
|
||||
def _candidate_still_offered(resolution: dict, choice: str) -> bool:
|
||||
"""True when the captured bytes still offer the chosen candidate.
|
||||
|
||||
Only what the decision depends on is re-checked: the bytes are still
|
||||
``ambiguous`` and the chosen candidate (its hash covers its sampler,
|
||||
sides and provenance) is still among their candidates. A resolver change
|
||||
that rewords the top-level reason or reorders candidates leaves a
|
||||
pending item applicable.
|
||||
"""
|
||||
if not isinstance(resolution, dict) or resolution.get("state") != "ambiguous":
|
||||
return False
|
||||
evidence, _count, limited = review_evidence(resolution)
|
||||
if limited:
|
||||
return False
|
||||
return any(c.get("hash") == choice for c in evidence.get("candidates", ()))
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Needs review change stream (#249 PR F2).
|
||||
|
||||
``GET /prompt_manager/review/events`` is a server-sent event stream for an
|
||||
open Needs review page: ``event: ready`` when it opens, then
|
||||
``event: change`` after each committed queue change (an item the scan or the
|
||||
monitor recorded, a decision from any tab, a spelling refresh) and after a
|
||||
restore. The first notice is sent at once; notices in the next
|
||||
``COALESCE_SECONDS`` collapse into one trailing ``change`` (M9: a busy scan
|
||||
records file after file, and each event costs every open page a re-read).
|
||||
The page re-reads its queue on each ``change``; nothing polls. ``change``
|
||||
carries no data (``{}``). ``ready`` carries the queue's version
|
||||
``{library_epoch, review_revision}``, read after the stream subscribed, the
|
||||
same pair a list read answers: a page whose first read already answered that
|
||||
version skips the re-read (F3, one read per visit), and any later change is a
|
||||
``change`` event. Nothing else about the queue is pushed.
|
||||
|
||||
A ``: keepalive`` comment every ``HEARTBEAT_SECONDS`` lets the server notice
|
||||
a closed tab (its write fails) and drop the subscription; clients ignore
|
||||
comments. Shutdown (``close_review_events``, an ``on_shutdown`` hook) ends
|
||||
every stream, and a stream asked for afterwards is a 503.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
try:
|
||||
from ...database.execution_history import subscribe_history_restore
|
||||
from ...database.review_changes import subscribe_review_changes
|
||||
except ImportError:
|
||||
from database.execution_history import subscribe_history_restore
|
||||
from database.review_changes import subscribe_review_changes
|
||||
|
||||
EVENTS_URL = "/prompt_manager/review/events"
|
||||
HEARTBEAT_SECONDS = 25
|
||||
# After a ``change``, the next one waits this long, collapsing the notices in between.
|
||||
COALESCE_SECONDS = 0.4
|
||||
_CHANGE = b"event: change\ndata: {}\n\n"
|
||||
_KEEPALIVE = b": keepalive\n\n"
|
||||
|
||||
|
||||
def _ready(version: dict) -> bytes:
|
||||
data = json.dumps(version, separators=(",", ":"), sort_keys=True)
|
||||
return f"event: ready\ndata: {data}\n\n".encode("utf-8")
|
||||
|
||||
|
||||
class _Stream:
|
||||
"""One open stream: a change flag set from any thread, and a close flag."""
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
|
||||
self._loop = loop
|
||||
self.changed = asyncio.Event()
|
||||
self.closed = asyncio.Event()
|
||||
self.ready = False
|
||||
|
||||
def wake(self) -> None:
|
||||
"""Thread-safe: mark a change (the loop may already be gone)."""
|
||||
try:
|
||||
self._loop.call_soon_threadsafe(self.changed.set)
|
||||
except RuntimeError: # the loop closed
|
||||
pass
|
||||
|
||||
async def next(self, timeout: float) -> str:
|
||||
"""``change``, ``closed`` or ``idle`` (nothing within ``timeout``)."""
|
||||
waits = {
|
||||
asyncio.ensure_future(self.changed.wait()): "change",
|
||||
asyncio.ensure_future(self.closed.wait()): "closed",
|
||||
}
|
||||
try:
|
||||
done, _ = await asyncio.wait(
|
||||
waits, timeout=timeout, return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
finally:
|
||||
for waiter in waits:
|
||||
waiter.cancel()
|
||||
if self.closed.is_set():
|
||||
return "closed"
|
||||
if done:
|
||||
self.changed.clear()
|
||||
return "change"
|
||||
return "idle"
|
||||
|
||||
async def closed_within(self, timeout: float) -> bool:
|
||||
"""Wait up to ``timeout``; true when the stream closed meanwhile."""
|
||||
try:
|
||||
await asyncio.wait_for(self.closed.wait(), timeout)
|
||||
except asyncio.TimeoutError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class ReviewEventsMixin:
|
||||
"""The Needs review change stream."""
|
||||
|
||||
def _register_review_event_routes(self, routes: web.RouteTableDef) -> None:
|
||||
routes.get(EVENTS_URL)(self.review_events)
|
||||
|
||||
def _review_streams(self) -> set:
|
||||
streams = getattr(self, "_open_review_streams", None)
|
||||
if streams is None:
|
||||
streams = self._open_review_streams = set()
|
||||
return streams
|
||||
|
||||
def review_stream_count(self) -> int:
|
||||
"""How many change streams have sent ``ready`` and are open (tests and
|
||||
diagnostics): a queue change after this count moved is a ``change``
|
||||
event on them, never part of the version ``ready`` named."""
|
||||
return sum(1 for stream in self._review_streams() if stream.ready)
|
||||
|
||||
async def close_review_events(self, app) -> None:
|
||||
"""End every open stream; later requests get 503 (shutdown hook)."""
|
||||
self._review_events_closed = True
|
||||
for stream in list(self._review_streams()):
|
||||
stream.closed.set()
|
||||
|
||||
async def _queue_version(self) -> dict:
|
||||
"""The queue version for ``ready``, read after subscribing (so a change
|
||||
after this read is an event); ``{}`` when it cannot be read, and the
|
||||
page then re-reads."""
|
||||
try:
|
||||
return await self._run_in_executor(self.db.review_queue_version)
|
||||
except Exception:
|
||||
self.logger.warning("Needs review version read failed", exc_info=True)
|
||||
return {}
|
||||
|
||||
async def review_events(self, request: web.Request) -> web.StreamResponse:
|
||||
if getattr(self, "_review_events_closed", False):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "The server is shutting down."},
|
||||
status=503,
|
||||
)
|
||||
stream = _Stream(asyncio.get_running_loop())
|
||||
db_path = self.db.model.db_path
|
||||
unsubscribers = (
|
||||
subscribe_review_changes(db_path, stream.wake),
|
||||
subscribe_history_restore(db_path, stream.wake),
|
||||
)
|
||||
streams = self._review_streams()
|
||||
streams.add(stream)
|
||||
response = web.StreamResponse(
|
||||
status=200,
|
||||
headers={
|
||||
"Content-Type": "text/event-stream",
|
||||
"Cache-Control": "no-store",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
try:
|
||||
version = await self._queue_version()
|
||||
await response.prepare(request)
|
||||
await response.write(_ready(version))
|
||||
stream.ready = True
|
||||
while True:
|
||||
outcome = await stream.next(HEARTBEAT_SECONDS)
|
||||
if outcome == "closed":
|
||||
break
|
||||
if outcome == "idle":
|
||||
await response.write(_KEEPALIVE)
|
||||
continue
|
||||
await response.write(_CHANGE)
|
||||
if await stream.closed_within(COALESCE_SECONDS):
|
||||
break
|
||||
await response.write_eof()
|
||||
except (ConnectionError, RuntimeError) as exc: # the tab went away
|
||||
self.logger.debug(f"Needs review stream closed ({exc})")
|
||||
finally:
|
||||
streams.discard(stream)
|
||||
for unsubscribe in unsubscribers:
|
||||
unsubscribe()
|
||||
return response
|
||||
@@ -0,0 +1,399 @@
|
||||
"""Needs review recording for the output scan and the live monitor (#249).
|
||||
|
||||
The rule (maintainer test of #284/#288): a file is queued only when its
|
||||
final outcome is ambiguous and unlinked. The strict resolver must find it
|
||||
``ambiguous``, the scan's legacy fallback (the browser parser,
|
||||
``_graph_prompt_text``) must not link it, and no saved row may link it
|
||||
already. A file the fallback links, an exact pair the ingester stored and an
|
||||
already linked file are never queued. Neither is an ambiguous file with
|
||||
nothing to choose: fewer than two of its candidates carry a prompt
|
||||
(``utils.review_capture.reviewable``); the scan reports it unresolved ("could
|
||||
not be traced") unless the inspection profile finds a choice in it.
|
||||
|
||||
Only scans add items (maintainer ruling, 2026-10-23). The monitor's pass
|
||||
records nothing: it only removes the pending items of an output it linked.
|
||||
|
||||
The queue heals itself: a pending item whose file is now linked (by the
|
||||
fallback, a later resolution or the monitor) is removed by the next scan
|
||||
that lists the file or by the monitor's next pass over it. The next scan
|
||||
also removes a pending item without a choice whose file is still ambiguous
|
||||
without one (``settle_untraced_review_items``), and an older version's
|
||||
item of bytes now untraced (``settle_stale_review_items``). Only an undecided item goes
|
||||
(``pending``, revision 0, no decision row); a decision is never deleted or
|
||||
changed (``settle_linked_review_items``).
|
||||
|
||||
The scan records through ``_record_scanned_review`` (best effort: a failure
|
||||
is logged and counted), which goes through
|
||||
``utils.review_capture.record_ambiguous_file`` on the API's capture store;
|
||||
the monitor calls ``record_review_file`` from its worker thread after its
|
||||
own link. Nothing here links a file.
|
||||
|
||||
Recording changes records, so it follows the registry's ``records``
|
||||
admission (#289). The scan records under its own ``records`` job. The
|
||||
monitor's pass lasts one file and is not a job: it is refused while a
|
||||
``records`` or ``guarded`` job runs (``deferred``), and the next scan reads
|
||||
the file again. Thumbnail work (the job and the gallery's generate and clear
|
||||
holds, kind ``thumbnails``) is ``guarded`` but only writes and removes preview
|
||||
files under ``thumbnails/`` and never touches the database, so it does not
|
||||
defer the recording (#249 S5 M7): a long rebuild no longer holds Needs review
|
||||
back for its whole run.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from .admin import SCAN_LOG_PREFIXES, _plural, _scan_location, _scan_roots
|
||||
from .jobs import THUMBNAILS_JOB_KIND
|
||||
|
||||
# The kind the monitor's recording checks admission under (never registered).
|
||||
REVIEW_RECORDING_KIND = "review_recording"
|
||||
# Guarded kinds that change no records, so the recording may run beside them.
|
||||
PREVIEW_ONLY_KINDS = (THUMBNAILS_JOB_KIND,)
|
||||
# record_ambiguous_file outcomes that leave the file in the pending queue.
|
||||
QUEUED_OUTCOMES = ("recorded", "limited", "duplicate")
|
||||
NEW_OUTCOMES = ("recorded", "limited")
|
||||
|
||||
try:
|
||||
from ...utils import review_capture
|
||||
from ...utils.review_capture import record_ambiguous_file
|
||||
except ImportError:
|
||||
from utils import review_capture
|
||||
from utils.review_capture import record_ambiguous_file
|
||||
|
||||
# Untraced files remembered per process so a rescan of an unchanged file does
|
||||
# not run the inspection profile again (oldest forgotten first). A plain dict
|
||||
# of 16-byte digests (the locator, and its size, mtime_ns and item version):
|
||||
# measured at 8.6 MiB with 65,536 entries (tracemalloc, CPython 3.14).
|
||||
MAX_UNTRACED_MEMORY = 65_536
|
||||
|
||||
|
||||
def _digest(text: str) -> bytes:
|
||||
data = text.encode("utf-8", "surrogatepass")
|
||||
return hashlib.blake2b(data, digest_size=16).digest()
|
||||
|
||||
|
||||
def _memory_key(locator: str) -> bytes:
|
||||
"""A 16-byte digest of ``locator`` (the untraced memory's key)."""
|
||||
return _digest(locator)
|
||||
|
||||
|
||||
# The scan log's reason for items that leave the queue with nothing to choose.
|
||||
NOTHING_TO_CHOOSE = "nothing to choose"
|
||||
|
||||
|
||||
class ReviewRecordingMixin:
|
||||
"""Record the scan's and the monitor's ambiguous files for Needs review."""
|
||||
|
||||
def _record_scanned_review(self, media_file, roots, review=None, scanned=None):
|
||||
"""Record one ambiguous, unlinked scanned file (logged and counted).
|
||||
|
||||
``scanned`` is the scan's ``(metadata, inspection_resolution)``.
|
||||
"""
|
||||
try:
|
||||
outcome = self._record_review_sync(
|
||||
media_file, roots, review, inspection=True, scanned=scanned
|
||||
)
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
f"{media_file.name} could not be added to Needs review",
|
||||
exc_info=True,
|
||||
)
|
||||
if review is not None:
|
||||
review["errors"] += 1
|
||||
return
|
||||
if review is None:
|
||||
return
|
||||
if outcome in NEW_OUTCOMES:
|
||||
review["added"] += 1
|
||||
if outcome in QUEUED_OUTCOMES:
|
||||
review["queued"] += 1
|
||||
|
||||
def _record_review_sync(
|
||||
self, media_file, roots, review=None, inspection=False, scanned=None
|
||||
):
|
||||
"""``record_ambiguous_file`` for a file under one of ``roots`` (blocking).
|
||||
|
||||
``inspection`` (the scan only) lets a file without a choice under the
|
||||
default profile be recorded from the inspection profile's choices.
|
||||
An ``untraced`` file (its captured bytes offer nothing to choose) also
|
||||
loses its pending items that offer nothing either.
|
||||
"""
|
||||
located = self._review_location(media_file, roots)
|
||||
if located is None:
|
||||
return "skipped"
|
||||
root, index, relative, _spelled = located
|
||||
outcome = record_ambiguous_file(
|
||||
self.db,
|
||||
self._capture_store(),
|
||||
Path(root),
|
||||
index,
|
||||
relative,
|
||||
inspection=inspection,
|
||||
scanned=scanned,
|
||||
)
|
||||
if outcome == "untraced":
|
||||
self._settle_untraced_reviews([media_file], roots, review)
|
||||
return outcome
|
||||
|
||||
def _untraced_scan_outcome(self, media_file, roots, canonical, review=None):
|
||||
"""The scan status of an unlinked file the default profile left
|
||||
ambiguous without a choice (blocking).
|
||||
|
||||
``ambiguous`` when the inspection profile offers a choice: the file is
|
||||
recorded for review from that profile (never linked), or ``unchanged``
|
||||
when a saved row links it already. Otherwise ``unresolved``, and its
|
||||
pending items without a choice leave the queue. The profile runs at
|
||||
most once per file state: not for bytes already queued (or decided)
|
||||
at the file's current size and ``mtime_ns`` under the version that
|
||||
its records depend on, and not again for a state it already found
|
||||
untraced in this process.
|
||||
"""
|
||||
try:
|
||||
choice = self._inspection_choice(media_file, roots, canonical)
|
||||
except Exception:
|
||||
# One bad file never stops the scan: it stays "could not be traced".
|
||||
self.logger.warning(
|
||||
f"{media_file.name} could not be inspected for Needs review",
|
||||
exc_info=True,
|
||||
)
|
||||
if review is not None:
|
||||
review["errors"] += 1
|
||||
return "unresolved"
|
||||
if choice is None:
|
||||
self._settle_untraced_reviews([media_file], roots, review)
|
||||
self._settle_stale_reviews(media_file, roots, canonical, review)
|
||||
return "unresolved"
|
||||
if self._review_file_linked(media_file, roots):
|
||||
return "unchanged"
|
||||
self._record_scanned_review(media_file, roots, review, scanned=choice)
|
||||
return "ambiguous"
|
||||
|
||||
def _inspection_choice(self, media_file, roots, canonical):
|
||||
"""``(canonical, resolution_or_None)`` when the file offers a choice.
|
||||
|
||||
The resolution is None when the file's current state is already in
|
||||
the queue (so the profile did not run). None when it offers none.
|
||||
"""
|
||||
if canonical is None:
|
||||
return None
|
||||
version = review_capture.inspection_review_version(canonical)
|
||||
located = self._review_location(media_file, roots)
|
||||
if version is None or located is None:
|
||||
return None
|
||||
locator = os.path.normcase(located[3])
|
||||
try:
|
||||
facts = os.lstat(located[3])
|
||||
except OSError:
|
||||
facts = None
|
||||
state = None
|
||||
if facts is not None:
|
||||
state = _digest(f"{facts.st_size}:{facts.st_mtime_ns}:{version}")
|
||||
if self.db.review_decided_at(
|
||||
locator, facts.st_size, facts.st_mtime_ns
|
||||
) or self.db.review_fingerprint_queued(
|
||||
locator, facts.st_size, facts.st_mtime_ns, version
|
||||
):
|
||||
return canonical, None
|
||||
memory = self._untraced_memory()
|
||||
key = _memory_key(locator)
|
||||
if state is not None and memory.get(key) == state:
|
||||
return None
|
||||
resolution = review_capture.inspection_choice(canonical)
|
||||
if resolution is not None:
|
||||
memory.pop(key, None)
|
||||
return canonical, resolution
|
||||
if state is not None:
|
||||
memory.pop(key, None) # re-inserted last: the newest state
|
||||
memory[key] = state
|
||||
while len(memory) > MAX_UNTRACED_MEMORY:
|
||||
del memory[next(iter(memory))]
|
||||
return None
|
||||
|
||||
def _settle_stale_reviews(self, media_file, roots, canonical, review=None):
|
||||
"""Remove the pending, undecided items of an untraced file's current
|
||||
bytes recorded under another version (best effort, counted in
|
||||
``review["untraced"]``).
|
||||
|
||||
The current resolver and table offer nothing to choose in these
|
||||
bytes, so an item of an older version would only refuse Apply.
|
||||
"""
|
||||
located = self._review_location(media_file, roots)
|
||||
if located is None or canonical is None:
|
||||
return 0
|
||||
# An unusable semantics table says nothing about the current
|
||||
# versions; never let it retire items a working table recorded.
|
||||
if review_capture.inspection_review_version(canonical) is None:
|
||||
return 0
|
||||
try:
|
||||
facts = os.lstat(located[3])
|
||||
removed = self.db.settle_stale_review_items(
|
||||
os.path.normcase(located[3]),
|
||||
facts.st_size,
|
||||
facts.st_mtime_ns,
|
||||
review_capture.item_versions(canonical),
|
||||
)
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
f"Stale Needs review items of {media_file.name} could not be "
|
||||
"removed",
|
||||
exc_info=True,
|
||||
)
|
||||
if review is not None:
|
||||
review["errors"] += 1
|
||||
return 0
|
||||
if review is not None:
|
||||
review["untraced"] += removed
|
||||
return removed
|
||||
|
||||
def _untraced_memory(self):
|
||||
memory = getattr(self, "_untraced_inspections", None)
|
||||
if memory is None:
|
||||
memory = self._untraced_inspections = {}
|
||||
return memory
|
||||
|
||||
@staticmethod
|
||||
def _review_location(media_file, roots, real_roots=None):
|
||||
"""``(root, root_index, relative, spelled)`` of a listed file, or None.
|
||||
|
||||
``spelled`` is the capture's spelling of the file (the real root joined
|
||||
with the relative path); ``os.path.normcase`` of it is the locator
|
||||
review items are stored under. ``real_roots`` caches each root's
|
||||
``realpath`` across a batch.
|
||||
"""
|
||||
relative, _label, index = _scan_location(media_file, roots)
|
||||
root = next((r for r, _l, i in roots if i == index), None)
|
||||
if index is None or root is None:
|
||||
return None
|
||||
if real_roots is None:
|
||||
real_roots = {}
|
||||
real = real_roots.get(index)
|
||||
if real is None:
|
||||
real = real_roots[index] = os.path.realpath(os.fspath(root))
|
||||
spelled = os.path.join(real, *relative.split("/"))
|
||||
return root, index, relative, spelled
|
||||
|
||||
def _review_file_linked(self, media_file, roots):
|
||||
"""True when a saved row links the scanned file (any spelling)."""
|
||||
located = self._review_location(media_file, roots)
|
||||
spellings = (str(media_file),) + ((located[3],) if located else ())
|
||||
return self.db.review_file_linked(spellings)
|
||||
|
||||
def _settle_scanned_reviews(self, media_files, roots, review=None):
|
||||
"""Remove the pending items of ``media_files`` that are now linked.
|
||||
|
||||
Blocking; best effort like recording (a failure is logged and
|
||||
counted in ``review["errors"]``). Returns how many items left.
|
||||
"""
|
||||
real_roots = {}
|
||||
files = []
|
||||
for media_file in media_files:
|
||||
located = self._review_location(media_file, roots, real_roots)
|
||||
if located is not None:
|
||||
spelled = located[3]
|
||||
files.append((os.path.normcase(spelled), (str(media_file), spelled)))
|
||||
try:
|
||||
settled = self.db.settle_linked_review_items(files) if files else 0
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"Needs review items of linked files could not be removed",
|
||||
exc_info=True,
|
||||
)
|
||||
if review is not None:
|
||||
review["errors"] += 1
|
||||
return 0
|
||||
if review is not None:
|
||||
review["settled"] += settled
|
||||
return settled
|
||||
|
||||
def _settle_untraced_reviews(self, media_files, roots, review=None):
|
||||
"""Remove the pending items without a choice of untraced ``media_files``.
|
||||
|
||||
``media_files`` are files found ambiguous with fewer than two
|
||||
candidates that carry a prompt. Blocking; best effort like the linked
|
||||
settle (a failure is logged and counted in ``review["errors"]``).
|
||||
Returns how many items left.
|
||||
"""
|
||||
locators = []
|
||||
for media_file in media_files:
|
||||
located = self._review_location(media_file, roots)
|
||||
if located is not None:
|
||||
locators.append(os.path.normcase(located[3]))
|
||||
try:
|
||||
removed = self.db.settle_untraced_review_items(locators) if locators else 0
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"Needs review items without a choice could not be removed",
|
||||
exc_info=True,
|
||||
)
|
||||
if review is not None:
|
||||
review["errors"] += 1
|
||||
return 0
|
||||
if review is not None:
|
||||
review["untraced"] += removed
|
||||
return removed
|
||||
|
||||
def record_review_file(self, image_path):
|
||||
"""Apply the Needs review rule to one monitor output (its worker thread).
|
||||
|
||||
Only scans add items (maintainer ruling, 2026-10-23): the monitor
|
||||
only clears the items of outputs it links. Maps ``image_path`` to the
|
||||
gallery root that lists it; a file under no current root is
|
||||
``skipped``. A linked file's pending items are removed and the answer
|
||||
is ``linked``. Any other file is ``skipped``: nothing is captured or
|
||||
recorded, and the next scan reads it. While a job runs that a
|
||||
``records`` job may not run beside, other than thumbnail work
|
||||
(``PREVIEW_ONLY_KINDS``), nothing is written and the answer is
|
||||
``deferred``. Errors propagate: the monitor logs them and keeps its
|
||||
link.
|
||||
"""
|
||||
busy = self._jobs.blocker(
|
||||
REVIEW_RECORDING_KIND, admission="records", compatible=PREVIEW_ONLY_KINDS
|
||||
)
|
||||
if busy is not None:
|
||||
self.logger.info(
|
||||
f"Needs review was not updated for "
|
||||
f"{os.path.basename(str(image_path))} while {busy.title} runs; "
|
||||
"the next scan reads it again"
|
||||
)
|
||||
return "deferred"
|
||||
full = os.path.abspath(str(image_path))
|
||||
located = Path(os.path.realpath(os.path.dirname(full))) / os.path.basename(full)
|
||||
roots = _scan_roots(self._get_all_output_dirs())
|
||||
if self._review_location(located, roots) is None:
|
||||
return "skipped"
|
||||
if self._review_file_linked(located, roots):
|
||||
self._settle_scanned_reviews([located], roots)
|
||||
return "linked"
|
||||
return "skipped"
|
||||
|
||||
@staticmethod
|
||||
def _log_review_items(job, review):
|
||||
"""The scan log's Needs review summary lines (none when nothing moved)."""
|
||||
prefix = SCAN_LOG_PREFIXES["review"]
|
||||
if review["added"]:
|
||||
job.log_line(
|
||||
f"{prefix} {_plural(review['added'], 'image')} added to Needs review"
|
||||
)
|
||||
already = review["queued"] - review["added"]
|
||||
if already:
|
||||
job.log_line(
|
||||
f"{prefix} {_plural(already, 'image')} already in Needs review"
|
||||
)
|
||||
if review["settled"]:
|
||||
job.log_line(
|
||||
f"{SCAN_LOG_PREFIXES['known']} "
|
||||
f"{_plural(review['settled'], 'image')} left Needs review · now linked"
|
||||
)
|
||||
if review["untraced"]:
|
||||
job.log_line(
|
||||
f"{SCAN_LOG_PREFIXES['known']} "
|
||||
f"{_plural(review['untraced'], 'image')} left Needs review · "
|
||||
f"{NOTHING_TO_CHOOSE}"
|
||||
)
|
||||
if review["errors"]:
|
||||
job.log_line(
|
||||
f"{SCAN_LOG_PREFIXES['errors']} "
|
||||
f"{_plural(review['errors'], 'image')} could not be added to "
|
||||
"Needs review"
|
||||
)
|
||||
@@ -0,0 +1,560 @@
|
||||
"""Settings API routes: GET and POST /prompt_manager/settings (#220).
|
||||
|
||||
A save validates the payload, commits it to config.json through the config
|
||||
store and publishes it at runtime inside that commit; a save that cannot be
|
||||
written changes nothing at runtime and says so. ``_config_commit_failure``
|
||||
is shared with the LoRA integration's save.
|
||||
|
||||
GET also reports the image sources and the monitor (``sources.py``). A save
|
||||
that changes the sources runs as a ``source_change`` registry hold, so it is
|
||||
refused while a record-changing job (a scan) runs. ``monitoring_enabled``
|
||||
takes effect at the next restart. ``POST /prompt_manager/settings/validate-source``
|
||||
checks one path and writes nothing.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from ..config_store import (
|
||||
ConfigUnreadable,
|
||||
RevisionConflict,
|
||||
get_config_store,
|
||||
)
|
||||
from . import sources
|
||||
from .admin import _current_image_monitor
|
||||
from .job_routes import busy_response
|
||||
from .jobs import JobBusy
|
||||
|
||||
try:
|
||||
from ...utils.validators import validate_result_timeout, validate_worker_threads
|
||||
except ImportError:
|
||||
from utils.validators import validate_result_timeout, validate_worker_threads
|
||||
|
||||
|
||||
# Settings owned by POST /settings: config attribute -> its keys in config.json.
|
||||
_SETTINGS_FILE_KEYS = {
|
||||
"RESULT_TIMEOUT": ("web_ui", "result_timeout"),
|
||||
"WEBUI_DISPLAY_MODE": ("web_ui", "webui_display_mode"),
|
||||
"INFINITE_SCROLL": ("web_ui", "infinite_scroll"),
|
||||
"WORKER_THREADS": ("performance", "worker_threads"),
|
||||
"MONITORING_ENABLED": ("gallery", "monitoring", "enabled"),
|
||||
"SAFETY_BACKUP": ("maintenance", "safety_backup"),
|
||||
}
|
||||
# Attributes that live on GalleryConfig; the others on PromptManagerConfig.
|
||||
_GALLERY_SETTINGS = frozenset({"MONITORING_ENABLED"})
|
||||
SOURCE_CHANGE_JOB_KIND = "source_change"
|
||||
SOURCE_CHANGE_JOB_TITLE = "Source change"
|
||||
|
||||
|
||||
def _source_identity(path):
|
||||
"""The case-folded spelling two roots are the same directory by (Windows)."""
|
||||
return os.path.normcase(path)
|
||||
|
||||
|
||||
def _stat_identity(path):
|
||||
"""``(st_dev, st_ino)`` of an existing directory, or None.
|
||||
|
||||
Catches case variants on a case-insensitive filesystem whose spelling
|
||||
normcase keeps (macOS).
|
||||
"""
|
||||
try:
|
||||
stat = os.stat(path)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
return (stat.st_dev, stat.st_ino) if stat.st_ino else None
|
||||
|
||||
|
||||
# Refusals that a reconnected drive or folder can lift.
|
||||
_RECONNECTABLE = frozenset({"not-found", "not-directory"})
|
||||
|
||||
|
||||
def _root_key(root):
|
||||
"""The directory a configured root names (resolved), else its spelling."""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
try:
|
||||
return GalleryConfig.resolve_gallery_root(root)
|
||||
except (OSError, TypeError, ValueError, AttributeError):
|
||||
return root
|
||||
|
||||
|
||||
def _request_error(message):
|
||||
return {"code": "invalid-request", "error": message}
|
||||
|
||||
|
||||
def _invalid_source(index, entry, code, reason):
|
||||
"""The 400 body for the source at ``index`` that cannot be used.
|
||||
|
||||
Names it by its public spelling (computed as text: a refused path is
|
||||
never resolved), so the page can mark that row and say why.
|
||||
"""
|
||||
source = sources.public_spelling(entry)
|
||||
advice = (
|
||||
"Remove it or reconnect it, then save."
|
||||
if code in _RECONNECTABLE
|
||||
else "Remove it, then save."
|
||||
)
|
||||
return {
|
||||
"code": "invalid-source",
|
||||
"source": source,
|
||||
"index": index,
|
||||
"reason": reason,
|
||||
"error": f"{source} can't be used: {reason}. {advice}",
|
||||
}
|
||||
|
||||
|
||||
def _section(config, name):
|
||||
"""``config[name]`` as a fresh dict (a non-object section starts empty)."""
|
||||
value = config.get(name)
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _settings_mutation(updates, new_roots):
|
||||
"""The config commit for a settings save: only the submitted keys change.
|
||||
|
||||
Every other section and key (integrations and the CivitAI key, database,
|
||||
unknown sections) is kept as it is on disk.
|
||||
"""
|
||||
|
||||
def mutate(config):
|
||||
for name, value in updates.items():
|
||||
*sections, key = _SETTINGS_FILE_KEYS[name]
|
||||
_set_nested(config, sections, key, value)
|
||||
if new_roots is not None:
|
||||
gallery = _section(config, "gallery")
|
||||
monitoring = _section(gallery, "monitoring")
|
||||
monitoring["directories"] = list(new_roots)
|
||||
gallery["monitoring"] = monitoring
|
||||
config["gallery"] = gallery
|
||||
return config
|
||||
|
||||
return mutate
|
||||
|
||||
|
||||
def _set_nested(config, sections, key, value):
|
||||
"""``config[s1][s2]...[key] = value``, each section a fresh dict."""
|
||||
if not sections:
|
||||
config[key] = value
|
||||
return
|
||||
merged = _section(config, sections[0])
|
||||
_set_nested(merged, sections[1:], key, value)
|
||||
config[sections[0]] = merged
|
||||
|
||||
|
||||
async def _read_json(request):
|
||||
"""Parse a JSON object body through the package's capped reader."""
|
||||
from . import _read_json_body
|
||||
|
||||
return await _read_json_body(request)
|
||||
|
||||
|
||||
class SettingsRoutesMixin:
|
||||
"""Mixin providing the settings read and save endpoints."""
|
||||
|
||||
def _register_settings_routes(self, routes):
|
||||
@routes.get("/prompt_manager/settings")
|
||||
async def get_settings_route(request):
|
||||
return await self.get_settings(request)
|
||||
|
||||
@routes.post("/prompt_manager/settings")
|
||||
async def save_settings_route(request):
|
||||
return await self.save_settings(request)
|
||||
|
||||
@routes.post("/prompt_manager/settings/validate-source")
|
||||
async def validate_source_route(request):
|
||||
return await self.validate_source(request)
|
||||
|
||||
async def get_settings(self, request):
|
||||
"""Get current settings."""
|
||||
try:
|
||||
from ..config import PromptManagerConfig, GalleryConfig
|
||||
|
||||
root_paths = [
|
||||
self._public_path(d) for d in GalleryConfig.MONITORING_DIRECTORIES
|
||||
]
|
||||
revision = await self._run_in_executor(self._config_revision)
|
||||
source_status, monitor = await self._run_in_executor(self._source_status)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
**revision,
|
||||
"sources": source_status,
|
||||
"monitor": monitor,
|
||||
"settings": {
|
||||
"monitoring_enabled": GalleryConfig.MONITORING_ENABLED,
|
||||
"result_timeout": PromptManagerConfig.RESULT_TIMEOUT,
|
||||
"webui_display_mode": PromptManagerConfig.WEBUI_DISPLAY_MODE,
|
||||
"infinite_scroll": PromptManagerConfig.INFINITE_SCROLL,
|
||||
"worker_threads": PromptManagerConfig.WORKER_THREADS,
|
||||
"safety_backup": PromptManagerConfig.SAFETY_BACKUP,
|
||||
"cpu_count": PromptManagerConfig.max_worker_threads(),
|
||||
"gallery_root_paths": root_paths,
|
||||
"gallery_root_path": root_paths[0] if root_paths else "",
|
||||
"monitored_directories": [
|
||||
self._public_path(d) for d in self._monitored_directories()
|
||||
],
|
||||
},
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Get settings error: {e}", exc_info=True)
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Failed to get settings"},
|
||||
status=500,
|
||||
)
|
||||
|
||||
def _source_status(self):
|
||||
"""``(sources, monitor)`` of GET /settings (blocking)."""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
output_dirs, auto_detected = self._output_dirs_with_origin()
|
||||
return sources.describe_sources(
|
||||
configured=list(GalleryConfig.MONITORING_DIRECTORIES),
|
||||
output_dirs=output_dirs,
|
||||
auto_detected=auto_detected,
|
||||
monitor=_current_image_monitor(),
|
||||
enabled=GalleryConfig.MONITORING_ENABLED,
|
||||
)
|
||||
|
||||
async def validate_source(self, request):
|
||||
"""Check one source path ``{path}``; writes nothing.
|
||||
|
||||
200 ``{success, valid, label, path, code, reason}``: a valid source
|
||||
answers its label and public path, an invalid one its code and reason
|
||||
only. A body without a string ``path`` is a 400 ``invalid-request``.
|
||||
"""
|
||||
data, error_response = await _read_json(request)
|
||||
if error_response is not None:
|
||||
too_large = error_response.status == 413
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "too-large" if too_large else "invalid-request",
|
||||
"error": (
|
||||
"Request body too large"
|
||||
if too_large
|
||||
else "Request body must be a JSON object"
|
||||
),
|
||||
},
|
||||
status=error_response.status,
|
||||
)
|
||||
path = data.get("path")
|
||||
if not isinstance(path, str):
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "invalid-request",
|
||||
"error": "path must be a string",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
try:
|
||||
result = await self._run_in_executor(sources.validate_source, path)
|
||||
except Exception:
|
||||
self.logger.exception("Source validation failed")
|
||||
return web.json_response(
|
||||
{"success": False, "code": None, "error": "Failed to check the source"},
|
||||
status=500,
|
||||
)
|
||||
return web.json_response({"success": True, **result})
|
||||
|
||||
def _monitored_directories(self):
|
||||
"""Directories the running image monitor watches, else the configured roots."""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
monitor = _current_image_monitor()
|
||||
if monitor is not None:
|
||||
return list(getattr(monitor, "monitored_directories", []))
|
||||
return list(GalleryConfig.MONITORING_DIRECTORIES)
|
||||
|
||||
async def save_settings(self, request):
|
||||
"""Save settings: validate, commit to config.json, then publish (#220).
|
||||
|
||||
Nothing changes at runtime until config.json holds the new values, so
|
||||
a save that cannot be written reports ``persisted: false`` and leaves
|
||||
the running settings as they were. ``expected_revision`` (the
|
||||
``revision`` from GET) refuses to overwrite a file changed meanwhile;
|
||||
without it the payload merges into what is on disk (legacy page).
|
||||
"""
|
||||
try:
|
||||
data, error_response = await _read_json(request)
|
||||
if error_response is not None:
|
||||
return error_response
|
||||
|
||||
expected_revision = data.get("expected_revision")
|
||||
if expected_revision is not None and not isinstance(expected_revision, str):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "expected_revision must be a string"},
|
||||
status=400,
|
||||
)
|
||||
try:
|
||||
updates = self._validate_settings(data)
|
||||
except ValueError as ve:
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(ve)}, status=400
|
||||
)
|
||||
|
||||
# Off the loop: an accepted root is resolved (filesystem access).
|
||||
new_roots, error = await self._run_in_executor(
|
||||
self._parse_gallery_roots, data
|
||||
)
|
||||
if error:
|
||||
return web.json_response({"success": False, **error}, status=400)
|
||||
roots_change = new_roots is not None and await self._run_in_executor(
|
||||
self._roots_change, new_roots
|
||||
)
|
||||
|
||||
published = {}
|
||||
|
||||
def publish(_config):
|
||||
# Under the commit lock: saves reach the runtime in the order
|
||||
# they were written, so the runtime ends on the last one.
|
||||
published.update(self._publish_settings(updates, new_roots))
|
||||
|
||||
async with contextlib.AsyncExitStack() as stack:
|
||||
hold = None
|
||||
if roots_change:
|
||||
try:
|
||||
hold = await stack.enter_async_context(
|
||||
self._jobs.hold(
|
||||
SOURCE_CHANGE_JOB_KIND,
|
||||
SOURCE_CHANGE_JOB_TITLE,
|
||||
admission="records",
|
||||
)
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
try:
|
||||
_saved, revision = await self._run_in_executor(
|
||||
get_config_store().commit,
|
||||
_settings_mutation(updates, new_roots),
|
||||
expected_revision=expected_revision,
|
||||
publish=publish,
|
||||
)
|
||||
except (ConfigUnreadable, RevisionConflict, OSError) as exc:
|
||||
if hold is not None:
|
||||
hold.finish("failed")
|
||||
return self._config_commit_failure(exc)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"message": "Settings saved successfully",
|
||||
"restart_required": bool(published["restart_required_fields"]),
|
||||
"persisted": True,
|
||||
"revision": revision,
|
||||
**published,
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Save settings error: {e}", exc_info=True)
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Failed to save settings"},
|
||||
status=500,
|
||||
)
|
||||
|
||||
def _publish_settings(self, updates, new_roots):
|
||||
"""Install committed settings at runtime (inside the config commit).
|
||||
|
||||
The values are already durable. A publication that fails is reported
|
||||
as ``runtime_applied: false`` with the fields that need a restart;
|
||||
nothing is retried or rolled back. Never raises.
|
||||
"""
|
||||
from ..config import GalleryConfig, PromptManagerConfig
|
||||
|
||||
for name, value in updates.items():
|
||||
owner = GalleryConfig if name in _GALLERY_SETTINGS else PromptManagerConfig
|
||||
setattr(owner, name, value)
|
||||
restart_fields = []
|
||||
runtime_applied = True
|
||||
if new_roots is not None:
|
||||
try:
|
||||
if self._apply_gallery_roots(new_roots):
|
||||
restart_fields.append("gallery_root_paths")
|
||||
except Exception as e:
|
||||
self.logger.error(
|
||||
f"Saved gallery roots could not be applied: {e}", exc_info=True
|
||||
)
|
||||
runtime_applied = False
|
||||
restart_fields.append("gallery_root_paths")
|
||||
if "MONITORING_ENABLED" in updates:
|
||||
# The monitor starts with the server: a restart applies the setting
|
||||
# unless the observer already matches it.
|
||||
if updates["MONITORING_ENABLED"] != self._observer_alive():
|
||||
restart_fields.append("monitoring_enabled")
|
||||
return {
|
||||
"runtime_applied": runtime_applied,
|
||||
"restart_required_fields": restart_fields,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _roots_change(new_roots):
|
||||
"""Whether a save changes the sources (blocking: it resolves them).
|
||||
|
||||
Compared as directories, not spellings: the legacy page resends the
|
||||
roots resolved, while config.json may hold them as hand-written
|
||||
(relative, another case), and an unchanged resend is no change.
|
||||
"""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
if new_roots is None:
|
||||
return False
|
||||
return [_root_key(root) for root in new_roots] != [
|
||||
_root_key(root) for root in GalleryConfig.MONITORING_DIRECTORIES
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _observer_alive():
|
||||
"""Whether the image monitor's observer is running now. Never raises."""
|
||||
try:
|
||||
return sources.monitor_state(_current_image_monitor())[1]
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _config_commit_failure(self, exc):
|
||||
"""The response for a config commit that wrote nothing."""
|
||||
not_saved = {"success": False, "persisted": False, "runtime_applied": False}
|
||||
if isinstance(exc, ConfigUnreadable):
|
||||
self.logger.warning(f"Settings not saved: {exc}")
|
||||
return web.json_response(
|
||||
{
|
||||
**not_saved,
|
||||
"code": "config-unreadable",
|
||||
"error": (
|
||||
"config.json cannot be read, so nothing was saved. "
|
||||
"Fix or remove the file, then save again."
|
||||
),
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
if isinstance(exc, RevisionConflict):
|
||||
return web.json_response(
|
||||
{
|
||||
**not_saved,
|
||||
"code": "settings-changed",
|
||||
"error": "Settings changed since they were loaded. Reload them.",
|
||||
"revision": exc.current,
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
self.logger.error(f"Could not write config.json: {exc}")
|
||||
return web.json_response(
|
||||
{
|
||||
**not_saved,
|
||||
"code": "config-write-failed",
|
||||
"error": "Settings could not be written to config.json.",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _config_revision():
|
||||
"""``{"revision": ...}``, or a null revision with ``config_error``."""
|
||||
try:
|
||||
_data, revision = get_config_store().read()
|
||||
except ConfigUnreadable:
|
||||
return {"revision": None, "config_error": "config-unreadable"}
|
||||
return {"revision": revision}
|
||||
|
||||
@staticmethod
|
||||
def _validate_settings(data):
|
||||
"""Scalar settings of a payload as ``{CONFIG_ATTR: value}``.
|
||||
|
||||
Raises ValueError with a user-facing message on the first bad value;
|
||||
nothing is applied until every value passed.
|
||||
"""
|
||||
from ..config import PromptManagerConfig
|
||||
|
||||
updates = {}
|
||||
if "result_timeout" in data:
|
||||
updates["RESULT_TIMEOUT"] = validate_result_timeout(data["result_timeout"])
|
||||
if "webui_display_mode" in data:
|
||||
mode = data["webui_display_mode"]
|
||||
if mode not in ("popup", "newtab"):
|
||||
raise ValueError("webui_display_mode must be 'popup' or 'newtab'")
|
||||
updates["WEBUI_DISPLAY_MODE"] = mode
|
||||
if "infinite_scroll" in data:
|
||||
if not isinstance(data["infinite_scroll"], bool):
|
||||
raise ValueError("infinite_scroll must be true or false")
|
||||
updates["INFINITE_SCROLL"] = data["infinite_scroll"]
|
||||
if "monitoring_enabled" in data:
|
||||
if not isinstance(data["monitoring_enabled"], bool):
|
||||
raise ValueError("monitoring_enabled must be true or false")
|
||||
updates["MONITORING_ENABLED"] = data["monitoring_enabled"]
|
||||
if "safety_backup" in data:
|
||||
if not isinstance(data["safety_backup"], bool):
|
||||
raise ValueError("safety_backup must be true or false")
|
||||
updates["SAFETY_BACKUP"] = data["safety_backup"]
|
||||
if "worker_threads" in data:
|
||||
updates["WORKER_THREADS"] = validate_worker_threads(
|
||||
data["worker_threads"], PromptManagerConfig.max_worker_threads()
|
||||
)
|
||||
return updates
|
||||
|
||||
def _parse_gallery_roots(self, data):
|
||||
"""Extract and validate gallery roots from a settings payload.
|
||||
|
||||
Accepts ``gallery_root_paths`` (list, preferred) or the legacy single
|
||||
``gallery_root_path`` string. Blank entries are dropped.
|
||||
|
||||
Each entry passes the same checks as validate-source
|
||||
(``sources.source_problem``): the form, then containment as text,
|
||||
before anything is resolved. Blocking (an accepted root is resolved).
|
||||
|
||||
Returns:
|
||||
(roots, error): ``roots`` is the validated list, or ``None`` when
|
||||
the payload carries no gallery root key; ``error`` is the coded
|
||||
body of the 400 when validation failed (``roots`` is then None).
|
||||
An unusable source is ``invalid-source`` naming its public path
|
||||
and its position in the list.
|
||||
"""
|
||||
if "gallery_root_paths" in data:
|
||||
raw = data["gallery_root_paths"]
|
||||
if not isinstance(raw, list):
|
||||
return None, _request_error("gallery_root_paths must be a list")
|
||||
elif "gallery_root_path" in data:
|
||||
raw = [data["gallery_root_path"]]
|
||||
else:
|
||||
return None, None
|
||||
|
||||
from ..config import GalleryConfig
|
||||
|
||||
roots = []
|
||||
seen = set()
|
||||
for index, entry in enumerate(raw):
|
||||
if not isinstance(entry, str):
|
||||
return None, _request_error("Gallery root paths must be strings")
|
||||
entry = entry.strip()
|
||||
if not entry:
|
||||
continue
|
||||
code, reason = sources.source_problem(entry)
|
||||
if code is not None:
|
||||
return None, _invalid_source(index, entry, code, reason)
|
||||
root = GalleryConfig.resolve_gallery_root(entry)
|
||||
keys = {("path", _source_identity(root)), ("file", _stat_identity(root))}
|
||||
keys.discard(("file", None))
|
||||
if keys & seen:
|
||||
continue # another spelling of a root already listed
|
||||
seen |= keys
|
||||
roots.append(root)
|
||||
return roots, None
|
||||
|
||||
def _apply_gallery_roots(self, roots):
|
||||
"""Install validated gallery roots; returns True when they changed.
|
||||
|
||||
Another spelling of the same directories installs the resolved
|
||||
spelling and is no change (nothing to restart for).
|
||||
"""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
if roots == list(GalleryConfig.MONITORING_DIRECTORIES):
|
||||
return False
|
||||
if not self._roots_change(roots):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = roots
|
||||
return False
|
||||
GalleryConfig.MONITORING_DIRECTORIES = roots
|
||||
self._cached_output_dir = None
|
||||
self.reset_gallery_roots_cache()
|
||||
return True
|
||||
@@ -0,0 +1,304 @@
|
||||
"""Image sources as System › Sources & scans shows them (#249 S4).
|
||||
|
||||
- ``source_form_error`` is the lexical check a submitted source path passes
|
||||
before any filesystem access: no control characters, no ``..`` part, and
|
||||
only the native path forms (on Windows a drive or a UNC share, never the
|
||||
``\\\\?\\`` and ``\\\\.\\`` namespaces, a drive-relative path, an alternate
|
||||
data stream, a device name or a part Windows would silently trim; on POSIX
|
||||
no Windows drive or backslash form).
|
||||
- ``source_problem`` is the one refusal every caller answers (validate-source
|
||||
and the settings save): the form check, then
|
||||
``GalleryConfig.gallery_root_problem``, which decides containment as text
|
||||
before it touches the filesystem.
|
||||
- ``describe_sources`` (blocking: it resolves paths) reports the configured
|
||||
sources with their validity, the effective sources with their listing
|
||||
index and watch state, and the image monitor's state. Every path in it is
|
||||
a public path (``_public_path``), never an absolute one; ``in_comfyui``
|
||||
says whether an effective source's public path is relative to the ComfyUI
|
||||
base (``in_comfyui_base``), so the page can read it as ``ComfyUI/<path>``.
|
||||
|
||||
A source's watch state is ``watching`` when the monitor's observer is alive,
|
||||
scheduled that directory and monitoring is on; ``stopping`` when it still
|
||||
watches it but monitoring was switched off (it stops at the next restart);
|
||||
``manual`` when monitoring is off or this process has no monitor; else
|
||||
``restart_pending`` (a source added since the monitor started, monitoring
|
||||
switched on, or an observer that stopped).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import ntpath
|
||||
import os
|
||||
import re
|
||||
|
||||
logger = logging.getLogger("prompt_manager.api")
|
||||
|
||||
MAX_SOURCE_PATH = 4096 # characters
|
||||
FORM_REASONS = {
|
||||
"empty": "Gallery root must be a non-empty path",
|
||||
"invalid-form": "Gallery root is not a folder path this server can use",
|
||||
"traversal": "Gallery root must not contain '..'",
|
||||
}
|
||||
_DRIVE = re.compile(r"[A-Za-z]:")
|
||||
_POSIX_FOREIGN = re.compile(r"^[A-Za-z]:")
|
||||
# Reserved device names: Windows opens the device for any folder of that name.
|
||||
_DEVICE_NAME = re.compile(r"^(con|prn|aux|nul|com[0-9¹²³]|lpt[0-9¹²³])(\..*)?$", re.I)
|
||||
# folder_paths getters and the label of the directory each one reports.
|
||||
_COMFYUI_LABELS = (
|
||||
("get_output_directory", "ComfyUI output"),
|
||||
("get_input_directory", "ComfyUI input"),
|
||||
("get_temp_directory", "ComfyUI temp"),
|
||||
("get_user_directory", "ComfyUI user"),
|
||||
)
|
||||
WATCH_STATES = ("watching", "stopping", "manual", "restart_pending")
|
||||
|
||||
|
||||
def source_form_error(text, *, windows=None):
|
||||
"""The form code that refuses ``text`` as a source path, or None.
|
||||
|
||||
``windows`` selects the path rules (default: this OS). No filesystem
|
||||
access happens here.
|
||||
"""
|
||||
if windows is None:
|
||||
windows = os.name == "nt"
|
||||
if not isinstance(text, str) or len(text) > MAX_SOURCE_PATH:
|
||||
return "invalid-form"
|
||||
if any(ord(char) < 32 or ord(char) == 127 for char in text):
|
||||
return "invalid-form"
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return "empty"
|
||||
return _windows_form_error(text) if windows else _posix_form_error(text)
|
||||
|
||||
|
||||
def _posix_form_error(text):
|
||||
if "\\" in text or _POSIX_FOREIGN.match(text):
|
||||
return "invalid-form" # a Windows path names nothing on this server
|
||||
return "traversal" if ".." in text.split("/") else None
|
||||
|
||||
|
||||
def _windows_form_error(text):
|
||||
portable = text.replace("/", "\\")
|
||||
if portable.startswith(("\\\\?\\", "\\\\.\\")):
|
||||
return "invalid-form" # verbatim and device namespaces
|
||||
if portable.startswith("\\\\"):
|
||||
drive, rest = ntpath.splitdrive(portable)
|
||||
share = drive[2:].split("\\")
|
||||
if len(share) != 2 or any(part in ("", ".", "..") for part in share):
|
||||
return "invalid-form"
|
||||
else:
|
||||
drive, rest = ntpath.splitdrive(portable)
|
||||
if drive and (not _DRIVE.fullmatch(drive) or not rest.startswith("\\")):
|
||||
return "invalid-form" # "C:folder" is relative to that drive's cwd
|
||||
if ":" in rest:
|
||||
return "invalid-form" # an alternate data stream
|
||||
parts = rest.split("\\")
|
||||
if ".." in parts:
|
||||
return "traversal"
|
||||
for part in parts:
|
||||
if part in ("", "."):
|
||||
continue
|
||||
if part[-1] in ". " or _DEVICE_NAME.match(part):
|
||||
return "invalid-form"
|
||||
return None
|
||||
|
||||
|
||||
def source_problem(text):
|
||||
"""``(code, reason)`` that refuses ``text`` as a source, else ``(None, "")``.
|
||||
|
||||
The form check runs first and touches nothing; then containment is
|
||||
decided as text before any filesystem access (see
|
||||
``GalleryConfig.gallery_root_problem``). Every caller answers alike.
|
||||
"""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
code = source_form_error(text)
|
||||
if code is not None:
|
||||
return code, FORM_REASONS[code]
|
||||
return GalleryConfig.gallery_root_problem(text)
|
||||
|
||||
|
||||
def public_spelling(text):
|
||||
"""How a refused source is named in a response, computed as text only.
|
||||
|
||||
Relative to the path anchor it lies under (its public path), else its
|
||||
last part: the caller sent the path, so naming it reveals nothing, and a
|
||||
refused path is never resolved to name it.
|
||||
"""
|
||||
from ..config import GalleryConfig, _lexical_path, _lexically_inside
|
||||
|
||||
text = str(text).strip()
|
||||
try:
|
||||
lexical = os.path.normpath(text)
|
||||
key = _lexical_path(text) # the same length as ``lexical``
|
||||
if not os.path.isabs(text):
|
||||
return lexical.replace("\\", "/")
|
||||
for anchor in GalleryConfig.path_anchors():
|
||||
if _lexically_inside(key, [anchor]):
|
||||
rest = lexical[len(anchor.rstrip(os.sep)) :].lstrip(os.sep)
|
||||
return rest.replace(os.sep, "/") or "."
|
||||
return os.path.basename(lexical) or lexical
|
||||
except (TypeError, ValueError):
|
||||
return ""
|
||||
|
||||
|
||||
def identity(path):
|
||||
"""The comparison key of a directory: resolved and case-folded (Windows)."""
|
||||
return os.path.normcase(os.path.realpath(str(path)))
|
||||
|
||||
|
||||
def _comfyui_labels():
|
||||
"""``{identity: label}`` of the directories ComfyUI reports."""
|
||||
try:
|
||||
import folder_paths
|
||||
except ImportError:
|
||||
return {}
|
||||
labels = {}
|
||||
for getter_name, label in _COMFYUI_LABELS:
|
||||
getter = getattr(folder_paths, getter_name, None)
|
||||
if getter is None:
|
||||
continue
|
||||
try:
|
||||
directory = getter()
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(directory, str) and directory:
|
||||
labels.setdefault(identity(directory), label)
|
||||
return labels
|
||||
|
||||
|
||||
def source_label(path, labels=None):
|
||||
""" "ComfyUI output" for ComfyUI's own directories, else the folder name."""
|
||||
if labels is None:
|
||||
labels = _comfyui_labels()
|
||||
label = labels.get(identity(path))
|
||||
if label:
|
||||
return label
|
||||
return os.path.basename(os.path.normpath(str(path))) or str(path)
|
||||
|
||||
|
||||
def _configured(entry, labels, public_path):
|
||||
from ..config import GalleryConfig
|
||||
|
||||
code, reason = GalleryConfig.gallery_root_problem(entry)
|
||||
try:
|
||||
resolved = GalleryConfig.resolve_gallery_root(entry)
|
||||
except (OSError, TypeError, ValueError):
|
||||
resolved = str(entry)
|
||||
return {
|
||||
"label": source_label(resolved, labels),
|
||||
"path": public_path(resolved),
|
||||
"valid": code is None,
|
||||
"code": code,
|
||||
"reason": reason or None,
|
||||
}
|
||||
|
||||
|
||||
def monitor_state(monitor):
|
||||
"""``(present, observer_alive, scheduled directories)`` of the image monitor."""
|
||||
if monitor is None:
|
||||
return False, False, []
|
||||
status = monitor.get_status()
|
||||
scheduled = [str(d) for d in status.get("monitored_directories") or []]
|
||||
return True, bool(status.get("observer_alive")), scheduled
|
||||
|
||||
|
||||
def watch_state(path, *, present, alive, scheduled_keys, enabled):
|
||||
"""One of ``WATCH_STATES`` (see the module docstring)."""
|
||||
if alive and identity(path) in scheduled_keys:
|
||||
return "watching" if enabled else "stopping"
|
||||
if not present or not enabled:
|
||||
return "manual"
|
||||
return "restart_pending"
|
||||
|
||||
|
||||
def in_comfyui_base(path):
|
||||
"""Whether ``_public_path(path)`` is relative to the ComfyUI base directory.
|
||||
|
||||
The base is the first anchor ``_public_path`` tries, so this agrees with
|
||||
it: the same canonical (realpath + normcase) spelling on both sides.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from ..config import _comfyui_base_directory
|
||||
|
||||
base = _comfyui_base_directory()
|
||||
if not path or not base:
|
||||
return False
|
||||
try:
|
||||
canonical = Path(os.path.normcase(os.path.realpath(str(path))))
|
||||
anchor = Path(os.path.normcase(os.path.realpath(base)))
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
return canonical.is_relative_to(anchor)
|
||||
|
||||
|
||||
def describe_sources(*, configured, output_dirs, auto_detected, monitor, enabled):
|
||||
"""``(sources, monitor)`` for GET /settings (blocking; public paths only).
|
||||
|
||||
``configured`` is the configured roots, ``output_dirs`` and
|
||||
``auto_detected`` what ``_output_dirs_with_origin`` returned, ``monitor``
|
||||
the running ImageMonitor or None, ``enabled`` the monitoring setting.
|
||||
"""
|
||||
from . import _public_path
|
||||
|
||||
labels = _comfyui_labels()
|
||||
present, alive, scheduled = monitor_state(monitor)
|
||||
scheduled_keys = {identity(path) for path in scheduled}
|
||||
effective = [
|
||||
{
|
||||
"label": source_label(path, labels),
|
||||
"path": _public_path(path),
|
||||
"root_index": index,
|
||||
"watch": watch_state(
|
||||
path,
|
||||
present=present,
|
||||
alive=alive,
|
||||
scheduled_keys=scheduled_keys,
|
||||
enabled=enabled,
|
||||
),
|
||||
"in_comfyui": in_comfyui_base(path),
|
||||
}
|
||||
for index, path in enumerate(output_dirs)
|
||||
]
|
||||
return (
|
||||
{
|
||||
"configured": [
|
||||
_configured(entry, labels, _public_path) for entry in configured
|
||||
],
|
||||
"effective": effective,
|
||||
"auto_detected": bool(auto_detected),
|
||||
},
|
||||
{
|
||||
"present": present,
|
||||
"observer_alive": alive,
|
||||
"scheduled": [_public_path(path) for path in scheduled],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def validate_source(text):
|
||||
"""``{valid, label, path, code, reason}`` for one submitted path (blocking).
|
||||
|
||||
Writes nothing. An invalid path answers its code and reason with no
|
||||
label or path, so a response never echoes what was typed.
|
||||
"""
|
||||
from ..config import GalleryConfig
|
||||
from . import _public_path
|
||||
|
||||
code, reason = source_problem(text)
|
||||
if code is not None:
|
||||
return {
|
||||
"valid": False,
|
||||
"label": None,
|
||||
"path": None,
|
||||
"code": code,
|
||||
"reason": reason,
|
||||
}
|
||||
resolved = GalleryConfig.resolve_gallery_root(text)
|
||||
return {
|
||||
"valid": True,
|
||||
"label": source_label(resolved),
|
||||
"path": _public_path(resolved),
|
||||
"code": None,
|
||||
"reason": None,
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
"""System routes: the read-only health check, the Overview and usage statistics.
|
||||
|
||||
- ``POST /prompt_manager/system/health`` starts the health check
|
||||
(``database/health.py``) off-loop as a ``read`` job of kind ``health``: one
|
||||
at a time, beside a scan, not cancellable and with no file outcomes. The
|
||||
API keeps the last report for this session; ``GET`` returns it or null.
|
||||
A report describes the database it was taken from: replacing the database
|
||||
(a restore) forgets it, and a check that started before the swap is not
|
||||
kept.
|
||||
- ``GET /prompt_manager/system/overview`` is the System › Overview aggregate.
|
||||
Every value is real or null: there is no backup status until backups report
|
||||
one and no health result until a check ran; ``needs_review.pending`` is the
|
||||
Needs review queue's pending total. The missing-thumbnail count is not part
|
||||
of it: the page reads ``/images/thumbnails/status``, which walks every
|
||||
source.
|
||||
- ``GET /prompt_manager/system/statistics`` lists runs and ratings per
|
||||
category, with the legacy ``/stats`` totals and the Overview's count of
|
||||
rated images (images whose linked prompt is rated).
|
||||
|
||||
New routes answer errors with ``{success:false, code, error}``.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from collections import namedtuple
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .job_routes import busy_response
|
||||
from .jobs import HEALTH_JOB_KIND, JobBusy
|
||||
|
||||
HEALTH_JOB_TITLE = "Health check"
|
||||
_HEALTH_STATUS_LINES = {
|
||||
"passed": "Health check passed",
|
||||
"issues": "Health check found issues",
|
||||
"incomplete": "Health check incomplete",
|
||||
}
|
||||
|
||||
|
||||
# The last health report and the database generation it was taken from.
|
||||
_StoredReport = namedtuple("_StoredReport", "generation report")
|
||||
|
||||
|
||||
async def _gather_settled(*reads):
|
||||
"""Await every read, then raise the first failure.
|
||||
|
||||
Unlike a plain gather, a failure never answers while the other executor
|
||||
reads still run against the database.
|
||||
"""
|
||||
results = await asyncio.gather(*reads, return_exceptions=True)
|
||||
for result in results:
|
||||
if isinstance(result, BaseException):
|
||||
raise result
|
||||
return results
|
||||
|
||||
|
||||
def _failure(message, status=500, code=None):
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": message}, status=status
|
||||
)
|
||||
|
||||
|
||||
def health_summary(report):
|
||||
"""What the Overview shows of a report: when, the outcome, checks with issues."""
|
||||
if report is None:
|
||||
return None
|
||||
return {
|
||||
"checked_at": report["checked_at"],
|
||||
"status": report["status"],
|
||||
"issues": sum(1 for check in report["checks"] if check["status"] == "issues"),
|
||||
}
|
||||
|
||||
|
||||
class SystemRoutesMixin:
|
||||
"""Mixin providing the /prompt_manager/system endpoints."""
|
||||
|
||||
def _register_system_routes(self, routes):
|
||||
@routes.post("/prompt_manager/system/health")
|
||||
async def start_health_check_route(request):
|
||||
return await self.start_health_check(request)
|
||||
|
||||
@routes.get("/prompt_manager/system/health")
|
||||
async def health_report_route(request):
|
||||
return await self.get_health_report(request)
|
||||
|
||||
@routes.get("/prompt_manager/system/overview")
|
||||
async def system_overview_route(request):
|
||||
return await self.get_system_overview(request)
|
||||
|
||||
@routes.get("/prompt_manager/system/statistics")
|
||||
async def system_statistics_route(request):
|
||||
return await self.get_system_statistics(request)
|
||||
|
||||
# ── health ────────────────────────────────────────────────────────
|
||||
|
||||
def _health_roots(self):
|
||||
"""The sources missing files are judged under.
|
||||
|
||||
The effective sources, plus every configured source that is not an
|
||||
existing directory (an unmounted drive): the check then reports
|
||||
missing files as unavailable instead of counting that source's links.
|
||||
"""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
roots = [str(path) for path in self._get_all_output_dirs()]
|
||||
for configured in GalleryConfig.MONITORING_DIRECTORIES:
|
||||
try:
|
||||
resolved = GalleryConfig.resolve_gallery_root(configured)
|
||||
except (OSError, TypeError, ValueError):
|
||||
continue
|
||||
if not os.path.isdir(resolved):
|
||||
roots.append(resolved)
|
||||
return roots
|
||||
|
||||
def _database_replaced(self):
|
||||
"""Forget what described the previous database (call after swapping ``db``)."""
|
||||
self._db_generation += 1
|
||||
self._health_report = None
|
||||
|
||||
def _current_health_report(self):
|
||||
"""The last report, or None when none was kept for the current database."""
|
||||
stored = self._health_report
|
||||
generation = self._db_generation
|
||||
if isinstance(stored, _StoredReport) and stored.generation == generation:
|
||||
return stored.report
|
||||
return None
|
||||
|
||||
async def start_health_check(self, request):
|
||||
"""Start the health check job (202), or 409 naming the running check."""
|
||||
generation = self._db_generation
|
||||
db = self.db
|
||||
try:
|
||||
roots = await self._run_in_executor(self._health_roots)
|
||||
except Exception:
|
||||
self.logger.exception("Health check could not resolve the image sources")
|
||||
return _failure("Failed to start the health check")
|
||||
|
||||
async def run(job):
|
||||
job.update(status_line="Checking the database and image links")
|
||||
report = await self._run_in_executor(db.run_health_checks, roots)
|
||||
if generation == self._db_generation:
|
||||
self._health_report = _StoredReport(generation, report)
|
||||
else:
|
||||
self.logger.info("Health report dropped: the database was replaced")
|
||||
job.update(progress=100, status_line=_HEALTH_STATUS_LINES[report["status"]])
|
||||
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
HEALTH_JOB_KIND,
|
||||
HEALTH_JOB_TITLE,
|
||||
admission="read",
|
||||
cancellable=False,
|
||||
scope={},
|
||||
runner=run,
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
async def get_health_report(self, request):
|
||||
return web.json_response(
|
||||
{"success": True, "report": self._current_health_report()}
|
||||
)
|
||||
|
||||
# ── overview ──────────────────────────────────────────────────────
|
||||
|
||||
async def get_system_overview(self, request):
|
||||
from ..config import GalleryConfig
|
||||
|
||||
try:
|
||||
(output_dirs, auto_detected), counts, location = await _gather_settled(
|
||||
self._run_in_executor(self._output_dirs_with_origin),
|
||||
self._run_in_executor(self.db.library_counts),
|
||||
self._run_in_executor(self._library_payload),
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("System overview failed")
|
||||
return _failure("Failed to read the overview")
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"library": {
|
||||
"positive_prompts": counts["positive_prompts"],
|
||||
"negative_variants": counts["negative_variants"],
|
||||
},
|
||||
"images": {
|
||||
"indexed": counts["images"],
|
||||
"rated": counts["rated_images"],
|
||||
},
|
||||
"health": health_summary(self._current_health_report()),
|
||||
"backup": None, # no verified backup status exists yet (S8)
|
||||
"needs_review": {"pending": counts["review_pending"]},
|
||||
"sources": {
|
||||
"configured": len(GalleryConfig.MONITORING_DIRECTORIES),
|
||||
"effective": len(output_dirs),
|
||||
"auto_detected": auto_detected,
|
||||
},
|
||||
# Where the library lives and why (py/api/library_location.py).
|
||||
"location": location,
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
# ── statistics ────────────────────────────────────────────────────
|
||||
|
||||
async def get_system_statistics(self, request):
|
||||
try:
|
||||
by_category, totals, counts = await _gather_settled(
|
||||
self._run_in_executor(self.db.usage_by_category),
|
||||
self._run_in_executor(self.db.get_statistics),
|
||||
self._run_in_executor(self.db.library_counts),
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("System statistics failed")
|
||||
return _failure("Failed to read statistics")
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"by_category": by_category,
|
||||
"totals": totals,
|
||||
"rated_images": counts["rated_images"],
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
@@ -0,0 +1,843 @@
|
||||
"""Automatic thumbnails (#249): previews are a cache the server keeps itself.
|
||||
|
||||
The approved mock has no repeated global thumbnail prompt
|
||||
(``docs/concepts/workspace/README.md``), so the workspace never asks. Every
|
||||
listing gives an image or video a thumbnail URL (``media_thumbnail``), and the
|
||||
previews behind those URLs are made here, in three ways:
|
||||
|
||||
- **On first view.** ``GET /prompt_manager/images/serve/thumbnails/<rel>_thumb
|
||||
<ext>`` for a preview that does not exist yet is admitted on the event loop
|
||||
(``_PreviewGate``): at most ``ON_DEMAND_LIMIT`` previews are written at once,
|
||||
on this API's own small pool (never the default executor), one flight runs
|
||||
per file, and the requests waiting are capped (``MAX_WAITING``); past the
|
||||
cap a request gets its fallback at once. Inside the admission the preview is
|
||||
mapped back to its media file under the same root (the request text is
|
||||
checked first: no traversal, drive, UNC or empty segment; then a regular
|
||||
file inside the root, never through a link), made and served. When it
|
||||
cannot be made an image's original is served and a video's placeholder
|
||||
frame, both ``Cache-Control: no-store``: a video is never sent to an
|
||||
``<img>``. A file that cannot be decoded is remembered for this session
|
||||
(keyed by its path, size and mtime) and counted for the System Overview
|
||||
(``failed`` in ``/images/thumbnails/status``); I/O errors and a preview
|
||||
another program holds open (Windows) are not remembered. The recorded
|
||||
history's thumbnail route goes through the same admission and flight key.
|
||||
- **On capture.** The image monitor hands every new output it settles to
|
||||
``queue_capture_thumbnail``, which queues its preview for one background
|
||||
worker thread (``AutoThumbnailer``). Each wake takes at most Worker threads
|
||||
(``WORKER_THREADS``) files and writes them on that many threads; enqueuing
|
||||
is O(1), so the monitor and ComfyUI's queue never wait. A file already
|
||||
queued is not queued again, the queue is bounded (``MAX_QUEUED``) and a
|
||||
preview left out is made on its first view instead.
|
||||
- **After a scan.** A scan that added files checks the missing previews; from
|
||||
``SCAN_THUMBNAIL_JOB_THRESHOLD`` up it starts the thumbnail Activity job
|
||||
(missing only, visible and cancellable) instead of writing them silently,
|
||||
and fewer go through the worker.
|
||||
|
||||
Every preview is written by ``_generate_one`` with the thumbnail job's default
|
||||
size and quality (``DEFAULT_THUMBNAIL_EDGE``, medium), into a temp sibling
|
||||
renamed over the old one, so a reader never sees a partial file.
|
||||
|
||||
Holds: automatic previews only write files under ``thumbnails/`` and never
|
||||
touch the database, like the thumbnail job, so they run beside a scan, image
|
||||
tagging and the Needs review recording (S5 M7). They are not written while a
|
||||
job runs that replaces the database or removes media and previews: a restore,
|
||||
a duplicate removal, a sources change, the gallery's thumbnail clear or an
|
||||
``exclusive`` job (``_auto_thumbnail_blocker``). The holds are checked when a
|
||||
write starts (after its slot) and again just before its rename.
|
||||
|
||||
Shutdown: ``close_auto_thumbnails`` stops the worker at a file boundary,
|
||||
kills a running ffmpeg, waits a bounded time (and logs when the wait ran
|
||||
out) and shuts the view pool down; after the app's cleanup
|
||||
(``final=True``) nothing starts a worker again. Importing this module again
|
||||
(a package reload) stops the workers the previous import left.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import collections
|
||||
import functools
|
||||
import io
|
||||
import os
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .images import (
|
||||
GALLERY_IMAGE_EXTENSIONS,
|
||||
MEDIA_EXTENSIONS,
|
||||
THUMBNAIL_CLEAR_TITLE,
|
||||
THUMBNAIL_ENCODINGS,
|
||||
VIDEO_EXTENSIONS,
|
||||
_has_traversal,
|
||||
_is_within,
|
||||
_thumbnail_is_fresh,
|
||||
_video_placeholder_image,
|
||||
_worker_threads,
|
||||
media_thumbnail_rel,
|
||||
)
|
||||
from .jobs import (
|
||||
DUPLICATE_CLEANUP_JOB_KIND,
|
||||
RESTORE_JOB_KIND,
|
||||
THUMBNAILS_JOB_KIND,
|
||||
JobBusy,
|
||||
)
|
||||
from .settings_routes import SOURCE_CHANGE_JOB_KIND
|
||||
from .thumbnail_jobs import DEFAULT_THUMBNAIL_EDGE, ThumbnailOptions
|
||||
|
||||
try:
|
||||
from ...utils.parallel import map_parallel
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils.parallel import map_parallel
|
||||
|
||||
AUTO_THUMBNAIL_EDGE = DEFAULT_THUMBNAIL_EDGE
|
||||
AUTO_THUMBNAIL_SIZE = (AUTO_THUMBNAIL_EDGE, AUTO_THUMBNAIL_EDGE)
|
||||
AUTO_THUMBNAIL_ENCODING = THUMBNAIL_ENCODINGS["medium"]
|
||||
# Previews views may write at once (the size of the view pool).
|
||||
ON_DEMAND_LIMIT = 3
|
||||
# View requests that may wait on the gate; one more gets its fallback at once.
|
||||
MAX_WAITING = 64
|
||||
# How long a pool thread waits for the capture worker's flight of the same file.
|
||||
FLIGHT_WAIT_SECONDS = 20
|
||||
# How long close() waits for the worker's batch in hand (ffmpeg's own limit).
|
||||
CLOSE_WAIT_SECONDS = 30
|
||||
# Files the capture worker holds; more are made on their first view.
|
||||
MAX_QUEUED = 10_000
|
||||
# Undecodable files remembered per session (oldest forgotten first).
|
||||
MAX_FAILURES_REMEMBERED = 4096
|
||||
# Missing previews after a scan from which the visible thumbnail job starts:
|
||||
# a fresh install or a newly added source. Fewer go through the worker.
|
||||
SCAN_THUMBNAIL_JOB_THRESHOLD = 200
|
||||
# Jobs beside which no automatic preview is written (see the module docstring).
|
||||
AUTO_REFUSED_KINDS = frozenset(
|
||||
{RESTORE_JOB_KIND, DUPLICATE_CLEANUP_JOB_KIND, SOURCE_CHANGE_JOB_KIND}
|
||||
)
|
||||
WORKER_NAME = "PromptManagerThumbnailWorker"
|
||||
POOL_PREFIX = "pm-preview"
|
||||
# Identifies this import's workers: a reload stops the ones it did not start.
|
||||
_MODULE_TOKEN = object()
|
||||
|
||||
READY, FAILED, BUSY, DEFERRED = "ready", "failed", "busy", "deferred"
|
||||
# What the gate answers a request past MAX_WAITING.
|
||||
OVER = object()
|
||||
|
||||
|
||||
def _fingerprint(src):
|
||||
"""``(path, size, mtime_ns)`` of *src*, or None when it cannot be read."""
|
||||
try:
|
||||
stat = os.stat(src)
|
||||
except OSError:
|
||||
return None
|
||||
return (os.path.normcase(str(src)), stat.st_size, stat.st_mtime_ns)
|
||||
|
||||
|
||||
def flight_key(root, rel_parts):
|
||||
"""The single-flight key of ``root/rel_parts``: its folder resolved, case
|
||||
folded. The serve route, the history route and the worker share it."""
|
||||
path = Path(root).joinpath(*rel_parts)
|
||||
return os.path.normcase(os.path.join(os.path.realpath(path.parent), path.name))
|
||||
|
||||
|
||||
def _dst_key(dst):
|
||||
return flight_key(Path(dst).parent, (Path(dst).name,))
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def video_placeholder_jpeg():
|
||||
"""The placeholder frame as JPEG bytes (a video whose preview is missing)."""
|
||||
buffer = io.BytesIO()
|
||||
_video_placeholder_image(AUTO_THUMBNAIL_SIZE).save(
|
||||
buffer, "JPEG", **AUTO_THUMBNAIL_ENCODING
|
||||
)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _is_preview_request(filepath):
|
||||
"""Text check only: ``thumbnails/<...>/<name>_thumb<lower-case ext>``.
|
||||
|
||||
Refuses, as text and before any path is joined, what a joined path could
|
||||
take as an anchor: a backslash, a drive or stream colon, and empty, ``.``
|
||||
or ``..`` segments (so a leading ``/`` and ``//server/share`` too).
|
||||
"""
|
||||
if "\\" in filepath or ":" in filepath:
|
||||
return False
|
||||
parts = filepath.split("/")
|
||||
if len(parts) < 2 or parts[0] != "thumbnails":
|
||||
return False
|
||||
if any(part in ("", ".", "..") for part in parts) or "thumbnails" in parts[1:-1]:
|
||||
return False
|
||||
stem, extension = os.path.splitext(parts[-1])
|
||||
return (
|
||||
stem.endswith("_thumb")
|
||||
and len(stem) > len("_thumb")
|
||||
and extension == extension.lower()
|
||||
and extension in GALLERY_IMAGE_EXTENSIONS
|
||||
)
|
||||
|
||||
|
||||
def _regular_file(path):
|
||||
"""True for a regular file that is not a link (one lstat)."""
|
||||
try:
|
||||
return not path.is_symlink() and path.is_file()
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _candidates(parts):
|
||||
"""``(folder parts, base, extensions)`` of a checked preview request."""
|
||||
stem, extension = os.path.splitext(parts[-1])
|
||||
wanted = [extension] + (sorted(VIDEO_EXTENSIONS) if extension == ".jpg" else [])
|
||||
return parts[1:-1], stem[: -len("_thumb")], wanted
|
||||
|
||||
|
||||
def _quick_source(root, parts):
|
||||
"""``(src, is_video)`` by exact name only (no folder listing), or None."""
|
||||
root = Path(root)
|
||||
folder_parts, base, wanted = _candidates(parts)
|
||||
folder = root.joinpath(*folder_parts)
|
||||
for extension in wanted:
|
||||
for name in (base + extension, base + extension.upper()):
|
||||
path = folder / name
|
||||
if _regular_file(path) and _is_within(path, root):
|
||||
return path, extension in VIDEO_EXTENSIONS
|
||||
return None
|
||||
|
||||
|
||||
def _thumbnail_source(root, parts):
|
||||
"""``(src, dst, is_video)`` for the preview *parts* under *root*, or None.
|
||||
|
||||
*parts* passed ``_is_preview_request``. The source has the preview's name
|
||||
without ``_thumb`` and an extension that previews as the requested one
|
||||
(any case: exact names first, then one listing of the folder); a ``.jpg``
|
||||
may also be a video's. It must be a regular file inside *root*. ``dst``
|
||||
is None when the preview's folder resolves outside ``root/thumbnails`` (a
|
||||
link): nothing is written there.
|
||||
"""
|
||||
root = Path(root)
|
||||
folder_parts, base, wanted = _candidates(parts)
|
||||
quick = _quick_source(root, parts)
|
||||
if quick is not None:
|
||||
found = quick[0]
|
||||
else:
|
||||
found = _scan_folder(root.joinpath(*folder_parts), base, set(wanted))
|
||||
if found is None or not _is_within(found, root):
|
||||
return None
|
||||
rel = Path(*folder_parts, found.name)
|
||||
dst = root / media_thumbnail_rel(rel)
|
||||
if dst != root.joinpath(*parts):
|
||||
return None
|
||||
if not _is_within(root / "thumbnails", root) or not _is_within(
|
||||
dst, root / "thumbnails"
|
||||
):
|
||||
dst = None
|
||||
return found, dst, found.suffix.lower() in VIDEO_EXTENSIONS
|
||||
|
||||
|
||||
def _scan_folder(folder, base, extensions):
|
||||
"""A mixed-case extension (``.Png``): one listing of *folder*, or None."""
|
||||
try:
|
||||
with os.scandir(folder) as entries:
|
||||
for entry in entries:
|
||||
stem, extension = os.path.splitext(entry.name)
|
||||
if stem == base and extension.lower() in extensions:
|
||||
path = Path(entry.path)
|
||||
if _regular_file(path):
|
||||
return path
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _stop_stale_workers():
|
||||
"""Stop capture workers a previous import of this module started."""
|
||||
for thread in threading.enumerate():
|
||||
token = getattr(thread, "pm_preview_token", None)
|
||||
close = getattr(thread, "pm_preview_close", None)
|
||||
if token is not None and token is not _MODULE_TOKEN and close is not None:
|
||||
close()
|
||||
|
||||
|
||||
class AutoThumbnailer:
|
||||
"""Single-flight preview writes and the capture worker (thread-safe).
|
||||
|
||||
``generate(src, dst, is_video, *, stop, guard, track)`` writes one
|
||||
preview and returns ``_generate_one``'s result dict; ``may_write()``
|
||||
says whether automatic previews may be written now. Views call
|
||||
``ensure`` on the view pool, the monitor calls ``enqueue`` from its
|
||||
thread.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
generate,
|
||||
may_write,
|
||||
logger,
|
||||
*,
|
||||
workers=_worker_threads,
|
||||
max_queued=MAX_QUEUED,
|
||||
start_worker=True,
|
||||
):
|
||||
self._generate = generate
|
||||
self._may_write = may_write
|
||||
self._logger = logger
|
||||
self._workers = workers
|
||||
self._max_queued = max_queued
|
||||
self._start_worker = start_worker
|
||||
self._changed = threading.Condition()
|
||||
self._inflight = {}
|
||||
self._failures = collections.OrderedDict()
|
||||
self._queue = collections.deque()
|
||||
self._queued = set()
|
||||
self._processes = []
|
||||
self._worker = None
|
||||
# True from a worker's start until it leaves (under the lock), so an
|
||||
# idle worker can exit and the next enqueue starts a fresh one.
|
||||
self._worker_running = False
|
||||
self._closed = False
|
||||
self._busy = False
|
||||
self._batch = 0
|
||||
|
||||
@property
|
||||
def failure_count(self):
|
||||
with self._changed:
|
||||
return len(self._failures)
|
||||
|
||||
@property
|
||||
def closed(self):
|
||||
return self._closed
|
||||
|
||||
def _allowed(self):
|
||||
return not self._closed and self._may_write()
|
||||
|
||||
def _track(self, process):
|
||||
"""Keep an ffmpeg process so close() can kill it."""
|
||||
with self._changed:
|
||||
self._processes = [p for p in self._processes if p.poll() is None]
|
||||
self._processes.append(process)
|
||||
closed = self._closed
|
||||
if closed:
|
||||
process.kill()
|
||||
|
||||
# ── one preview ───────────────────────────────────────────────────
|
||||
|
||||
def ensure(self, src, dst, is_video):
|
||||
"""Make *dst* for *src* unless it is fresh; returns the outcome.
|
||||
|
||||
``ready`` (the preview is there), ``failed`` (the file cannot be
|
||||
decoded, remembered), ``busy`` (an I/O error, the file is gone, or
|
||||
another flight did not finish) or ``deferred`` (a hold refuses
|
||||
automatic writes, checked when the write starts and before its
|
||||
rename). Blocking: call it off the event loop.
|
||||
"""
|
||||
if _thumbnail_is_fresh(src, dst):
|
||||
return READY
|
||||
fingerprint = _fingerprint(src)
|
||||
if fingerprint is None:
|
||||
return BUSY
|
||||
key = _dst_key(dst)
|
||||
with self._changed:
|
||||
if fingerprint in self._failures:
|
||||
return FAILED
|
||||
flight = self._inflight.get(key)
|
||||
if flight is None:
|
||||
self._inflight[key] = done = threading.Event()
|
||||
if flight is not None:
|
||||
flight.wait(FLIGHT_WAIT_SECONDS)
|
||||
if _thumbnail_is_fresh(src, dst):
|
||||
return READY
|
||||
with self._changed:
|
||||
return FAILED if fingerprint in self._failures else BUSY
|
||||
try:
|
||||
return self._lead(src, dst, is_video, fingerprint)
|
||||
finally:
|
||||
with self._changed:
|
||||
self._inflight.pop(key, None)
|
||||
self._changed.notify_all()
|
||||
done.set()
|
||||
|
||||
def _lead(self, src, dst, is_video, fingerprint):
|
||||
if not self._allowed():
|
||||
return DEFERRED
|
||||
if _thumbnail_is_fresh(src, dst):
|
||||
return READY
|
||||
result = self._generate(
|
||||
src,
|
||||
dst,
|
||||
is_video,
|
||||
stop=lambda: self._closed,
|
||||
guard=self._allowed,
|
||||
track=self._track,
|
||||
)
|
||||
if result.get("action") in ("generated", "skipped"):
|
||||
return READY
|
||||
if result.get("held"):
|
||||
return DEFERRED
|
||||
if not result.get("decode"):
|
||||
return BUSY # an I/O error says nothing about the file
|
||||
error = result.get("error") or "failed"
|
||||
with self._changed:
|
||||
self._failures[fingerprint] = error
|
||||
while len(self._failures) > MAX_FAILURES_REMEMBERED:
|
||||
self._failures.popitem(last=False)
|
||||
self._logger.info(
|
||||
f"No preview for {os.path.basename(str(src))} ({error}); "
|
||||
"the original is shown"
|
||||
)
|
||||
return FAILED
|
||||
|
||||
# ── the capture worker ────────────────────────────────────────────
|
||||
|
||||
def enqueue(self, src, dst, is_video):
|
||||
"""Queue *dst*'s preview for the worker; False when already queued,
|
||||
full or closed. Never blocks on a write."""
|
||||
key = _dst_key(dst)
|
||||
with self._changed:
|
||||
if self._closed or key in self._queued:
|
||||
return False
|
||||
if len(self._queue) >= self._max_queued:
|
||||
return False
|
||||
self._queued.add(key)
|
||||
self._queue.append((key, Path(src), Path(dst), bool(is_video)))
|
||||
self._busy = True
|
||||
if self._start_worker and not self._worker_running:
|
||||
self._start_worker_locked()
|
||||
self._changed.notify_all()
|
||||
return True
|
||||
|
||||
def _start_worker_locked(self):
|
||||
"""Start the worker (lock held). A start that fails leaves the file
|
||||
queued and the next enqueue tries again."""
|
||||
worker = threading.Thread(target=self._run, name=WORKER_NAME, daemon=True)
|
||||
worker.pm_preview_token = _MODULE_TOKEN
|
||||
worker.pm_preview_close = self.close
|
||||
self._worker_running = True
|
||||
try:
|
||||
worker.start()
|
||||
except RuntimeError:
|
||||
self._worker_running = False
|
||||
self._busy = False
|
||||
self._logger.warning(
|
||||
"The automatic preview worker could not start; the next output "
|
||||
"tries again",
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
self._worker = worker
|
||||
|
||||
def _run(self):
|
||||
try:
|
||||
self._drain()
|
||||
except Exception:
|
||||
self._logger.warning("The automatic preview worker stopped", exc_info=True)
|
||||
finally:
|
||||
# Whatever ended the loop, the next enqueue starts a fresh worker.
|
||||
with self._changed:
|
||||
self._worker_running = False
|
||||
self._busy = False
|
||||
self._batch = 0
|
||||
self._changed.notify_all()
|
||||
|
||||
def _drain(self):
|
||||
while True:
|
||||
with self._changed:
|
||||
if self._closed or not self._queue:
|
||||
# Idle or closed: leave, so no thread outlives its work.
|
||||
self._busy = False
|
||||
self._worker_running = False
|
||||
self._changed.notify_all()
|
||||
return
|
||||
take = min(len(self._queue), max(1, int(self._workers() or 1)))
|
||||
batch = [self._queue.popleft() for _ in range(take)]
|
||||
self._batch = len(batch)
|
||||
self._changed.notify_all()
|
||||
try:
|
||||
map_parallel(self._write_queued, batch, len(batch))
|
||||
finally:
|
||||
with self._changed:
|
||||
self._batch = 0
|
||||
for key, *_rest in batch:
|
||||
self._queued.discard(key)
|
||||
|
||||
def _write_queued(self, item):
|
||||
_key, src, dst, is_video = item
|
||||
try:
|
||||
self.ensure(src, dst, is_video)
|
||||
except Exception: # one file never stops the worker
|
||||
self._logger.warning(
|
||||
f"Automatic preview failed for {src.name}", exc_info=True
|
||||
)
|
||||
|
||||
def close(self):
|
||||
"""Stop the worker at its next file boundary (queued files are
|
||||
dropped), kill a running ffmpeg and wait up to CLOSE_WAIT_SECONDS;
|
||||
a worker still running after that is logged."""
|
||||
with self._changed:
|
||||
self._closed = True
|
||||
self._queue.clear()
|
||||
self._queued.clear()
|
||||
processes, self._processes = self._processes, []
|
||||
self._changed.notify_all()
|
||||
worker = self._worker
|
||||
for process in processes:
|
||||
try:
|
||||
process.kill()
|
||||
except OSError:
|
||||
pass
|
||||
if worker is None or worker is threading.current_thread():
|
||||
return
|
||||
worker.join(CLOSE_WAIT_SECONDS)
|
||||
if worker.is_alive():
|
||||
self._logger.warning(
|
||||
"The automatic preview worker is still running after "
|
||||
f"{CLOSE_WAIT_SECONDS} s; it stops at its next file"
|
||||
)
|
||||
|
||||
# ── waits (tests and shutdown; never timers) ──────────────────────
|
||||
|
||||
def _wait_for(self, predicate, timeout):
|
||||
with self._changed:
|
||||
return self._changed.wait_for(predicate, timeout)
|
||||
|
||||
def wait_idle(self, timeout=None):
|
||||
"""True once the worker has nothing queued or in hand."""
|
||||
return self._wait_for(lambda: not self._busy, timeout)
|
||||
|
||||
def wait_batch(self, size, timeout=None):
|
||||
"""True once the worker holds a batch of *size* files."""
|
||||
return self._wait_for(lambda: self._batch == size, timeout)
|
||||
|
||||
|
||||
class _PreviewGate:
|
||||
"""Admission of view requests on the event loop (#249 I2).
|
||||
|
||||
Waiters are coroutines: at most ``limit`` previews are written at once
|
||||
on the view pool, one flight runs per key, and past ``MAX_WAITING``
|
||||
waiting requests ``run`` answers ``OVER`` at once.
|
||||
"""
|
||||
|
||||
def __init__(self, limit, logger):
|
||||
self.loop = asyncio.get_running_loop()
|
||||
self._slots = asyncio.Semaphore(limit)
|
||||
self._flights = {}
|
||||
self._tasks = set()
|
||||
self._logger = logger
|
||||
self._tick = asyncio.Event()
|
||||
self.waiting = 0
|
||||
|
||||
def _changed(self):
|
||||
tick, self._tick = self._tick, asyncio.Event()
|
||||
tick.set()
|
||||
|
||||
async def settled(self, predicate):
|
||||
"""Return once ``predicate()`` holds (re-checked at each change)."""
|
||||
while not predicate():
|
||||
await self._tick.wait()
|
||||
|
||||
async def run(self, key, work, pool):
|
||||
"""``work()``'s answer for *key* (shared by every request of the
|
||||
flight), None when it raised, or OVER past the cap."""
|
||||
if self.waiting >= MAX_WAITING:
|
||||
return OVER
|
||||
flight = self._flights.get(key)
|
||||
if flight is None:
|
||||
flight = self.loop.create_future()
|
||||
self._flights[key] = flight
|
||||
task = self.loop.create_task(self._lead(key, flight, work, pool))
|
||||
self._tasks.add(task)
|
||||
task.add_done_callback(self._tasks.discard)
|
||||
self.waiting += 1
|
||||
self._changed()
|
||||
try:
|
||||
return await asyncio.shield(flight)
|
||||
finally:
|
||||
self.waiting -= 1
|
||||
self._changed()
|
||||
|
||||
async def _lead(self, key, flight, work, pool):
|
||||
result = None
|
||||
try:
|
||||
async with self._slots:
|
||||
result = await self.loop.run_in_executor(pool, work)
|
||||
except Exception:
|
||||
self._logger.warning("Automatic preview failed", exc_info=True)
|
||||
finally:
|
||||
self._flights.pop(key, None)
|
||||
if not flight.done():
|
||||
flight.set_result(result)
|
||||
|
||||
|
||||
class AutoThumbnailMixin:
|
||||
"""Automatic previews on the API: views, the monitor's queue, scans."""
|
||||
|
||||
_auto_thumbnailer = None
|
||||
_auto_thumbnails_shut = False
|
||||
_preview_gate_obj = None
|
||||
_preview_pool_obj = None
|
||||
_auto_thumbnailer_lock = threading.Lock()
|
||||
|
||||
def _auto_thumbnails(self):
|
||||
"""This API's AutoThumbnailer, created on first use (no worker after
|
||||
the app's cleanup)."""
|
||||
with AutoThumbnailMixin._auto_thumbnailer_lock:
|
||||
if self._auto_thumbnailer is None:
|
||||
self._auto_thumbnailer = AutoThumbnailer(
|
||||
self._write_auto_thumbnail,
|
||||
lambda: self._auto_thumbnail_blocker() is None,
|
||||
self.logger,
|
||||
start_worker=not self._auto_thumbnails_shut,
|
||||
)
|
||||
return self._auto_thumbnailer
|
||||
|
||||
def close_auto_thumbnails(self, final=False):
|
||||
"""Stop the capture worker and the view pool, and forget them.
|
||||
|
||||
Bounded (see AutoThumbnailer.close). ``final`` (the app's cleanup)
|
||||
also keeps a late monitor event from starting a worker again; a
|
||||
test reset passes False and the next use starts fresh ones.
|
||||
"""
|
||||
with AutoThumbnailMixin._auto_thumbnailer_lock:
|
||||
if final:
|
||||
self._auto_thumbnails_shut = True
|
||||
thumbnailer, self._auto_thumbnailer = self._auto_thumbnailer, None
|
||||
pool, self._preview_pool_obj = self._preview_pool_obj, None
|
||||
self._preview_gate_obj = None
|
||||
if thumbnailer is not None:
|
||||
thumbnailer.close()
|
||||
if pool is not None:
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
def _auto_thumbnail_failures(self):
|
||||
thumbnailer = self._auto_thumbnailer
|
||||
return 0 if thumbnailer is None else thumbnailer.failure_count
|
||||
|
||||
def _write_auto_thumbnail(self, src, dst, is_video, *, stop, guard, track):
|
||||
return self._generate_one(
|
||||
src,
|
||||
dst,
|
||||
AUTO_THUMBNAIL_SIZE,
|
||||
is_video,
|
||||
encoding=AUTO_THUMBNAIL_ENCODING,
|
||||
stop=stop,
|
||||
guard=guard,
|
||||
track=track,
|
||||
)
|
||||
|
||||
def _auto_thumbnail_blocker(self):
|
||||
"""The running job automatic previews wait for, or None (any thread)."""
|
||||
for job in self._jobs.running():
|
||||
if job.admission == "exclusive" or job.kind in AUTO_REFUSED_KINDS:
|
||||
return job
|
||||
if job.kind == THUMBNAILS_JOB_KIND and job.title == THUMBNAIL_CLEAR_TITLE:
|
||||
return job
|
||||
return None
|
||||
|
||||
# ── the view admission ────────────────────────────────────────────
|
||||
|
||||
def _preview_gate(self):
|
||||
"""The gate of the running loop (a new loop gets a new one)."""
|
||||
gate = self._preview_gate_obj
|
||||
if gate is None or gate.loop is not asyncio.get_running_loop():
|
||||
gate = self._preview_gate_obj = _PreviewGate(ON_DEMAND_LIMIT, self.logger)
|
||||
return gate
|
||||
|
||||
def _preview_pool(self):
|
||||
with AutoThumbnailMixin._auto_thumbnailer_lock:
|
||||
if self._preview_pool_obj is None:
|
||||
self._preview_pool_obj = ThreadPoolExecutor(
|
||||
ON_DEMAND_LIMIT, thread_name_prefix=POOL_PREFIX
|
||||
)
|
||||
return self._preview_pool_obj
|
||||
|
||||
def _preview_pool_threads(self):
|
||||
pool = self._preview_pool_obj
|
||||
return [] if pool is None else list(getattr(pool, "_threads", ()))
|
||||
|
||||
def _video_placeholder_bytes(self):
|
||||
return video_placeholder_jpeg()
|
||||
|
||||
async def _admit_preview(self, src, dst, is_video):
|
||||
"""Make one preview through the view admission (the history route).
|
||||
|
||||
Shares the serve route's flight key; every flight answers the
|
||||
``ensure`` outcome (or None, or OVER), whichever route started it.
|
||||
"""
|
||||
return await self._preview_gate().run(
|
||||
flight_key(Path(dst).parent, (Path(dst).name,)),
|
||||
functools.partial(self._ensure_view, src, dst, is_video),
|
||||
self._preview_pool(),
|
||||
)
|
||||
|
||||
def _ensure_view(self, src, dst, is_video):
|
||||
"""``ensure``, reporting a preview removed under us as busy."""
|
||||
outcome = self._auto_thumbnails().ensure(src, dst, is_video)
|
||||
if outcome == READY and not Path(dst).is_file():
|
||||
return BUSY # removed under us: the fallback this once
|
||||
return outcome
|
||||
|
||||
# ── on first view ─────────────────────────────────────────────────
|
||||
|
||||
def _preview_on_demand(self, root, parts):
|
||||
"""The ``ensure`` outcome for the preview *parts* under *root*
|
||||
(``deferred`` when its folder is linked out of the root), or None
|
||||
when no media file previews as *parts*."""
|
||||
located = _thumbnail_source(root, parts)
|
||||
if located is None:
|
||||
return None
|
||||
src, dst, is_video = located
|
||||
if dst is None:
|
||||
return DEFERRED
|
||||
return self._ensure_view(src, dst, is_video)
|
||||
|
||||
def _preview_response(self, outcome, src, dst, is_video):
|
||||
if outcome == READY and dst is not None:
|
||||
return self._file_response(dst)
|
||||
if is_video:
|
||||
return web.Response(
|
||||
body=video_placeholder_jpeg(),
|
||||
content_type="image/jpeg",
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
response = web.FileResponse(src)
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return response
|
||||
|
||||
async def _serve_missing_thumbnail(self, output_dirs, filepath):
|
||||
"""The made preview, or its fallback; None when *filepath* names no
|
||||
preview of a media file in *output_dirs*.
|
||||
|
||||
The flight answers only the outcome (a history flight may be the one
|
||||
joined); the file it previews is derived here: the preview path is
|
||||
the request's own, and a fallback looks its source up again on the
|
||||
view pool (bounded by it, never the default executor).
|
||||
"""
|
||||
if _has_traversal(filepath) or not _is_preview_request(filepath):
|
||||
return None
|
||||
parts = tuple(filepath.split("/"))
|
||||
gate = self._preview_gate()
|
||||
pool = self._preview_pool()
|
||||
for root in output_dirs:
|
||||
root = Path(root)
|
||||
outcome = await gate.run(
|
||||
flight_key(root, parts),
|
||||
functools.partial(self._preview_on_demand, root, parts),
|
||||
pool,
|
||||
)
|
||||
if outcome is None:
|
||||
continue
|
||||
if outcome is OVER:
|
||||
# Past the cap: exact names only, no listing, no write.
|
||||
quick = _quick_source(root, parts)
|
||||
if quick is None:
|
||||
continue
|
||||
return self._preview_response(BUSY, quick[0], None, quick[1])
|
||||
dst = root.joinpath(*parts)
|
||||
if outcome == READY:
|
||||
return self._file_response(dst)
|
||||
located = await gate.loop.run_in_executor(
|
||||
pool, _thumbnail_source, root, parts
|
||||
)
|
||||
if located is None:
|
||||
continue
|
||||
src, _dst, is_video = located
|
||||
return self._preview_response(outcome, src, None, is_video)
|
||||
return None
|
||||
|
||||
# ── on capture ────────────────────────────────────────────────────
|
||||
|
||||
def queue_capture_thumbnail(self, image_path):
|
||||
"""Queue the preview of a new output (the monitor's thread).
|
||||
|
||||
Returns ``queued``, ``fresh`` (its preview is current) or ``skipped``
|
||||
(not a media file under a current root, a link, a preview itself,
|
||||
already queued, or after the app's cleanup). Never writes on the
|
||||
caller's thread.
|
||||
"""
|
||||
if self._auto_thumbnails_shut:
|
||||
return "skipped"
|
||||
located = self._capture_target(image_path)
|
||||
if located is None:
|
||||
return "skipped"
|
||||
src, dst, is_video = located
|
||||
if _thumbnail_is_fresh(src, dst):
|
||||
return "fresh"
|
||||
if self._auto_thumbnails().enqueue(src, dst, is_video):
|
||||
return "queued"
|
||||
return "skipped"
|
||||
|
||||
def _capture_target(self, image_path):
|
||||
path = Path(os.path.abspath(str(image_path)))
|
||||
if path.suffix.lower() not in MEDIA_EXTENSIONS or not _regular_file(path):
|
||||
return None
|
||||
folded = os.path.normcase(str(path))
|
||||
for root in self._get_all_output_dirs():
|
||||
root = Path(os.path.abspath(str(root)))
|
||||
base = os.path.normcase(str(root)).rstrip("\\/") + os.sep
|
||||
if not folded.startswith(base) or not _is_within(path, root):
|
||||
continue
|
||||
rel = Path(os.path.relpath(path, root))
|
||||
if "thumbnails" in rel.parts[:-1]:
|
||||
return None
|
||||
dst = root / media_thumbnail_rel(rel)
|
||||
if not _is_within(root / "thumbnails", root) or not _is_within(
|
||||
dst, root / "thumbnails"
|
||||
):
|
||||
return None
|
||||
return path, dst, path.suffix.lower() in VIDEO_EXTENSIONS
|
||||
return None
|
||||
|
||||
# ── after a scan ──────────────────────────────────────────────────
|
||||
|
||||
async def _thumbnails_after_scan(self, job, output_dirs=None):
|
||||
"""Hand a finished scan's missing previews on; returns what it did.
|
||||
|
||||
None (the scan added nothing), ``job`` (the thumbnail job started for
|
||||
``SCAN_THUMBNAIL_JOB_THRESHOLD`` or more missing previews), ``busy``
|
||||
(a job refused it; the previews are made on view), ``queued`` (fewer,
|
||||
through the worker) or ``none`` (nothing missing). It counts the
|
||||
sources the scan walked (``output_dirs``, narrowed to the job's
|
||||
``roots``; every source when None). The server log says which.
|
||||
"""
|
||||
if job.counters.get("added", 0) <= 0:
|
||||
return None
|
||||
if output_dirs is None:
|
||||
output_dirs = await self._run_in_executor(self._get_all_output_dirs)
|
||||
roots = job.scope.get("roots")
|
||||
if roots:
|
||||
output_dirs = [output_dirs[i] for i in roots if 0 <= i < len(output_dirs)]
|
||||
plan = await self._run_in_executor(self._plan_gallery_thumbnails, output_dirs)
|
||||
missing = len(plan["targets"]) + plan["remaining"]
|
||||
if missing >= SCAN_THUMBNAIL_JOB_THRESHOLD:
|
||||
options = ThumbnailOptions("missing", AUTO_THUMBNAIL_EDGE, "medium")
|
||||
try:
|
||||
self._start_thumbnail_job(options)
|
||||
except JobBusy as busy:
|
||||
self.logger.info(
|
||||
f"{missing:,} previews missing after the scan; the thumbnail "
|
||||
f"job did not start while {busy.running.title} runs"
|
||||
)
|
||||
return "busy"
|
||||
self.logger.info(
|
||||
f"{missing:,} previews missing after the scan; thumbnail job started"
|
||||
)
|
||||
return "job"
|
||||
if not missing:
|
||||
return "none"
|
||||
thumbnailer = self._auto_thumbnails()
|
||||
for src, dst, is_video in plan["targets"]:
|
||||
thumbnailer.enqueue(src, dst, is_video)
|
||||
self.logger.info(f"{missing:,} previews missing after the scan; queued")
|
||||
return "queued"
|
||||
|
||||
async def _hand_off_scan_thumbnails(self, job, output_dirs=None):
|
||||
"""``_thumbnails_after_scan`` for the scan runner: never raises."""
|
||||
try:
|
||||
return await self._thumbnails_after_scan(job, output_dirs)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
self.logger.warning("Previews after the scan failed", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
# A package reload re-runs this module: stop the workers the old one left.
|
||||
_stop_stale_workers()
|
||||
@@ -0,0 +1,494 @@
|
||||
"""Thumbnail generation as an Activity job (#249 System S5, Task 12).
|
||||
|
||||
- ``GET /prompt_manager/images/thumbnails/plan?mode=&size=&quality=`` counts
|
||||
what a run would do over every gallery root and writes nothing:
|
||||
``{success, targets, fresh, remaining, roots, mode, size, quality}``.
|
||||
- ``POST /prompt_manager/jobs/thumbnails {mode, size, quality}`` starts the
|
||||
``thumbnails`` job (202 ``{job}``). Its admission class is ``guarded``: it
|
||||
never runs beside a record-changing job, and one thumbnail job runs at a
|
||||
time. It stops at a file boundary when cancelled, keeping the previews it
|
||||
wrote. Its outcome groups are ``generated``, ``skipped`` (already fresh)
|
||||
and ``errors``.
|
||||
|
||||
``mode`` is ``missing`` (previews that are absent or older than their file)
|
||||
or ``rebuild`` (every preview again, fresh ones too). ``size`` is the
|
||||
preview's longest edge, 128 to 2048 pixels. ``quality`` picks the encoder
|
||||
settings (``THUMBNAIL_ENCODINGS``). Every preview is written next to the old
|
||||
one and renamed over it, so nothing is cleared first and a failure keeps the
|
||||
old preview; originals are only read.
|
||||
|
||||
The legacy gallery stream ``POST /images/generate-thumbnails/progress``
|
||||
starts this job (its quality preset still sizes the previews) or attaches to
|
||||
the running one. Its first frame, ``job {id, started}``, names that job and
|
||||
whether this stream started it (the gallery cancels by that id, and only a
|
||||
job it started); then come the job's legacy ``status``/``start``/
|
||||
``progress``/``file_error``/``complete`` frames. A client that goes away no
|
||||
longer stops the work: the job outlives the request, as a scan does.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .admin import _plural, _scan_location, _scan_roots, _scan_status_line
|
||||
from .images import (
|
||||
THUMBNAIL_ENCODINGS,
|
||||
_BodyTooLarge,
|
||||
_new_thumbnail_stats,
|
||||
_read_json_body,
|
||||
_record_thumbnail_result,
|
||||
_thumbnail_batch_size,
|
||||
_thumbnail_complete_payload,
|
||||
_thumbnail_progress_payload,
|
||||
_worker_threads,
|
||||
)
|
||||
from .job_routes import busy_response
|
||||
from .jobs import THUMBNAIL_GROUP_NAMES, THUMBNAILS_JOB_KIND, JobBusy
|
||||
|
||||
THUMBNAIL_JOB_TITLE = "Thumbnail generation"
|
||||
THUMBNAIL_MODES = ("missing", "rebuild")
|
||||
THUMBNAIL_QUALITIES = tuple(THUMBNAIL_ENCODINGS)
|
||||
MIN_THUMBNAIL_EDGE = 128
|
||||
MAX_THUMBNAIL_EDGE = 2048
|
||||
# The concept's "Preview size limit" default.
|
||||
DEFAULT_THUMBNAIL_EDGE = 512
|
||||
DEFAULT_THUMBNAIL_OPTIONS = {
|
||||
"mode": "missing",
|
||||
"size": DEFAULT_THUMBNAIL_EDGE,
|
||||
"quality": "medium",
|
||||
}
|
||||
_OPTIONS_ERROR = (
|
||||
'mode must be "missing" or "rebuild", quality "low", "medium" or "high", '
|
||||
f"and size a whole number of pixels from {MIN_THUMBNAIL_EDGE} "
|
||||
f"to {MAX_THUMBNAIL_EDGE}"
|
||||
)
|
||||
CANCELLED_ERROR = "Thumbnail generation was cancelled"
|
||||
CANCELLED_BEFORE_WRITING = "Cancelled before any preview was written"
|
||||
NOTHING_TO_PREVIEW = "No images or videos to preview"
|
||||
_MODE_LOG = {"missing": "missing previews", "rebuild": "every preview"}
|
||||
# Outcome rows: what happened to each file's preview.
|
||||
_DETAILS = {
|
||||
"generated": "Preview written",
|
||||
"skipped": "Preview already up to date",
|
||||
}
|
||||
_LOG_PREFIXES = {"generated": "+", "errors": "!"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ThumbnailOptions:
|
||||
"""One run's choices: the mode, the preview's longest edge and quality."""
|
||||
|
||||
mode: str
|
||||
edge: int
|
||||
quality: str
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
return (self.edge, self.edge)
|
||||
|
||||
@property
|
||||
def encoding(self):
|
||||
return THUMBNAIL_ENCODINGS[self.quality]
|
||||
|
||||
@property
|
||||
def rebuild(self):
|
||||
return self.mode == "rebuild"
|
||||
|
||||
def scope(self):
|
||||
return {"mode": self.mode, "size": self.edge, "quality": self.quality}
|
||||
|
||||
|
||||
def _edge(value, *, from_query):
|
||||
"""The preview edge as an int, or None when it is not one in range."""
|
||||
if from_query and isinstance(value, str):
|
||||
if not value.isascii() or not value.isdigit() or len(value) > 5:
|
||||
return None
|
||||
value = int(value)
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
return None
|
||||
return value if MIN_THUMBNAIL_EDGE <= value <= MAX_THUMBNAIL_EDGE else None
|
||||
|
||||
|
||||
def thumbnail_options(raw, *, from_query=False):
|
||||
"""``(ThumbnailOptions, None)`` from a body or query, or ``(None, message)``.
|
||||
|
||||
Absent fields take ``DEFAULT_THUMBNAIL_OPTIONS``. Query values are
|
||||
strings, so a query ``size`` may be decimal digits; a JSON ``size`` must
|
||||
be an int.
|
||||
"""
|
||||
if not isinstance(raw, dict) and not hasattr(raw, "get"):
|
||||
return None, _OPTIONS_ERROR
|
||||
mode = raw.get("mode", DEFAULT_THUMBNAIL_OPTIONS["mode"])
|
||||
quality = raw.get("quality", DEFAULT_THUMBNAIL_OPTIONS["quality"])
|
||||
edge = _edge(raw.get("size", DEFAULT_THUMBNAIL_EDGE), from_query=from_query)
|
||||
if mode not in THUMBNAIL_MODES or quality not in THUMBNAIL_QUALITIES:
|
||||
return None, _OPTIONS_ERROR
|
||||
if edge is None:
|
||||
return None, _OPTIONS_ERROR
|
||||
return ThumbnailOptions(mode, edge, quality), None
|
||||
|
||||
|
||||
def _invalid(message):
|
||||
return web.json_response(
|
||||
{"success": False, "code": "invalid-request", "error": message}, status=400
|
||||
)
|
||||
|
||||
|
||||
def _frame(event, data):
|
||||
"""A legacy stream frame as the job publishes it."""
|
||||
return {"event": event, "data": data}
|
||||
|
||||
|
||||
def _finished_line(plan, stats):
|
||||
"""A finished job's status line: what it wrote, or that it had nothing to do."""
|
||||
if not plan["targets"]:
|
||||
if plan["skipped"]:
|
||||
return f"No previews to generate · {plan['skipped']:,} already up to date"
|
||||
return NOTHING_TO_PREVIEW
|
||||
parts = [f"{_plural(stats['generated'], 'preview')} written"]
|
||||
if stats["errors"]:
|
||||
parts.append(_plural(len(stats["errors"]), "error"))
|
||||
if plan["remaining"]:
|
||||
parts.append(f"{plan['remaining']:,} more on the next run")
|
||||
return " · ".join(parts)
|
||||
|
||||
|
||||
class ThumbnailJobsMixin:
|
||||
"""The thumbnail plan, the thumbnail job and the legacy stream on it."""
|
||||
|
||||
async def get_thumbnail_plan(self, request):
|
||||
"""What a thumbnail run would do (read-only; capped like a run)."""
|
||||
options, error = thumbnail_options(request.query, from_query=True)
|
||||
if error:
|
||||
return _invalid(error)
|
||||
try:
|
||||
output_dirs = await self._run_in_executor(self._get_all_output_dirs)
|
||||
plan = await self._run_in_executor(
|
||||
self._plan_gallery_thumbnails, output_dirs, rebuild=options.rebuild
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Thumbnail plan failed")
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": None,
|
||||
"error": "The thumbnail plan could not be read",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"targets": len(plan["targets"]),
|
||||
"fresh": plan["fresh"],
|
||||
"remaining": plan["remaining"],
|
||||
"roots": [self._public_path(root) for root in output_dirs],
|
||||
**options.scope(),
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
async def start_thumbnail_job(self, request):
|
||||
"""Start the thumbnail job (202), or 409 naming the job in the way."""
|
||||
body = {}
|
||||
if request.can_read_body:
|
||||
try:
|
||||
body = await _read_json_body(request)
|
||||
except _BodyTooLarge:
|
||||
return web.json_response(
|
||||
{"success": False, "code": "too-large", "error": "Body too large"},
|
||||
status=413,
|
||||
)
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return _invalid(_OPTIONS_ERROR)
|
||||
options, error = thumbnail_options(body)
|
||||
if error:
|
||||
return _invalid(error)
|
||||
try:
|
||||
job = self._start_thumbnail_job(options)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
def _start_thumbnail_job(self, options):
|
||||
"""Register the thumbnail job synchronously; raises JobBusy."""
|
||||
return self._jobs.start(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
THUMBNAIL_JOB_TITLE,
|
||||
admission="guarded",
|
||||
cancellable=True,
|
||||
scope=options.scope(),
|
||||
runner=functools.partial(self._run_thumbnail_job, options=options),
|
||||
groups=THUMBNAIL_GROUP_NAMES,
|
||||
)
|
||||
|
||||
# ── the job ───────────────────────────────────────────────────────
|
||||
|
||||
async def _run_thumbnail_job(self, job, options):
|
||||
try:
|
||||
state = await self._thumbnail_job_body(job, options)
|
||||
except Exception:
|
||||
self.logger.exception("Thumbnail generation failed")
|
||||
job.update(status_line="Thumbnail generation stopped on an internal error.")
|
||||
job.log_line("Thumbnails failed · internal error")
|
||||
job.publish(
|
||||
_frame(
|
||||
"error",
|
||||
{
|
||||
"error": "An internal error occurred",
|
||||
"message": "Thumbnail generation failed. Check server logs.",
|
||||
},
|
||||
)
|
||||
)
|
||||
state = "failed"
|
||||
job.finish(state)
|
||||
|
||||
async def _thumbnail_job_body(self, job, options):
|
||||
"""Plan, then generate in worker batches; returns the final state.
|
||||
|
||||
Every exit sets the job's final status line (I3): what was written,
|
||||
that there was nothing to do, or what a cancel kept.
|
||||
"""
|
||||
output_dirs = await self._run_in_executor(self._get_all_output_dirs)
|
||||
if not output_dirs:
|
||||
message = "ComfyUI output directory not found"
|
||||
job.update(status_line=message)
|
||||
job.log_line(f"Thumbnails failed · {message}")
|
||||
job.publish(_frame("error", {"error": message}))
|
||||
return "failed"
|
||||
folders = _plural(len(output_dirs), "output folder")
|
||||
job.scope["output_folders"] = len(output_dirs)
|
||||
job.update(status_line=f"Looking for media files in {folders}")
|
||||
job.log_line(
|
||||
f"Thumbnails started · {folders} · {_MODE_LOG[options.mode]} · "
|
||||
f"{options.edge} px · {options.quality}"
|
||||
)
|
||||
job.publish(
|
||||
_frame(
|
||||
"status",
|
||||
{"phase": "scanning", "message": "Scanning for images and videos..."},
|
||||
)
|
||||
)
|
||||
plan = await self._run_in_executor(
|
||||
self._plan_gallery_thumbnails,
|
||||
output_dirs,
|
||||
rebuild=options.rebuild,
|
||||
stop=lambda: job.cancelled,
|
||||
)
|
||||
if job.cancelled: # M2: stopped during the walk, before any start frame
|
||||
return self._end_cancelled(job, 0, CANCELLED_BEFORE_WRITING)
|
||||
roots = _scan_roots(output_dirs)
|
||||
self._record_fresh(job, plan, roots)
|
||||
stats, start = await self._generate_thumbnail_targets(job, options, plan, roots)
|
||||
self.invalidate_gallery_cache()
|
||||
generated = _plural(stats["generated"], "preview")
|
||||
if job.cancelled:
|
||||
return self._end_cancelled(
|
||||
job, stats["generated"], f"Cancelled · {generated} written"
|
||||
)
|
||||
payload = _thumbnail_complete_payload(
|
||||
len(plan["targets"]) + plan["skipped"] + plan["remaining"],
|
||||
stats,
|
||||
start,
|
||||
remaining=plan["remaining"],
|
||||
processed=len(plan["targets"]),
|
||||
)
|
||||
job.update(progress=100, status_line=_finished_line(plan, stats))
|
||||
if plan["remaining"]:
|
||||
job.log_line(
|
||||
f"{plan['remaining']:,} more previews are left for the next run"
|
||||
)
|
||||
errors = _plural(len(stats["errors"]), "error")
|
||||
job.log_line(f"Thumbnails finished · {generated} written · {errors}")
|
||||
self.logger.info(payload["message"])
|
||||
job.publish(_frame("complete", payload))
|
||||
return "finished"
|
||||
|
||||
@staticmethod
|
||||
def _end_cancelled(job, generated, status_line):
|
||||
"""Close a cancelled job: its status line, log line and error frame."""
|
||||
job.update(status_line=status_line)
|
||||
job.log_line(f"Thumbnails cancelled · {_plural(generated, 'preview')} written")
|
||||
job.publish(
|
||||
_frame("error", {"error": CANCELLED_ERROR, "message": CANCELLED_ERROR})
|
||||
)
|
||||
return "cancelled"
|
||||
|
||||
def _record_fresh(self, job, plan, roots):
|
||||
"""List the already fresh previews under Skipped (no log line each)."""
|
||||
rows = []
|
||||
for src in plan["fresh_files"]:
|
||||
path, label, index = _scan_location(src, roots)
|
||||
rows.append(
|
||||
{
|
||||
"path": path,
|
||||
"root": label,
|
||||
"root_index": index,
|
||||
"detail": _DETAILS["skipped"],
|
||||
}
|
||||
)
|
||||
job.record_counted("skipped", plan["skipped"], rows)
|
||||
if plan["skipped"]:
|
||||
job.log_line(f"= {plan['skipped']:,} previews already up to date")
|
||||
|
||||
async def _generate_thumbnail_targets(self, job, options, plan, roots):
|
||||
"""Generate ``plan``'s targets in worker batches, publishing each file."""
|
||||
targets = plan["targets"]
|
||||
total = len(targets)
|
||||
video_count = sum(1 for _, _, is_video in targets if is_video)
|
||||
message = (
|
||||
f"Found {total - video_count} images and {video_count} videos to process"
|
||||
)
|
||||
if plan["skipped"]:
|
||||
message += f" ({plan['skipped']} already have thumbnails)"
|
||||
job.update(total=total)
|
||||
if total: # never "Processing file 0 of 0" (I3)
|
||||
job.update(status_line=_scan_status_line(0, total, 0, 0))
|
||||
job.publish(
|
||||
_frame(
|
||||
"start",
|
||||
{
|
||||
"total_images": total,
|
||||
"skipped": plan["skipped"],
|
||||
"remaining": plan["remaining"],
|
||||
"phase": "processing",
|
||||
"image_count": total - video_count,
|
||||
"video_count": video_count,
|
||||
"message": message,
|
||||
},
|
||||
)
|
||||
)
|
||||
stats = _new_thumbnail_stats()
|
||||
stats["skipped"] = plan["skipped"]
|
||||
start = time.monotonic()
|
||||
batch_size = _thumbnail_batch_size(_worker_threads())
|
||||
generate = {
|
||||
"encoding": options.encoding,
|
||||
"rebuild": options.rebuild,
|
||||
"stop": lambda: job.cancelled,
|
||||
}
|
||||
index = 0
|
||||
batches = 0
|
||||
for batch_start in range(0, total, batch_size):
|
||||
if job.cancelled:
|
||||
break
|
||||
batch = targets[batch_start : batch_start + batch_size]
|
||||
results = await self._run_in_executor(
|
||||
self._generate_batch, batch, options.size, **generate
|
||||
)
|
||||
for (src, _dst, _video), result in zip(batch, results):
|
||||
if result["action"] == "cancelled":
|
||||
continue
|
||||
index += 1
|
||||
_record_thumbnail_result(stats, result)
|
||||
self._record_thumbnail_outcome(job, src, result, roots)
|
||||
job.update(processed=index, progress=index * 100 // total)
|
||||
if result["action"] == "error":
|
||||
job.publish(
|
||||
_frame(
|
||||
"file_error",
|
||||
{"file": result["file"], "error": result["error"]},
|
||||
)
|
||||
)
|
||||
job.publish(
|
||||
_frame(
|
||||
"progress",
|
||||
_thumbnail_progress_payload(index, total, stats, result, start),
|
||||
)
|
||||
)
|
||||
batches += 1
|
||||
job.update(
|
||||
status_line=_scan_status_line(
|
||||
index, total, batches, time.monotonic() - start
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0) # let other requests run between batches
|
||||
return stats, start
|
||||
|
||||
@staticmethod
|
||||
def _record_thumbnail_outcome(job, src, result, roots):
|
||||
path, label, index = _scan_location(src, roots)
|
||||
where = f"{label}/{path}" if label else path
|
||||
if result["action"] == "error":
|
||||
group, detail = "errors", result.get("error") or "failed"
|
||||
elif result["action"] == "skipped":
|
||||
group, detail = "skipped", _DETAILS["skipped"]
|
||||
else:
|
||||
group, detail = "generated", _DETAILS["generated"]
|
||||
row = {"path": path, "root": label, "root_index": index, "detail": detail}
|
||||
if group == "skipped":
|
||||
job.record_counted(group, 1, [row])
|
||||
return
|
||||
job.record(group, row, f"{_LOG_PREFIXES[group]} {where} · {detail}")
|
||||
|
||||
# ── the legacy gallery stream ─────────────────────────────────────
|
||||
|
||||
async def generate_thumbnails_with_progress(self, request):
|
||||
"""The gallery's thumbnail stream (POST, server-sent events).
|
||||
|
||||
Starts the thumbnail job with the gallery's quality preset as the
|
||||
preview size (missing previews, medium encoding, as before), or
|
||||
attaches to the running thumbnail job, and streams its frames. A
|
||||
legacy hold (the blocking generate or clear) or a record-changing
|
||||
job refuses it with the stream's error frame. The job goes on when
|
||||
the client goes away.
|
||||
"""
|
||||
thumbnail_size = self._thumbnail_size(request.query.get("quality", "medium"))
|
||||
response = web.StreamResponse(
|
||||
status=200,
|
||||
headers={
|
||||
"Content-Type": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
},
|
||||
)
|
||||
await response.prepare(request)
|
||||
# M3: read the registry after the last await, so a job that started
|
||||
# while the headers went out is attached to, not refused.
|
||||
job, started, busy = self._legacy_stream_job(thumbnail_size)
|
||||
if busy is not None:
|
||||
message = f"{busy.title} already running"
|
||||
await self._emit_sse(
|
||||
response, "error", {"error": message, "message": message}
|
||||
)
|
||||
return response
|
||||
# No await between the start and the subscription: the lead frame is
|
||||
# sent after it, so the stream misses none of the runner's frames.
|
||||
events = job.events(lead=_frame("job", {"id": job.id, "started": started}))
|
||||
try:
|
||||
async for payload in events:
|
||||
frame = (
|
||||
f"event: {payload['event']}\ndata: {json.dumps(payload['data'])}"
|
||||
)
|
||||
await response.write(f"{frame}\n\n".encode("utf-8"))
|
||||
except Exception as exc: # a write failed: the client went away
|
||||
self.logger.info(
|
||||
f"Thumbnail progress client disconnected ({exc}); "
|
||||
"the thumbnails continue in the background"
|
||||
)
|
||||
finally:
|
||||
# Also when aiohttp cancels the handler on disconnect: the job is
|
||||
# its own task and goes on.
|
||||
await events.aclose()
|
||||
return response
|
||||
|
||||
def _legacy_stream_job(self, thumbnail_size):
|
||||
"""``(job, started, busy)`` for the legacy stream, synchronously.
|
||||
|
||||
Attaches to the running cancellable thumbnail job, or starts one with
|
||||
the gallery's preset as the size (missing previews, medium encoding,
|
||||
as the gallery always wrote), or names the job in the way.
|
||||
"""
|
||||
running = self._jobs.active(THUMBNAILS_JOB_KIND)
|
||||
if running is not None and running.cancellable:
|
||||
return running, False, None
|
||||
busy = self._jobs.blocker(THUMBNAILS_JOB_KIND, admission="guarded")
|
||||
if busy is not None:
|
||||
return None, False, busy
|
||||
# blocker() said free and nothing awaited since, so the start is admitted.
|
||||
options = ThumbnailOptions("missing", thumbnail_size[0], "medium")
|
||||
return self._start_thumbnail_job(options), True, None
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Store each workflow once: the migration and Reclaim space jobs (#249 S6).
|
||||
|
||||
Reviewed like every maintenance operation (``POST /prompt_manager/
|
||||
maintenance/preview {operation:"workflow_storage"}``, a ``Plan`` with the
|
||||
records to rewrite, the backup and the disk check) and applied with its token
|
||||
(``POST /prompt_manager/maintenance/apply``). The apply starts:
|
||||
|
||||
1. **Store each workflow once**: kind ``maintenance``, admission ``records``,
|
||||
cancellable, groups ``changed``/``protected``/``errors``. It always takes
|
||||
a verified pre-migration backup, whatever the safety backup policy, named
|
||||
``<db>.pre-workflow-storage_<stamp>`` outside the pruned ``<db>.backup_*``
|
||||
pool and kept until the user removes it (``backup-failed`` ends the job
|
||||
with nothing rewritten). It then rewrites old-form rows batch by batch
|
||||
(``database/workflow_storage.py``): one transaction per batch of at most
|
||||
``BATCH_SECONDS``, then ``BATCH_PAUSE`` with the write lock free, progress
|
||||
after each, Cancel honoured between batches. A row that cannot be
|
||||
rewritten stays inline and is listed in ``errors``; any other failure
|
||||
rolls its batch back whole and fails the job. Every record still reads as
|
||||
before, and the next apply continues from the first old-form row.
|
||||
2. **Reclaim space**, once every record was looked at: kind ``maintenance``,
|
||||
admission ``exclusive`` (it runs alone; VACUUM holds the write lock), not
|
||||
cancellable. It checks the disk for VACUUM's copies of the live pages
|
||||
(``disk_plan``), then sweeps, runs VACUUM and truncates the WAL. When
|
||||
another job holds the registry, it does not start and the finished
|
||||
migration says so and names Optimize database.
|
||||
|
||||
Only the database changes: no image or video file is opened.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
|
||||
from .jobs import MAINTENANCE_JOB_KIND, JobBusy
|
||||
from .maintenance import (
|
||||
MAINTENANCE_GROUPS,
|
||||
PRE_MIGRATION_LABEL,
|
||||
SafetyBackupFailed,
|
||||
backup_failed_message,
|
||||
_fail_job,
|
||||
_megabytes,
|
||||
disk_plan,
|
||||
short_space_message,
|
||||
take_safety_backup,
|
||||
)
|
||||
|
||||
try:
|
||||
from ...database import workflow_storage
|
||||
from ...database.health import readonly_uri
|
||||
except ImportError:
|
||||
from database import workflow_storage
|
||||
from database.health import readonly_uri
|
||||
|
||||
OPERATION = workflow_storage.OPERATION
|
||||
TITLE = "Store each workflow once"
|
||||
RECLAIM_TITLE = "Reclaim space"
|
||||
REWRITE_FAILED = (
|
||||
"Storing workflows once stopped on an error. The batch in progress was "
|
||||
"rolled back; every record reads as before. Run it again to continue."
|
||||
)
|
||||
RECLAIM_FAILED = "Reclaim space failed; the database is unchanged."
|
||||
ROW_KEPT = "could not be rewritten; it keeps its inline text and reads as before"
|
||||
|
||||
|
||||
NOTHING_LEFT = "Every image record already stores its workflow once"
|
||||
|
||||
|
||||
def _plural(count, one, many=None):
|
||||
return f"{count:,} {one if count == 1 else (many or one + 's')}"
|
||||
|
||||
|
||||
def live_bytes(db_path):
|
||||
"""The pages VACUUM copies: page_count minus the free list (blocking)."""
|
||||
conn = sqlite3.connect(readonly_uri(db_path), uri=True)
|
||||
try:
|
||||
page_size = conn.execute("PRAGMA page_size").fetchone()[0]
|
||||
pages = conn.execute("PRAGMA page_count").fetchone()[0]
|
||||
free = conn.execute("PRAGMA freelist_count").fetchone()[0]
|
||||
return page_size * (pages - free)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class WorkflowStorageRoutesMixin:
|
||||
"""Mixin for the maintenance routes: the workflow storage jobs."""
|
||||
|
||||
def _start_workflow_storage(self, db, backup):
|
||||
"""Start the migration job (raises JobBusy); the token is the caller's."""
|
||||
return self._jobs.start(
|
||||
MAINTENANCE_JOB_KIND,
|
||||
TITLE,
|
||||
admission="records",
|
||||
cancellable=True,
|
||||
scope={"operation": OPERATION},
|
||||
groups=MAINTENANCE_GROUPS,
|
||||
runner=lambda job: self._run_workflow_storage(job, db, backup),
|
||||
)
|
||||
|
||||
async def _run_workflow_storage(self, job, db, _backup_on):
|
||||
# Always backed up first, whatever the policy: the way back. The first
|
||||
# apply keeps a pre-migration backup; later ones (after a Cancel, a
|
||||
# rerun) take a pooled safety backup, so only one copy is never pruned.
|
||||
counts = await self._run_in_executor(db.workflow_storage_status)
|
||||
if not counts["rows"]:
|
||||
job.log_line(NOTHING_LEFT)
|
||||
job.update(progress=100, status_line=NOTHING_LEFT)
|
||||
return
|
||||
kept = await self._run_in_executor(
|
||||
workflow_storage.pre_migration_backups, db.model.db_path
|
||||
)
|
||||
label = "backup" if kept else PRE_MIGRATION_LABEL
|
||||
backup = await self._backup_before_rewrite(job, db, label, kept)
|
||||
if backup is None:
|
||||
return
|
||||
try:
|
||||
done = await self._rewrite_workflows(job, db, backup)
|
||||
finally:
|
||||
if kept:
|
||||
await self._prune_backup_pool(job, db)
|
||||
if done:
|
||||
job.finish("finished")
|
||||
self._start_reclaim_space(job, db)
|
||||
|
||||
async def _backup_before_rewrite(self, job, db, label, kept):
|
||||
"""The verified backup's status note, or None (the job failed).
|
||||
|
||||
``label`` ``pre-workflow-storage``: kept until the user removes it.
|
||||
``backup``: a pooled safety backup, while the earliest kept
|
||||
pre-migration backup (``kept[0]``) stays. A failure ends the job
|
||||
``backup-failed``, nothing changed.
|
||||
"""
|
||||
noun = "safety backup" if kept else "pre-migration backup"
|
||||
job.update(status_line=f"Taking the {noun}")
|
||||
job.log_line(f"Taking the {noun} of the database")
|
||||
try:
|
||||
name = await self._run_in_executor(take_safety_backup, db.model, label)
|
||||
except SafetyBackupFailed as error:
|
||||
self.logger.error("Backup before the rewrite failed", exc_info=True)
|
||||
_fail_job(job, "backup-failed", backup_failed_message(error))
|
||||
return None
|
||||
if kept:
|
||||
job.log_line(
|
||||
f"Safety backup {name} verified; the pre-migration backup "
|
||||
f"{kept[0]} is kept"
|
||||
)
|
||||
return f"Safety backup {name}"
|
||||
job.log_line(
|
||||
f"Pre-migration backup {name} verified; it is kept until you remove it"
|
||||
)
|
||||
return f"Pre-migration backup {name}"
|
||||
|
||||
async def _rewrite_workflows(self, job, db, backup):
|
||||
"""Rewrite every old-form row; True when every row was looked at."""
|
||||
counts = await self._run_in_executor(db.workflow_storage_status)
|
||||
job.update(total=counts["rows"], processed=0)
|
||||
job.update(status_line="Rewriting image records")
|
||||
job.log_line(f"{_plural(counts['rows'], 'image record')} to rewrite")
|
||||
rows = protected = errors = 0
|
||||
last_id = 0
|
||||
while True:
|
||||
if job.cancelled:
|
||||
return self._stopped(job, rows, backup)
|
||||
try:
|
||||
batch = await self._run_in_executor(
|
||||
db.migrate_workflow_batch,
|
||||
last_id,
|
||||
workflow_storage.BATCH_ROWS,
|
||||
workflow_storage.BATCH_BYTES,
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Storing workflows once failed")
|
||||
_fail_job(job, None, REWRITE_FAILED)
|
||||
return False
|
||||
if batch["done"]:
|
||||
break
|
||||
last_id = batch["last_id"]
|
||||
rows += batch["rows"]
|
||||
protected += batch["protected"]
|
||||
errors += len(batch["errors"])
|
||||
job.record_counted("changed", batch["rows"])
|
||||
job.record_counted("protected", batch["protected"])
|
||||
for image_id in batch["errors"]:
|
||||
job.record(
|
||||
"errors",
|
||||
{"path": "", "detail": f"Image record {image_id}: {ROW_KEPT}"},
|
||||
f"! Image record {image_id}: {ROW_KEPT}",
|
||||
)
|
||||
total = max(job.total, rows)
|
||||
job.update(
|
||||
processed=rows,
|
||||
total=total,
|
||||
progress=100 * rows // total if total else 100,
|
||||
status_line=f"Rewrote {_plural(rows, 'image record')}",
|
||||
)
|
||||
job.publish({"type": "progress", "processed": rows, "total": total})
|
||||
# Leave the write lock free between batches (monitor, run counts).
|
||||
await asyncio.sleep(workflow_storage.BATCH_PAUSE)
|
||||
removed = await self._run_in_executor(db.sweep_workflow_documents)
|
||||
line = f"Rewrote {_plural(rows, 'image record')}"
|
||||
if protected:
|
||||
line += f"; {_plural(protected, 'stored value')} not text, kept as is"
|
||||
if errors:
|
||||
kept = "keeps its" if errors == 1 else "keep their"
|
||||
line += (
|
||||
f"; {_plural(errors, 'image record')} could not be rewritten and "
|
||||
f"{kept} inline text"
|
||||
)
|
||||
job.log_line(line)
|
||||
if removed:
|
||||
job.log_line(f"Removed {_plural(removed, 'unused workflow document')}")
|
||||
job.update(progress=100, status_line=f"{line} · {backup}")
|
||||
return True
|
||||
|
||||
def _stopped(self, job, rows, backup):
|
||||
line = (
|
||||
f"Stopped after {_plural(rows, 'image record')}; "
|
||||
"run it again to continue"
|
||||
)
|
||||
job.log_line(line)
|
||||
job.update(status_line=f"{line} · {backup}")
|
||||
job.finish("cancelled")
|
||||
return False
|
||||
|
||||
def _start_reclaim_space(self, finished, db):
|
||||
"""Start Reclaim space after the migration ``finished``, or say why not."""
|
||||
try:
|
||||
self._jobs.start(
|
||||
MAINTENANCE_JOB_KIND,
|
||||
RECLAIM_TITLE,
|
||||
admission="exclusive",
|
||||
cancellable=False,
|
||||
scope={"operation": OPERATION},
|
||||
groups=MAINTENANCE_GROUPS,
|
||||
runner=lambda job: self._run_reclaim_space(job, db),
|
||||
)
|
||||
except JobBusy as busy:
|
||||
line = (
|
||||
f"Reclaim space did not start: {busy.running.title} is running. "
|
||||
"Run Optimize database to give the space back."
|
||||
)
|
||||
finished.log_line(line)
|
||||
finished.update(status_line=f"{finished.status_line} · {line}")
|
||||
return
|
||||
finished.log_line("Reclaim space started")
|
||||
|
||||
async def _run_reclaim_space(self, job, db):
|
||||
job.update(status_line="Checking free disk space")
|
||||
try:
|
||||
live = await self._run_in_executor(live_bytes, db.model.db_path)
|
||||
_disk, short, _size = await self._run_in_executor(
|
||||
disk_plan,
|
||||
db.model.db_path,
|
||||
backup=False,
|
||||
vacuum=True,
|
||||
vacuum_bytes=live,
|
||||
)
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Could not measure the disk space for Reclaim space")
|
||||
return _fail_job(job, None, RECLAIM_FAILED)
|
||||
if short is not None:
|
||||
return _fail_job(job, "insufficient-space", short_space_message(short))
|
||||
job.update(status_line="Reclaiming space (the database is locked)")
|
||||
job.log_line("Rebuilding the database file (VACUUM)")
|
||||
try:
|
||||
result = await self._run_in_executor(db.reclaim_space)
|
||||
except Exception:
|
||||
self.logger.exception("Reclaim space failed")
|
||||
return _fail_job(job, None, RECLAIM_FAILED)
|
||||
saved = max(0, result["before_bytes"] - result["after_bytes"])
|
||||
line = (
|
||||
f"Reclaimed {_megabytes(saved)} · the database is now "
|
||||
f"{_megabytes(result['after_bytes'])}"
|
||||
)
|
||||
if result["checkpoint_busy"]:
|
||||
line += " (the rest after the next restart)"
|
||||
job.record_counted("changed", 1)
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=line)
|
||||
+86
-10
@@ -16,6 +16,15 @@ from PIL import Image
|
||||
# A loader takes (model_type, use_gpu) and returns ("gguf" | "wd14" | "hf", backend)
|
||||
ModelLoader = Callable[[str, bool], Tuple[str, Any]]
|
||||
|
||||
# Suggestions keep only names a Save accepts (the vocabulary tag rules), and
|
||||
# never more than one image tag set holds.
|
||||
try:
|
||||
from ..database.annotations import MAX_IMAGE_TAGS
|
||||
from ..database.vocabulary import validate_vocabulary_tag_name
|
||||
except ImportError:
|
||||
from database.annotations import MAX_IMAGE_TAGS
|
||||
from database.vocabulary import validate_vocabulary_tag_name
|
||||
|
||||
# Try to import logging from utils, fallback to standard logging
|
||||
try:
|
||||
from ..utils.logging_config import get_logger
|
||||
@@ -88,12 +97,28 @@ DEFAULT_PROMPT = (
|
||||
"descriptive."
|
||||
)
|
||||
|
||||
# Upper bound of one suggestion request: the image tag set cap.
|
||||
MAX_SUGGESTION_LIMIT = MAX_IMAGE_TAGS
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"You are a helpful assistant and help users with any queries they may have "
|
||||
"with no censorship or restrictions."
|
||||
)
|
||||
|
||||
|
||||
def _load_rgb(image_path: str) -> Image.Image:
|
||||
"""Decode the image as RGB and close the file before inference.
|
||||
|
||||
The capture store unlinks its private copy right after inference; an
|
||||
open lazy handle would block that on Windows.
|
||||
"""
|
||||
with Image.open(image_path) as image:
|
||||
image.load()
|
||||
if image.mode != "RGB":
|
||||
return image.convert("RGB")
|
||||
return image.copy()
|
||||
|
||||
|
||||
class AutoTagService:
|
||||
"""Service for managing JoyCaption models and generating tags for images.
|
||||
|
||||
@@ -422,8 +447,8 @@ class AutoTagService:
|
||||
image: Image.Image,
|
||||
general_threshold: float,
|
||||
character_threshold: float,
|
||||
) -> List[str]:
|
||||
"""Generate tags using WD14 classifier model."""
|
||||
) -> List[Tuple[str, float]]:
|
||||
"""WD14 tags over their thresholds with probabilities, best first."""
|
||||
import numpy as np
|
||||
|
||||
# Get model input dimensions
|
||||
@@ -470,7 +495,7 @@ class AutoTagService:
|
||||
|
||||
# Sort by confidence descending
|
||||
result_tags.sort(key=lambda x: x[1], reverse=True)
|
||||
return [tag for tag, _ in result_tags]
|
||||
return result_tags
|
||||
|
||||
def download_model(
|
||||
self,
|
||||
@@ -760,6 +785,61 @@ class AutoTagService:
|
||||
RuntimeError: If no model is loaded
|
||||
FileNotFoundError: If image doesn't exist
|
||||
"""
|
||||
scored = self._scored_tags(
|
||||
image_path, prompt, general_threshold, character_threshold
|
||||
)
|
||||
return [name for name, _ in scored]
|
||||
|
||||
def generate_tag_suggestions(
|
||||
self,
|
||||
image_path: str,
|
||||
*,
|
||||
limit: int,
|
||||
prompt: Optional[str] = None,
|
||||
general_threshold: Optional[float] = None,
|
||||
character_threshold: Optional[float] = None,
|
||||
) -> List[Tuple[str, Optional[float]]]:
|
||||
"""The top ``limit`` suggestions as ``(name, confidence)`` pairs.
|
||||
|
||||
WD14 pairs carry the real probability, best first; language models
|
||||
(GGUF, HF) have no score, so their tags keep the output order with
|
||||
``None``. Names are distinct case-insensitively (first one wins).
|
||||
A name Save would reject (over 50 characters, a control character
|
||||
such as a newline, a comma) is skipped before the dedupe and the
|
||||
``limit`` count. ``prompt`` is used for this call only;
|
||||
``custom_prompt`` is never changed.
|
||||
"""
|
||||
if type(limit) is not int or not 1 <= limit <= MAX_SUGGESTION_LIMIT:
|
||||
raise ValueError(
|
||||
f"limit must be an integer from 1 to {MAX_SUGGESTION_LIMIT}"
|
||||
)
|
||||
scored = self._scored_tags(
|
||||
image_path, prompt, general_threshold, character_threshold
|
||||
)
|
||||
suggestions: List[Tuple[str, Optional[float]]] = []
|
||||
seen = set()
|
||||
for raw, score in scored:
|
||||
try:
|
||||
name = validate_vocabulary_tag_name(raw)
|
||||
except ValueError:
|
||||
continue
|
||||
key = name.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
suggestions.append((name, score))
|
||||
if len(suggestions) == limit:
|
||||
break
|
||||
return suggestions
|
||||
|
||||
def _scored_tags(
|
||||
self,
|
||||
image_path: str,
|
||||
prompt: Optional[str],
|
||||
general_threshold: Optional[float],
|
||||
character_threshold: Optional[float],
|
||||
) -> List[Tuple[str, Optional[float]]]:
|
||||
"""Run the loaded model: WD14 ``(name, probability)``, LLM ``(name, None)``."""
|
||||
if self._tagger is None:
|
||||
raise RuntimeError("No model loaded. Call load_model() first.")
|
||||
|
||||
@@ -767,18 +847,14 @@ class AutoTagService:
|
||||
raise FileNotFoundError(errno.ENOENT, "Image not found", str(image_path))
|
||||
|
||||
use_prompt = prompt or self._custom_prompt
|
||||
|
||||
# Load image
|
||||
image = Image.open(image_path)
|
||||
if image.mode != "RGB":
|
||||
image = image.convert("RGB")
|
||||
image = _load_rgb(image_path)
|
||||
|
||||
# Generate based on model type
|
||||
model_type, tagger_obj = self._tagger
|
||||
|
||||
if model_type == "gguf":
|
||||
raw_tags = self._generate_gguf(tagger_obj, image, use_prompt)
|
||||
return self._parse_tags(raw_tags)
|
||||
return [(tag, None) for tag in self._parse_tags(raw_tags)]
|
||||
elif model_type == "wd14":
|
||||
session, tags_list = tagger_obj
|
||||
gen_thresh = (
|
||||
@@ -799,7 +875,7 @@ class AutoTagService:
|
||||
raw_tags = self._generate_hf(
|
||||
model, processor, device, compute_dtype, image, use_prompt
|
||||
)
|
||||
return self._parse_tags(raw_tags)
|
||||
return [(tag, None) for tag in self._parse_tags(raw_tags)]
|
||||
|
||||
def _generate_gguf(self, model, image: Image.Image, prompt: str) -> str:
|
||||
"""Generate tags using GGUF model."""
|
||||
|
||||
+207
-32
@@ -31,18 +31,19 @@ routes = server_instance.routes
|
||||
extension_uri = None # Will be set in __init__.py
|
||||
|
||||
import os # noqa: E402
|
||||
from pathlib import Path # noqa: E402
|
||||
from typing import Any, Dict, List, Optional, Tuple # noqa: E402
|
||||
|
||||
# Import logging system
|
||||
try:
|
||||
from ..utils.logging_config import get_logger
|
||||
from ..utils import library_location
|
||||
except ImportError:
|
||||
import sys
|
||||
|
||||
current_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, current_dir)
|
||||
from utils.logging_config import get_logger
|
||||
from utils import library_location
|
||||
|
||||
# Initialize logger for config operations
|
||||
config_logger = get_logger("prompt_manager.config")
|
||||
@@ -51,6 +52,14 @@ config_logger = get_logger("prompt_manager.config")
|
||||
# may be used as gallery roots in addition to ComfyUI's own directories.
|
||||
EXTRA_GALLERY_ROOTS_ENV = "PROMPT_MANAGER_EXTRA_GALLERY_ROOTS"
|
||||
|
||||
# The one answer for every path outside the allowed folders. The filesystem
|
||||
# root and the home directory answer it too, so no caller learns the server's
|
||||
# layout (home path, user name) from a refusal (#249 S4).
|
||||
OUTSIDE_ALLOWED_REASON = (
|
||||
"Gallery root must be inside a ComfyUI directory (output, input, temp, user) "
|
||||
f"or one listed in {EXTRA_GALLERY_ROOTS_ENV}"
|
||||
)
|
||||
|
||||
# Environment variable overriding where config.json is read from and saved to.
|
||||
CONFIG_PATH_ENV = "PROMPT_MANAGER_CONFIG_PATH"
|
||||
|
||||
@@ -89,6 +98,29 @@ def write_private_json(path: str, data: Any) -> None:
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
|
||||
def _lexical_path(path: str) -> str:
|
||||
"""``path`` normalised for comparison without touching the filesystem.
|
||||
|
||||
normpath collapses ``.`` and ``..`` as text and normcase folds case on
|
||||
Windows; neither opens, stats nor resolves anything (no SMB request for a
|
||||
UNC path). A symlink is only seen by :func:`_canonical_path` later.
|
||||
"""
|
||||
return os.path.normcase(os.path.normpath(path))
|
||||
|
||||
|
||||
def _lexically_inside(path: str, parents: List[str]) -> bool:
|
||||
"""Whether the normalised ``path`` is one of ``parents`` or below one.
|
||||
|
||||
Pure text on normalised spellings (:func:`_lexical_path` or
|
||||
:func:`_canonical_path`): no filesystem access.
|
||||
"""
|
||||
sep = os.path.sep
|
||||
return any(
|
||||
path == parent or path.startswith(parent.rstrip(sep) + sep)
|
||||
for parent in parents
|
||||
)
|
||||
|
||||
|
||||
def _is_filesystem_root(path: str) -> bool:
|
||||
"""True for '/' on POSIX and drive roots such as 'C:\\' on Windows."""
|
||||
return os.path.dirname(path) == path
|
||||
@@ -268,7 +300,12 @@ class GalleryConfig:
|
||||
"""
|
||||
monitoring = new_config.get("monitoring", {})
|
||||
if "enabled" in monitoring:
|
||||
cls.MONITORING_ENABLED = monitoring["enabled"]
|
||||
if isinstance(monitoring["enabled"], bool):
|
||||
cls.MONITORING_ENABLED = monitoring["enabled"]
|
||||
else: # a hand-edited "false" would otherwise switch it on
|
||||
config_logger.warning(
|
||||
"Ignoring gallery monitoring enabled: expected true or false"
|
||||
)
|
||||
if "directories" in monitoring:
|
||||
directories = cls._validated_directories(monitoring["directories"])
|
||||
if directories is not None:
|
||||
@@ -337,31 +374,56 @@ class GalleryConfig:
|
||||
Filesystem roots are never allowed as parents, even when listed in
|
||||
the environment, because that would re-open the whole disk.
|
||||
"""
|
||||
parents = []
|
||||
return [canonical for canonical, _spelling in cls._allowed_parent_pairs()]
|
||||
|
||||
@classmethod
|
||||
def _allowed_parent_pairs(cls) -> List[Tuple[str, str]]:
|
||||
"""``(canonical, lexical spelling)`` of each allowed parent.
|
||||
|
||||
Only trusted directories (ComfyUI's own and the environment's) are
|
||||
resolved here; the spelling is how ComfyUI or the environment wrote
|
||||
the directory, so a submitted path can be compared without resolving it.
|
||||
"""
|
||||
pairs = []
|
||||
seen = set()
|
||||
for candidate in _comfyui_directories() + _extra_gallery_roots():
|
||||
try:
|
||||
canonical = _canonical_path(candidate)
|
||||
spelling = _lexical_path(os.path.abspath(candidate))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if _is_filesystem_root(canonical) or not os.path.isdir(canonical):
|
||||
continue
|
||||
if canonical not in parents:
|
||||
parents.append(canonical)
|
||||
return parents
|
||||
if canonical not in seen:
|
||||
seen.add(canonical)
|
||||
pairs.append((canonical, spelling))
|
||||
return pairs
|
||||
|
||||
@classmethod
|
||||
def path_anchors(cls) -> List[str]:
|
||||
"""Canonical directories that relative gallery paths are resolved against.
|
||||
|
||||
The ComfyUI base directory comes first, followed by the parent of each
|
||||
directory listed in PROMPT_MANAGER_EXTRA_GALLERY_ROOTS, so an extra
|
||||
root is addressed by its own name (``gallery/sub``) rather than by an
|
||||
absolute path. The same anchors drive the public (relative) form of
|
||||
paths in API responses.
|
||||
ComfyUI directory outside it (``--output-directory D:\\Renders``) and of
|
||||
each directory listed in PROMPT_MANAGER_EXTRA_GALLERY_ROOTS, so such a
|
||||
directory is addressed by its own name (``Renders/sub``) rather than by
|
||||
an absolute path, and two folders of one name stay apart. The same
|
||||
anchors drive the public (relative) form of paths in API responses.
|
||||
"""
|
||||
anchors = []
|
||||
base = _comfyui_base_directory()
|
||||
candidates = [base] if base else []
|
||||
try:
|
||||
base_key = _canonical_path(base) if base else None
|
||||
except (OSError, ValueError):
|
||||
base_key = None
|
||||
for directory in _comfyui_directories():
|
||||
try:
|
||||
canonical = _canonical_path(directory)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if base_key is None or not _lexically_inside(canonical, [base_key]):
|
||||
candidates.append(os.path.dirname(canonical))
|
||||
candidates.extend(os.path.dirname(root) for root in _extra_gallery_roots())
|
||||
for candidate in candidates:
|
||||
try:
|
||||
@@ -399,33 +461,69 @@ class GalleryConfig:
|
||||
(True, "") when the directory lies inside one of
|
||||
:meth:`allowed_gallery_parents`; otherwise (False, reason).
|
||||
"""
|
||||
code, reason = cls.gallery_root_problem(path)
|
||||
return code is None, reason
|
||||
|
||||
@classmethod
|
||||
def gallery_root_problem(cls, path: Any) -> Tuple[Optional[str], str]:
|
||||
"""``(code, reason)`` why ``path`` may not be a gallery root, else (None, "").
|
||||
|
||||
Codes: ``empty``, ``unresolvable``, ``outside-allowed``, ``not-found``
|
||||
and ``not-directory``. No reason names the path.
|
||||
|
||||
Containment is decided twice. First lexically, with no filesystem
|
||||
access at all: a path (UNC, ``\\\\?\\``, a device, anything) whose
|
||||
normalised spelling lies outside every allowed parent answers
|
||||
``outside-allowed`` before it is resolved, so the server never opens
|
||||
or stats it (no SMB connection, no existence or timing oracle). Only
|
||||
then is it resolved and checked again, which catches a symlink out.
|
||||
The filesystem root and the home directory answer ``outside-allowed``
|
||||
too: one answer for every caller (validate-source, the settings save,
|
||||
config.json), and the detail goes to the log only.
|
||||
"""
|
||||
if not isinstance(path, str) or not path.strip():
|
||||
return False, "Gallery root must be a non-empty path"
|
||||
return "empty", "Gallery root must be a non-empty path"
|
||||
|
||||
pairs = cls._allowed_parent_pairs()
|
||||
spellings = [spelling for pair in pairs for spelling in pair]
|
||||
try:
|
||||
lexical = cls._lexical_candidates(path)
|
||||
except (TypeError, ValueError):
|
||||
return "unresolvable", "Gallery root could not be resolved"
|
||||
if not any(_lexically_inside(candidate, spellings) for candidate in lexical):
|
||||
return "outside-allowed", OUTSIDE_ALLOWED_REASON
|
||||
|
||||
try:
|
||||
canonical = cls.resolve_gallery_root(path)
|
||||
except (OSError, ValueError):
|
||||
return False, "Gallery root could not be resolved"
|
||||
return "unresolvable", "Gallery root could not be resolved"
|
||||
|
||||
if _is_filesystem_root(canonical):
|
||||
return False, "Gallery root cannot be a filesystem root"
|
||||
config_logger.info("Gallery root refused: a filesystem root")
|
||||
return "outside-allowed", OUTSIDE_ALLOWED_REASON
|
||||
if canonical == _canonical_path(os.path.expanduser("~")):
|
||||
return False, "Gallery root cannot be the home directory"
|
||||
config_logger.info("Gallery root refused: the home directory")
|
||||
return "outside-allowed", OUTSIDE_ALLOWED_REASON
|
||||
if not _lexically_inside(canonical, [canonical for canonical, _ in pairs]):
|
||||
return "outside-allowed", OUTSIDE_ALLOWED_REASON
|
||||
if not os.path.exists(canonical):
|
||||
return False, "Gallery root does not exist"
|
||||
return "not-found", "Gallery root does not exist"
|
||||
if not os.path.isdir(canonical):
|
||||
return False, "Gallery root is not a directory"
|
||||
return "not-directory", "Gallery root is not a directory"
|
||||
return None, ""
|
||||
|
||||
candidate = Path(canonical)
|
||||
for parent in cls.allowed_gallery_parents():
|
||||
if candidate.is_relative_to(Path(parent)):
|
||||
return True, ""
|
||||
@classmethod
|
||||
def _lexical_candidates(cls, path: str) -> List[str]:
|
||||
"""Every spelling ``path`` may resolve to, normalised as text (no IO).
|
||||
|
||||
return (
|
||||
False,
|
||||
"Gallery root must be inside a ComfyUI directory "
|
||||
f"(output, input, temp, user) or one listed in {EXTRA_GALLERY_ROOTS_ENV}",
|
||||
)
|
||||
An absolute path is itself; a relative one is joined to each
|
||||
:meth:`path_anchors` entry, as :meth:`resolve_gallery_root` tries them.
|
||||
"""
|
||||
path = path.strip()
|
||||
if os.path.isabs(path):
|
||||
return [_lexical_path(path)]
|
||||
anchors = cls.path_anchors()
|
||||
return [_lexical_path(os.path.join(anchor, path)) for anchor in anchors]
|
||||
|
||||
|
||||
class IntegrationConfig:
|
||||
@@ -465,6 +563,59 @@ class IntegrationConfig:
|
||||
cls.CIVITAI_API_KEY = lora["civitai_api_key"]
|
||||
|
||||
|
||||
# Number of tag suggestions the Tag dialogs request by default (D68). The cap
|
||||
# is the image tag set limit: more suggestions could never all be kept.
|
||||
DEFAULT_SUGGESTION_LIMIT = 10
|
||||
MAX_SUGGESTION_LIMIT = 100 # pinned to MAX_IMAGE_TAGS by tests/test_config.py
|
||||
|
||||
|
||||
class AutotagConfig:
|
||||
"""Persisted tagging defaults under ``autotag.defaults`` in config.json.
|
||||
|
||||
``suggestion_limit`` is how many suggestions Generate returns (the top N
|
||||
by score), an integer in ``1..MAX_SUGGESTION_LIMIT``. A hand-edited value
|
||||
outside that range, or of another type, falls back to the default with a
|
||||
warning rather than failing the load.
|
||||
"""
|
||||
|
||||
SUGGESTION_LIMIT = DEFAULT_SUGGESTION_LIMIT
|
||||
|
||||
@classmethod
|
||||
def suggestion_limit(cls) -> int:
|
||||
return cls.SUGGESTION_LIMIT
|
||||
|
||||
@staticmethod
|
||||
def valid_suggestion_limit(value: Any) -> bool:
|
||||
return (
|
||||
type(value) is int and 1 <= value <= MAX_SUGGESTION_LIMIT
|
||||
) # bool, float and str are rejected
|
||||
|
||||
@classmethod
|
||||
def get_config(cls) -> Dict[str, Any]:
|
||||
return {"defaults": {"suggestion_limit": cls.SUGGESTION_LIMIT}}
|
||||
|
||||
@classmethod
|
||||
def update_config(cls, new_config: Any) -> None:
|
||||
if not isinstance(new_config, dict):
|
||||
config_logger.warning("Ignoring autotag settings that are not an object")
|
||||
return
|
||||
defaults = new_config.get("defaults", {})
|
||||
if not isinstance(defaults, dict):
|
||||
config_logger.warning("Ignoring autotag.defaults that is not an object")
|
||||
return
|
||||
if "suggestion_limit" not in defaults:
|
||||
return
|
||||
value = defaults["suggestion_limit"]
|
||||
if cls.valid_suggestion_limit(value):
|
||||
cls.SUGGESTION_LIMIT = value
|
||||
return
|
||||
config_logger.warning(
|
||||
"autotag.defaults.suggestion_limit must be an integer from 1 to "
|
||||
f"{MAX_SUGGESTION_LIMIT}; using {DEFAULT_SUGGESTION_LIMIT}"
|
||||
)
|
||||
cls.SUGGESTION_LIMIT = DEFAULT_SUGGESTION_LIMIT
|
||||
|
||||
|
||||
class PromptManagerConfig:
|
||||
"""Main configuration class for PromptManager core functionality.
|
||||
|
||||
@@ -510,6 +661,14 @@ class PromptManagerConfig:
|
||||
DEFAULT_WORKER_THREADS = max(1, (os.cpu_count() or 1) // 2)
|
||||
WORKER_THREADS = DEFAULT_WORKER_THREADS
|
||||
|
||||
# Maintenance: a reviewed apply takes a verified safety backup of the
|
||||
# database first (System › Backup & export › Recovery policy, #249 S6).
|
||||
SAFETY_BACKUP = True
|
||||
|
||||
# "Keep it here" (#249 S6b): the user chose to keep a node-folder library
|
||||
# where it is, so the workspace stops offering to move it.
|
||||
KEEP_LIBRARY_IN_NODE_FOLDER = False
|
||||
|
||||
@classmethod
|
||||
def max_worker_threads(cls) -> int:
|
||||
"""Largest accepted worker-thread count: the detected core count."""
|
||||
@@ -564,22 +723,23 @@ class PromptManagerConfig:
|
||||
"auto_backup_interval": cls.AUTO_BACKUP_INTERVAL,
|
||||
"worker_threads": cls.WORKER_THREADS,
|
||||
},
|
||||
"maintenance": {"safety_backup": cls.SAFETY_BACKUP},
|
||||
"library": {"keep_in_node_folder": cls.KEEP_LIBRARY_IN_NODE_FOLDER},
|
||||
"gallery": GalleryConfig.get_config(),
|
||||
"integrations": IntegrationConfig.get_config(),
|
||||
"autotag": AutotagConfig.get_config(),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_config_path(cls) -> str:
|
||||
"""Path of the persisted config.json.
|
||||
|
||||
Honours ``PROMPT_MANAGER_CONFIG_PATH`` when set; otherwise the file
|
||||
lives at the repository root next to ``pyproject.toml``.
|
||||
Honours ``PROMPT_MANAGER_CONFIG_PATH`` when set; otherwise it lives in
|
||||
the library's data folder (``utils.library_location``): the node
|
||||
folder for a library that was not moved, else ComfyUI's
|
||||
``user/__promptmanager``.
|
||||
"""
|
||||
override = os.environ.get(CONFIG_PATH_ENV, "").strip()
|
||||
if override:
|
||||
return override
|
||||
repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
return os.path.join(repo_root, "config.json")
|
||||
return library_location.config_location()[0]
|
||||
|
||||
@classmethod
|
||||
def load_from_file(cls, config_path: Optional[str] = None):
|
||||
@@ -698,6 +858,18 @@ class PromptManagerConfig:
|
||||
performance["worker_threads"], cls.WORKER_THREADS
|
||||
)
|
||||
|
||||
maintenance = new_config.get("maintenance", {})
|
||||
if isinstance(maintenance, dict) and isinstance(
|
||||
maintenance.get("safety_backup"), bool
|
||||
):
|
||||
cls.SAFETY_BACKUP = maintenance["safety_backup"]
|
||||
|
||||
library = new_config.get("library", {})
|
||||
if isinstance(library, dict) and isinstance(
|
||||
library.get("keep_in_node_folder"), bool
|
||||
):
|
||||
cls.KEEP_LIBRARY_IN_NODE_FOLDER = library["keep_in_node_folder"]
|
||||
|
||||
# Update gallery config
|
||||
if "gallery" in new_config:
|
||||
GalleryConfig.update_config(new_config["gallery"])
|
||||
@@ -706,6 +878,9 @@ class PromptManagerConfig:
|
||||
if "integrations" in new_config:
|
||||
IntegrationConfig.update_config(new_config["integrations"])
|
||||
|
||||
if "autotag" in new_config:
|
||||
AutotagConfig.update_config(new_config["autotag"])
|
||||
|
||||
|
||||
# Load configuration on import
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
"""The one serialized writer of config.json (#220).
|
||||
|
||||
Every save that changes config.json goes through ``ConfigStore.commit``: the
|
||||
settings page, the LoRA integration and, later, every System section. A
|
||||
commit runs under one process-wide lock, re-reads the file inside it, applies
|
||||
the caller's change to a copy and replaces the file atomically. So two saves
|
||||
never overwrite each other's keys, sections this process does not know are
|
||||
kept byte-for-byte in meaning, and a write that fails leaves the old file in
|
||||
place and raises instead of being logged and reported as saved.
|
||||
|
||||
A revision names the file content a client last saw: ``"sha256:<hex>"`` of
|
||||
the file bytes, or ``"absent"`` when there is no file. A commit that names a
|
||||
revision refuses to write over different content (``RevisionConflict``).
|
||||
|
||||
Reads take the same lock, so a read handle is never open while the file is
|
||||
replaced (Windows refuses that), and a caller's ``publish`` runs inside the
|
||||
lock right after its write, so runtimes publish saves in the order they were
|
||||
written. A symlinked config.json is written beside its real target.
|
||||
|
||||
config.json can hold the CivitAI key, so the file is written owner-only.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
ABSENT_REVISION = "absent"
|
||||
|
||||
# One lock for the process: every ConfigStore writes the same file. It is
|
||||
# re-entrant so a commit's ``publish`` callback may read the config again.
|
||||
_COMMIT_LOCK = threading.RLock()
|
||||
|
||||
# Windows: an antivirus, indexer or backup handle on config.json makes
|
||||
# os.replace fail with PermissionError for a moment, so it is retried briefly.
|
||||
_REPLACE_ATTEMPTS = 5 if os.name == "nt" else 1
|
||||
_REPLACE_BACKOFF_SECONDS = 0.05
|
||||
|
||||
|
||||
class ConfigUnreadable(Exception):
|
||||
"""config.json exists but is not a readable JSON object; it is left alone."""
|
||||
|
||||
|
||||
class RevisionConflict(Exception):
|
||||
"""config.json changed since the revision the caller saw."""
|
||||
|
||||
def __init__(self, current: str):
|
||||
super().__init__("config.json changed since it was read")
|
||||
self.current = current
|
||||
|
||||
|
||||
def revision_of(data: Optional[bytes]) -> str:
|
||||
"""The revision of a file's bytes; ``None`` means the file is absent."""
|
||||
if data is None:
|
||||
return ABSENT_REVISION
|
||||
return "sha256:" + hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def _default_path() -> str:
|
||||
from .config import PromptManagerConfig
|
||||
|
||||
return PromptManagerConfig.get_config_path()
|
||||
|
||||
|
||||
def _replace(source: str, target: str) -> None:
|
||||
"""``os.replace`` with a short retry on a transient Windows PermissionError."""
|
||||
for attempt in range(1, _REPLACE_ATTEMPTS + 1):
|
||||
try:
|
||||
os.replace(source, target)
|
||||
return
|
||||
except PermissionError:
|
||||
if attempt == _REPLACE_ATTEMPTS:
|
||||
raise
|
||||
time.sleep(_REPLACE_BACKOFF_SECONDS)
|
||||
|
||||
|
||||
def _write_atomically(path: str, payload: bytes) -> None:
|
||||
"""Replace ``path`` with ``payload``: temp file beside it, fsync, replace.
|
||||
|
||||
The descriptor is closed before ``os.replace`` (Windows refuses to replace
|
||||
over an open handle), and on any failure the temp file is closed and
|
||||
removed, so a failed write leaves the old file and no stray temp file.
|
||||
``path`` is resolved first: a symlink keeps pointing at the updated
|
||||
target. A single-file bind mount cannot be replaced (EBUSY) and fails as
|
||||
an OSError, which callers report as not saved.
|
||||
"""
|
||||
path = os.path.realpath(path)
|
||||
parent = os.path.dirname(path) or "."
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
fd, temp_path = tempfile.mkstemp(dir=parent, prefix=".config-", suffix=".tmp")
|
||||
try:
|
||||
try:
|
||||
handle = os.fdopen(fd, "wb")
|
||||
except BaseException:
|
||||
os.close(fd)
|
||||
raise
|
||||
with handle: # closed before the replace, also when a write fails
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
if os.name == "posix":
|
||||
os.chmod(temp_path, 0o600)
|
||||
_replace(temp_path, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(temp_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
class ConfigStore:
|
||||
"""Serialized, atomic reads and commits of config.json."""
|
||||
|
||||
def __init__(self, path_getter: Optional[Callable[[], str]] = None):
|
||||
self._path_getter = path_getter or _default_path
|
||||
|
||||
@property
|
||||
def path(self) -> str:
|
||||
"""The config file this store reads and writes (resolved per call)."""
|
||||
return self._path_getter()
|
||||
|
||||
def read(self) -> Tuple[dict, str]:
|
||||
"""``(config, revision)``; a missing file is ``({}, "absent")``.
|
||||
|
||||
Raises ConfigUnreadable when the file cannot be read, is not JSON, or
|
||||
is not a JSON object. Waits for a running commit (see the module
|
||||
docstring).
|
||||
"""
|
||||
with _COMMIT_LOCK:
|
||||
return self._read(self.path)
|
||||
|
||||
@staticmethod
|
||||
def _read(path: str) -> Tuple[dict, str]:
|
||||
try:
|
||||
with open(path, "rb") as handle:
|
||||
raw = handle.read()
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
# No file there (a parent that is a file cannot hold one either);
|
||||
# a commit then fails on the write, as config-write-failed.
|
||||
return {}, ABSENT_REVISION
|
||||
except OSError as exc:
|
||||
raise ConfigUnreadable(f"config.json cannot be read: {exc}") from exc
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError) as exc:
|
||||
raise ConfigUnreadable("config.json is not valid JSON") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise ConfigUnreadable("config.json is not a JSON object")
|
||||
return data, revision_of(raw)
|
||||
|
||||
def commit(
|
||||
self,
|
||||
mutate: Callable[[dict], dict],
|
||||
*,
|
||||
expected_revision: Optional[str],
|
||||
publish: Optional[Callable[[dict], None]] = None,
|
||||
) -> Tuple[dict, str]:
|
||||
"""Apply ``mutate`` to the current config and write the result.
|
||||
|
||||
``mutate`` receives a deep copy of the file's content and returns the
|
||||
whole new config (an object). ``expected_revision`` None merges into
|
||||
whatever is on disk (legacy partial updates); a revision refuses to
|
||||
write over different content. Returns ``(config, revision)`` as
|
||||
written. Raises ConfigUnreadable, RevisionConflict, TypeError (a
|
||||
non-object result) or OSError (the write failed; the old file stays).
|
||||
|
||||
``publish(config)``, when given, runs under the lock once the file
|
||||
holds the new config, so concurrent saves reach the runtime in the
|
||||
order they were written. It must handle its own errors: the values
|
||||
are already durable when it runs.
|
||||
"""
|
||||
with _COMMIT_LOCK:
|
||||
path = self.path
|
||||
current, revision = self._read(path)
|
||||
if expected_revision is not None and expected_revision != revision:
|
||||
raise RevisionConflict(revision)
|
||||
candidate = mutate(copy.deepcopy(current))
|
||||
if not isinstance(candidate, dict):
|
||||
raise TypeError("A config commit must produce a JSON object")
|
||||
payload = json.dumps(candidate, indent=2).encode("utf-8")
|
||||
_write_atomically(path, payload)
|
||||
if publish is not None:
|
||||
publish(candidate)
|
||||
return candidate, revision_of(payload)
|
||||
|
||||
|
||||
_store = ConfigStore()
|
||||
|
||||
|
||||
def get_config_store() -> ConfigStore:
|
||||
"""The process's config store (on ``PromptManagerConfig.get_config_path``)."""
|
||||
return _store
|
||||
+14
-3
@@ -567,10 +567,21 @@ def download_civitai_images(
|
||||
|
||||
|
||||
def get_lora_image_cache_dir() -> Path:
|
||||
"""Get the directory used to cache downloaded LoRA example images."""
|
||||
# Store in the extension's own directory
|
||||
"""Get the directory used to cache downloaded LoRA example images.
|
||||
|
||||
``<data folder>/data/lora_images`` (``utils.library_location``): the node
|
||||
folder for a library that was not moved, else ComfyUI's
|
||||
``user/__promptmanager``.
|
||||
"""
|
||||
try:
|
||||
from ..utils import library_location
|
||||
except ImportError:
|
||||
from utils import library_location
|
||||
|
||||
ext_root = Path(__file__).resolve().parent.parent
|
||||
cache = ext_root / "data" / "lora_images"
|
||||
cache = Path(
|
||||
library_location.cache_dir("lora_images", extension_root=str(ext_root))
|
||||
)
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
return cache
|
||||
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
"""One process-wide, non-blocking gate around the tagging models (#249 B2).
|
||||
|
||||
Loading, swapping, running, downloading and unloading a tagging model all
|
||||
happen under this gate, whether the caller is a legacy Auto-tag route, the
|
||||
Tag dialog's Generate or (later) a bulk job. A caller that cannot take the
|
||||
gate is refused at once (``409 model-busy``), never queued.
|
||||
|
||||
Work that runs in a worker thread is admitted with :meth:`ModelGate.run`. A
|
||||
release requested while such work still runs (a client disconnect cancels
|
||||
the waiting coroutine, not the thread) takes effect only when the last
|
||||
admitted call really exits, so nobody can swap the model under a running
|
||||
inference.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import uuid
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
|
||||
class GateNotHeld(RuntimeError):
|
||||
"""Raised when work is admitted for an owner that does not hold the gate."""
|
||||
|
||||
|
||||
class ModelGate:
|
||||
"""A single-owner gate whose release waits for admitted worker calls."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._holder: Optional[str] = None
|
||||
self._inflight = 0
|
||||
self._releasing = False
|
||||
|
||||
def try_acquire(self, owner: str) -> bool:
|
||||
"""Take the gate for *owner*; False at once when anyone holds it."""
|
||||
with self._lock:
|
||||
if self._holder is not None:
|
||||
return False
|
||||
self._holder = owner
|
||||
self._inflight = 0
|
||||
self._releasing = False
|
||||
return True
|
||||
|
||||
def release(self, owner: str) -> None:
|
||||
"""Let go; deferred until admitted work of *owner* has exited."""
|
||||
with self._lock:
|
||||
if self._holder != owner:
|
||||
return
|
||||
if self._inflight:
|
||||
self._releasing = True
|
||||
else:
|
||||
self._clear_locked()
|
||||
|
||||
def holder(self) -> Optional[str]:
|
||||
with self._lock:
|
||||
return self._holder
|
||||
|
||||
def begin(self, owner: str) -> None:
|
||||
"""Admit one worker call; the holder must still hold the gate."""
|
||||
with self._lock:
|
||||
if self._holder != owner or self._releasing:
|
||||
raise GateNotHeld(f"{owner} does not hold the model gate")
|
||||
self._inflight += 1
|
||||
|
||||
def end(self, owner: str) -> None:
|
||||
"""One admitted call exited; completes a deferred release."""
|
||||
with self._lock:
|
||||
self._end_locked(owner)
|
||||
|
||||
async def run(
|
||||
self,
|
||||
owner: str,
|
||||
func: Callable[..., Any],
|
||||
*args,
|
||||
on_skipped: Optional[Callable[[], None]] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Run ``func`` in the default executor while *owner* holds the gate.
|
||||
|
||||
The executor future is shielded: cancelling the caller never cancels
|
||||
a call that has not started yet (which would leave it admitted
|
||||
forever) and never ends the admission of a call that is running.
|
||||
``on_skipped`` runs once when ``func`` never starts (the gate is not
|
||||
held, the executor refuses the call, or a loop shutdown drops it), so
|
||||
a caller can free what ``func`` would have freed.
|
||||
"""
|
||||
try:
|
||||
self.begin(owner)
|
||||
except GateNotHeld:
|
||||
if on_skipped is not None:
|
||||
on_skipped()
|
||||
raise
|
||||
state = {"started": False, "finished": False}
|
||||
|
||||
def finish() -> None:
|
||||
with self._lock:
|
||||
if not state["finished"]:
|
||||
state["finished"] = True
|
||||
self._end_locked(owner)
|
||||
|
||||
def work():
|
||||
with self._lock:
|
||||
if state["finished"]:
|
||||
# Already declared skipped (the future was cancelled as
|
||||
# the thread picked the call up): never run it twice over.
|
||||
return None
|
||||
state["started"] = True
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
finally:
|
||||
finish()
|
||||
|
||||
def never_ran(future) -> None:
|
||||
# Only a loop shutdown cancels the shielded future; a call that
|
||||
# never started must still give its admission back.
|
||||
if future.cancelled():
|
||||
with self._lock:
|
||||
skipped = not state["started"] and not state["finished"]
|
||||
if skipped:
|
||||
state["finished"] = True
|
||||
self._end_locked(owner)
|
||||
if skipped and on_skipped is not None:
|
||||
on_skipped()
|
||||
|
||||
try:
|
||||
future = asyncio.get_running_loop().run_in_executor(None, work)
|
||||
except BaseException:
|
||||
finish()
|
||||
if on_skipped is not None:
|
||||
on_skipped()
|
||||
raise
|
||||
future.add_done_callback(never_ran)
|
||||
return await asyncio.shield(future)
|
||||
|
||||
def _end_locked(self, owner: str) -> None:
|
||||
if self._holder != owner or self._inflight == 0:
|
||||
return
|
||||
self._inflight -= 1
|
||||
if self._releasing and self._inflight == 0:
|
||||
self._clear_locked()
|
||||
|
||||
def _clear_locked(self) -> None:
|
||||
self._holder = None
|
||||
self._inflight = 0
|
||||
self._releasing = False
|
||||
|
||||
|
||||
_GATE = ModelGate()
|
||||
|
||||
|
||||
def new_owner(kind: str) -> str:
|
||||
"""A unique owner name such as ``legacy-single:<uuid>``."""
|
||||
return f"{kind}:{uuid.uuid4()}"
|
||||
|
||||
|
||||
def try_acquire(owner: str) -> bool:
|
||||
return _GATE.try_acquire(owner)
|
||||
|
||||
|
||||
def release(owner: str) -> None:
|
||||
_GATE.release(owner)
|
||||
|
||||
|
||||
def holder() -> Optional[str]:
|
||||
return _GATE.holder()
|
||||
|
||||
|
||||
async def run(
|
||||
owner: str,
|
||||
func: Callable[..., Any],
|
||||
*args,
|
||||
on_skipped: Optional[Callable[[], None]] = None,
|
||||
**kwargs,
|
||||
):
|
||||
return await _GATE.run(owner, func, *args, on_skipped=on_skipped, **kwargs)
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
|
||||
// Activity-1 Task 3. The review dialog is compared with the concept's own `scan-review` ("Review metadata scan",
|
||||
@@ -63,16 +63,35 @@ function measureNotice(selector) {
|
||||
};
|
||||
}
|
||||
|
||||
/** D55: the concept's dialog with the production differences applied. */
|
||||
/** A concept `.setting-row` switch: its frame, its label and hint, and its checkbox (not its place-dependent margin). */
|
||||
function measureToggle(selector) {
|
||||
const row = document.querySelector(selector);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
return {
|
||||
children: [...row.children].map(node => [node.localName, ...node.classList].join('.')),
|
||||
frame: css(row, 'display', 'alignItems', 'justifyContent', 'columnGap', 'paddingTop', 'paddingBottom',
|
||||
'borderTopWidth', 'borderTopStyle', 'borderTopColor'),
|
||||
label: css(row.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
hint: css(row.querySelector('p'), 'fontSize', 'color', 'marginTop'),
|
||||
box: css(row.querySelector('input[type="checkbox"]'), 'accentColor', 'width', 'height', 'appearance',
|
||||
'backgroundColor', 'borderTopLeftRadius'),
|
||||
role: row.querySelector('input').getAttribute('role'),
|
||||
labelled: row.querySelector('label').htmlFor === row.querySelector('input').id,
|
||||
};
|
||||
}
|
||||
|
||||
/** D55 and D58: the concept's dialog with the production differences applied. */
|
||||
function mapReviewConcept(reference) {
|
||||
return {
|
||||
...reference,
|
||||
// No sample-plan note (prototype copy); the backup box sits before the actions.
|
||||
// No sample-plan note (prototype copy); the D58 "All files" toggle follows the grid; the backup box sits
|
||||
// before the actions.
|
||||
body: [...reference.body.filter(item => item !== 'p.demo-note').slice(0, -1), 'div.notice', reference.body.at(-1)]
|
||||
.map(item => item === 'div.meta-grid' ? 'dl.meta-grid' : item),
|
||||
// A `dl`, as in the concept's maintenance plan; real sources and the fixed Activity-1 scope.
|
||||
.flatMap(item => item === 'div.meta-grid' ? ['dl.meta-grid', 'div.setting-row'] : [item]),
|
||||
// A `dl`, as in the concept's maintenance plan; real sources. The scope keeps the concept's
|
||||
// "New and unresolved files" until "All files" is checked.
|
||||
grid: { ...reference.grid, tag: 'dl', entries: reference.grid.entries.map(entry => ({ ...entry,
|
||||
dd: { Sources: '2 output folders', Scope: 'All files' }[entry.dt] })) },
|
||||
dd: { Sources: '2 output folders', Scope: 'New and unresolved files' }[entry.dt] })) },
|
||||
actions: { ...reference.actions, buttons: reference.actions.buttons.map(button => button.text === 'Simulate scan'
|
||||
? { ...button, text: 'Start scan' } : button) },
|
||||
};
|
||||
@@ -92,7 +111,8 @@ async function conceptReview(page, theme) {
|
||||
await page.getByRole('combobox', { name: 'Theme', exact: true }).selectOption(theme);
|
||||
await page.locator('#main').getByRole('button', { name: 'Manage sources', exact: true }).click();
|
||||
await expect(page.locator('#main .notice')).toBeVisible();
|
||||
return { review, notice: await page.evaluate(measureNotice, '#main .notice') };
|
||||
return { review, notice: await page.evaluate(measureNotice, '#main .notice'),
|
||||
toggle: await page.evaluate(measureToggle, '#main .setting-row:has(input[type="checkbox"])') };
|
||||
}
|
||||
|
||||
async function productionReview(page, theme) {
|
||||
@@ -103,7 +123,16 @@ async function productionReview(page, theme) {
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review metadata scan');
|
||||
// The swapped dialog can open under the pointer that pressed Review scan; measure it at rest.
|
||||
await page.mouse.move(0, 0);
|
||||
return { review: await page.evaluate(measureReview), notice: await page.evaluate(measureNotice, 'dialog#modal[open] .notice') };
|
||||
const review = await page.evaluate(measureReview);
|
||||
const notice = await page.evaluate(measureNotice, 'dialog#modal[open] .notice');
|
||||
// The concept's switch is on; compare ours on too, then turn it off again (the default).
|
||||
const allFiles = page.getByRole('switch', { name: 'All files', exact: true });
|
||||
await allFiles.check();
|
||||
await page.mouse.move(0, 0);
|
||||
const toggle = await page.evaluate(measureToggle, 'dialog#modal[open] .setting-row');
|
||||
await allFiles.uncheck();
|
||||
await page.mouse.move(0, 0);
|
||||
return { review, notice, toggle };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) test(`review dialog matches the concept scan-review ${theme}`, async ({ page, request }) => {
|
||||
@@ -114,19 +143,29 @@ for (const theme of THEMES) test(`review dialog matches the concept scan-review
|
||||
const production = await productionReview(page, theme);
|
||||
expect(diffInventory(mapReviewConcept(concept.review), production.review)).toEqual([]);
|
||||
expect(diffInventory(concept.notice, production.notice)).toEqual([]);
|
||||
expect(diffInventory(concept.toggle, production.toggle)).toEqual([]);
|
||||
await expect(page.locator('dialog#modal[open] .setting-row > div > label')).toHaveText('All files');
|
||||
await expect(page.locator('dialog#modal[open] .setting-row input')).not.toBeChecked();
|
||||
await expect(page.getByRole('switch', { name: 'All files', exact: true })).toBeVisible();
|
||||
await expect(page.locator('dialog#modal[open] .notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
await expect(page.locator('dialog#modal[open] .notice .row-actions button')).toHaveText(['Back up now']);
|
||||
|
||||
// Mutation check: one CSS change and one removed element are both noticed, then undone to an exact match again.
|
||||
const undoCss = await mutate(page, { css: '#modal .meta-grid { gap: 3px; } #modal .notice { padding: 2px; }' });
|
||||
const undoCss = await mutate(page, { css: '#modal .meta-grid { gap: 3px; } #modal .notice { padding: 2px; } #modal .setting-row { gap: 3px; }' });
|
||||
const cssMutated = await page.evaluate(measureReview);
|
||||
const noticeMutated = await page.evaluate(measureNotice, 'dialog#modal[open] .notice');
|
||||
await page.getByRole('switch', { name: 'All files', exact: true }).check();
|
||||
await page.mouse.move(0, 0);
|
||||
const toggleMutated = await page.evaluate(measureToggle, 'dialog#modal[open] .setting-row');
|
||||
await page.getByRole('switch', { name: 'All files', exact: true }).uncheck();
|
||||
await page.mouse.move(0, 0);
|
||||
expect(diffInventory(concept.toggle, toggleMutated).map(item => item.path)).toEqual(['frame.columnGap']);
|
||||
expect(diffInventory(mapReviewConcept(concept.review), cssMutated).map(item => item.path)).toEqual(expect.arrayContaining(['grid.columnGap', 'grid.rowGap']));
|
||||
expect(diffInventory(concept.notice, noticeMutated).map(item => item.path)).toEqual(['frame.paddingTop', 'frame.paddingLeft']);
|
||||
await undoCss();
|
||||
const undoDom = await mutate(page, { remove: ['#modal-body > p.small'] });
|
||||
const domMutated = await page.evaluate(() => [...document.querySelectorAll('#modal-body > *')].map(node => node.localName));
|
||||
expect(domMutated).toEqual(['dl', 'div', 'div']);
|
||||
expect(domMutated).toEqual(['dl', 'div', 'div', 'div']);
|
||||
await undoDom();
|
||||
expect(diffInventory(mapReviewConcept(concept.review), await page.evaluate(measureReview))).toEqual([]);
|
||||
});
|
||||
@@ -210,14 +249,16 @@ const EXPECTED_JOB_VIEW = {
|
||||
pressed: { border: true, background: true, added: true, bad: true, warn: true },
|
||||
};
|
||||
|
||||
async function openRunningJob(page, request, theme, viewport = { width: 1600, height: 1100 }) {
|
||||
const SCAN_START = { start: { total: 19622 }, step: { processed: 8240,
|
||||
counters: { known: 6511, review: 37, no_metadata: 254, errors: 18 },
|
||||
records: [{ group: 'added', path: 'CCC/mabes9sans-dataset-v1/IMG_GEN_00369_.png', detail: 'Added to the Library as “Mabes portrait”' }] } };
|
||||
|
||||
async function openRunningJob(page, request, theme, viewport = { width: 1600, height: 1100 }, data = SCAN_START) {
|
||||
await page.setViewportSize(viewport);
|
||||
const started = await request.post('/__test__/jobs/start', { data: { total: 19622 } });
|
||||
const started = await request.post('/__test__/jobs/start', { data: data.start });
|
||||
await expect(started).toBeOK();
|
||||
const { job } = await started.json();
|
||||
await expect(await request.post(`/__test__/jobs/${job.id}/step`, { data: { processed: 8240,
|
||||
counters: { known: 6511, review: 37, no_metadata: 254, errors: 18 },
|
||||
records: [{ group: 'added', path: 'CCC/mabes9sans-dataset-v1/IMG_GEN_00369_.png', detail: 'Added to the Library as “Mabes portrait”' }] } })).toBeOK();
|
||||
await expect(await request.post(`/__test__/jobs/${job.id}/step`, { data: data.step })).toBeOK();
|
||||
await openProduction(page, `activity/${job.id}`, theme);
|
||||
await expect(page.locator('#workspace-main .file-row')).toHaveCount(1);
|
||||
return job;
|
||||
@@ -239,6 +280,30 @@ for (const theme of THEMES) test(`job view follows the approved mockup ${theme}`
|
||||
expect(await page.evaluate(measureJobView)).toEqual(EXPECTED_JOB_VIEW);
|
||||
});
|
||||
|
||||
// D89 (plan D66): an image tagging job's view is the same mockup with its own four tiles (Tagged green, Unchanged,
|
||||
// Skipped warning, Errors red) in the mockup's five-column row, its own outcome list and "Tagging log".
|
||||
const TAG_START = { start: { kind: 'image-tags', title: 'Auto-tag images', mode: 'auto', images: 4, total: 4 }, step: { processed: 2,
|
||||
counters: { unchanged: 1, errors: 1 }, records: [{ group: 'tagged', path: 'CCC/mabes9sans-dataset-v1/IMG_GEN_00369_.png', detail: 'Added 3 tags' }] } };
|
||||
const EXPECTED_TAG_VIEW = {
|
||||
...EXPECTED_JOB_VIEW,
|
||||
progress: [MOCKUP_PROGRESS[0], MOCKUP_PROGRESS[1], 'dl.stat-row(div.added.stat,div.stat,div.stat.warn,div.bad.stat)'],
|
||||
summary: '● Tagging log — raw detail, click to show',
|
||||
};
|
||||
|
||||
for (const theme of THEMES) test(`image tagging job view follows the approved mockup ${theme}`, async ({ page, request }) => {
|
||||
await seedConcept(request);
|
||||
await openRunningJob(page, request, theme, undefined, TAG_START);
|
||||
expect(await page.evaluate(measureJobView)).toEqual(EXPECTED_TAG_VIEW);
|
||||
await expect(page.locator('#workspace-main .outcome-list > button')).toHaveText(['Tagged1', 'Errors1', 'Skipped0', 'Unchanged1']);
|
||||
// Mutation check: the scan's five tiles and the scan's log label are both noticed.
|
||||
expect(diffInventory(EXPECTED_JOB_VIEW, await page.evaluate(measureJobView)).map(item => item.path).sort())
|
||||
.toEqual(['progress.2', 'summary']);
|
||||
const undoCss = await mutate(page, { css: '#workspace-main .stat-row .stat.warn dd { color: red; }' });
|
||||
expect(diffInventory(EXPECTED_TAG_VIEW, await page.evaluate(measureJobView)).map(item => item.path)).toEqual(['pressed.warn']);
|
||||
await undoCss();
|
||||
expect(await page.evaluate(measureJobView)).toEqual(EXPECTED_TAG_VIEW);
|
||||
});
|
||||
|
||||
test('job view fits a phone without page scroll', async ({ page, request }) => {
|
||||
await seedConcept(request);
|
||||
await openRunningJob(page, request, 'dark', { width: 390, height: 844 });
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
test.beforeEach(async ({page, browserName}) => {
|
||||
@@ -57,7 +57,8 @@ test('Classic keeps immediate selection, search query and metadata serialization
|
||||
});
|
||||
|
||||
test('Classic detaches native text synchronization on removal',async({page})=>{
|
||||
await page.waitForTimeout(60);
|
||||
// The page installs the text sync (serializeValue and its listeners) in a 50 ms timer; wait for it, not for time.
|
||||
await expect.poll(()=>page.evaluate(()=>typeof node.widgets.find(item=>item.name==='text').serializeValue)).toBe('function');
|
||||
const values=await page.evaluate(()=>{
|
||||
const widget=node.widgets.find(item=>item.name==='text');
|
||||
widget.inputEl.value='Before removal';widget.inputEl.dispatchEvent(new Event('input'));
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// same concept selectors in the immutable concept (/__test__/canonical/workspace) and in production, which now renders
|
||||
// the concept's classes. The concept is never rewritten: each page's map<Page>Concept(reference) applies only the
|
||||
// recorded spec-§8 deviations, each commented with its D-number, and diffInventory must then report nothing.
|
||||
import {expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {expect} from './fixtures.mjs';
|
||||
export {diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
export const CONCEPT_URL='/__test__/canonical/workspace';
|
||||
@@ -78,6 +78,34 @@ export async function seedConcept(request,data={}) {
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Seed the concept fixture and fix the browser's clock at the fixture's own instant (`manifest.now`), so the page and
|
||||
* the fixture read one "now": a prompt used now reads "Used today" and the caption names the fixture's day even when
|
||||
* the run crosses midnight between seeding and rendering. Timers keep running; only `Date` stands still.
|
||||
*/
|
||||
export async function seedConceptAt(page,request,data={}) {
|
||||
const manifest=await seedConcept(request,data);
|
||||
await page.clock.setFixedTime(new Date(manifest.now*1000));
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save `tags` as the current image tag set of the listed file at `path` (root 0) through the real observe and tag
|
||||
* routes (#249 PR C), releasing the capture afterwards; returns the receipt's saved names.
|
||||
*/
|
||||
export async function saveImageTags(request,path,tags) {
|
||||
const listing=await (await request.get('/prompt_manager/images/output?mode=workspace&limit=40')).json();
|
||||
const target={root_index:0,root_scope:listing.images.find(item=>item.root_index===0).root_scope,path};
|
||||
const observed=await (await request.post('/prompt_manager/image-annotations/observe',{data:{target}})).json();
|
||||
expect(observed.success).toBe(true);
|
||||
try {
|
||||
const receipt=await (await request.put('/prompt_manager/image-annotations/tags',{data:{target,capture_token:observed.capture.token,
|
||||
sha256:observed.capture.sha256,expected_epoch:observed.library_epoch,expected_tag_revision:observed.tag_revision,tags}})).json();
|
||||
expect(receipt.success).toBe(true);
|
||||
return receipt.tags;
|
||||
} finally {await request.delete(`/prompt_manager/image-annotations/capture/${observed.capture.token}`);}
|
||||
}
|
||||
|
||||
async function selectTheme(page,theme) {
|
||||
await page.getByRole('combobox',{name:'Theme',exact:true}).selectOption(theme);
|
||||
}
|
||||
@@ -99,6 +127,8 @@ export async function openProduction(page,route,theme,motion='no-preference') {
|
||||
await expect(page.locator('#primary-nav .nav-count')).toHaveCount(3);
|
||||
// SAVED VIEWS' Favorites count (D51), the concept's starred images in the concept fixture.
|
||||
await expect(page.locator('#saved-nav a[data-route="favorites"] .nav-count')).toHaveCount(1);
|
||||
// SAVED VIEWS' Needs review count (ruling R5): the exact pending count.
|
||||
await expect(page.locator('#saved-nav a[data-route="review"] .nav-count')).toHaveCount(1);
|
||||
}
|
||||
|
||||
/** Computed outline of the focused element, after keyboard focus so `:focus-visible` applies. */
|
||||
@@ -121,10 +151,12 @@ export async function hoverStateOf(page,locator) {
|
||||
/**
|
||||
* Mutate the open page so a comparison can be shown to notice a change (the A7 mutation check): `css` is appended as
|
||||
* the last stylesheet, so it wins over equally specific concept rules, and every element matching a `remove` selector
|
||||
* is detached. Returns an async undo that restores the stylesheet and the removed elements in place.
|
||||
* is detached; each `clone` ({selector, text}) inserts a copy of the first match after it, holding `text` and no
|
||||
* `data-action` (a control production adds beside a concept one, D85). Returns an async undo that restores the
|
||||
* stylesheet and the removed elements in place and drops the copies.
|
||||
*/
|
||||
export async function mutate(page,{css=null,remove=[]}={}) {
|
||||
const token=await page.evaluate(({css,remove})=>{
|
||||
export async function mutate(page,{css=null,remove=[],clone=[]}={}) {
|
||||
const token=await page.evaluate(({css,remove,clone})=>{
|
||||
const token=`mutation-${Math.random().toString(36).slice(2)}`;
|
||||
const undo=[];
|
||||
if(css) {
|
||||
@@ -139,9 +171,15 @@ export async function mutate(page,{css=null,remove=[]}={}) {
|
||||
node.remove();undo.push(()=>parent.insertBefore(node,next));
|
||||
}
|
||||
}
|
||||
for(const {selector,text} of clone) {
|
||||
const node=document.querySelector(selector);
|
||||
if(!node)throw new Error(`mutate: nothing matches ${selector}`);
|
||||
const copy=node.cloneNode(true);copy.textContent=text;copy.removeAttribute('data-action');
|
||||
node.after(copy);undo.push(()=>copy.remove());
|
||||
}
|
||||
(window.__conceptMutations??=new Map()).set(token,undo);
|
||||
return token;
|
||||
},{css,remove});
|
||||
},{css,remove,clone});
|
||||
return ()=>page.evaluate(token=>{
|
||||
const undo=window.__conceptMutations?.get(token);
|
||||
if(!undo)throw new Error('mutate: this mutation was already undone or the page reloaded');
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
// Shared `test` for the page workflows: every Playwright worker owns one
|
||||
// tests/browser/server.py process on a port the OS chose for it, with its own temp root
|
||||
// and database. `baseURL` (and so `page`, `request` and `browser.newContext`)
|
||||
// points at that worker's server. A server that dies fails the running test
|
||||
// with its stderr tail, and the next test in the worker gets a fresh server.
|
||||
// A test that fails gets the stderr its server wrote during it attached as
|
||||
// `server-stderr`.
|
||||
import { test as base, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { attachServerStderr, createServerSupervisor, startBrowserServer } from '../../tools/vendor/start-browser-server.mjs';
|
||||
|
||||
const SERVER_FIXTURE_TIMEOUT_MS = 90_000;
|
||||
|
||||
export const test = base.extend({
|
||||
testServer: [async ({}, use) => {
|
||||
const supervisor = createServerSupervisor({ start: () => startBrowserServer() });
|
||||
try {
|
||||
await use(supervisor);
|
||||
} finally {
|
||||
await supervisor.stop();
|
||||
}
|
||||
}, { scope: 'worker', timeout: SERVER_FIXTURE_TIMEOUT_MS }],
|
||||
workerServer: [async ({ testServer }, use, testInfo) => {
|
||||
const server = await testServer.ensure();
|
||||
const mark = server.stderrMark();
|
||||
await use(server);
|
||||
await attachServerStderr(testInfo, server, mark);
|
||||
const crash = testServer.crash(server);
|
||||
if (crash) throw crash;
|
||||
}, { auto: true, timeout: SERVER_FIXTURE_TIMEOUT_MS }],
|
||||
baseURL: async ({ workerServer }, use) => use(workerServer.url),
|
||||
});
|
||||
|
||||
export { expect };
|
||||
@@ -1,5 +1,5 @@
|
||||
import {readFileSync} from 'node:fs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
for (const variant of ['Focused', 'Split']) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction, SIZES } from './concept-inventory.mjs';
|
||||
// D10/D20: the heading is the linked positive's title (its confirmed first line here), never the pair's
|
||||
// "Inspector recipe", which names the negative variant.
|
||||
@@ -92,7 +92,7 @@ test('inspection wire preserves exact rooted envelope', async ({ request }) => {
|
||||
const inspector=page=>page.locator('.detail-panel');
|
||||
/** D19: the selected card's image opens the viewer (the side panel has no full-preview cover). */
|
||||
const openSelected=page=>page.locator('.media-card.selected .image-open').click();
|
||||
const selectFile=async(page,name,root=0)=>page.locator(`article[data-image-key="${root}:inspector/${name}"] .image-card-actions button`).click();
|
||||
const selectFile=async(page,name,root=0)=>page.locator(`article[data-image-key="${root}:inspector/${name}"] .image-card-actions button:first-child`).click();
|
||||
async function fullFixture(page,request) {
|
||||
await expect(await request.post('/__test__/reset',{data:{full_inspector:true}})).toBeOK();
|
||||
await page.goto('/prompt_manager/workspace#/images');await selectFile(page,'complete.png');
|
||||
@@ -237,9 +237,12 @@ const viewerStructure=details=>details.evaluate(node=>{
|
||||
test('viewer details hold provenance, raw records and downloads',async({page,request})=>{
|
||||
await page.setViewportSize({width:1600,height:1100});const reads=await patchInspection(page);
|
||||
await fullFixture(page,request);let details=await viewerDetails(page,'complete.png');
|
||||
// The panel content without the cover, then the two disclosures, then the pair send and history buttons (D20).
|
||||
// The panel content without the cover, D117's Copy settings summary after the facts, then the two disclosures,
|
||||
// then the pair send and history buttons (D20).
|
||||
expect(await viewerStructure(details)).toEqual({outer:['div','button.send','button'],
|
||||
inner:['div.panel-heading','p.small','div.tags','section.prompt-section.positive','section.prompt-section.negative','div.tags','dl.meta-grid','details','details']});
|
||||
inner:['div.panel-heading','p.small','div.tags','section.prompt-section.positive','section.prompt-section.negative','div.tags','dl.meta-grid','button.settings-summary','details','details']});
|
||||
await expect(details.locator(':scope > div > button')).toHaveText('Copy settings summary');
|
||||
await expect(details.locator(':scope > div > button.settings-summary')).toHaveCSS('white-space','normal');
|
||||
await expect(details.locator('.detail-cover,.detail-inner,.detail-footer')).toHaveCount(0);
|
||||
await expect(details.locator('.panel-heading h2')).toHaveText(POSITIVE_TITLE);await expect(details.locator('.panel-heading .chip')).toHaveText('PNG');
|
||||
await expect(details.locator(':scope > div > p.small')).toHaveText('complete.png');
|
||||
@@ -460,7 +463,7 @@ test('full inspector labels container representations and releases action URLs',
|
||||
test('full inspector distinguishes missing, limited and legacy unknown generation facts',async({page,request})=>{
|
||||
await fullFixture(page,request);const panel=inspector(page);
|
||||
await selectFile(page,'empty.png');await expect(panel.locator('.meta-grid dd')).toHaveText(Array(6).fill('Not recorded'));
|
||||
await page.locator('article[data-image-key="0:architecture/study-01.png"] .image-card-actions button').click();await expect(panel.locator('.meta-grid dd')).toHaveText(Array(6).fill('Not recorded'));
|
||||
await page.locator('article[data-image-key="0:architecture/study-01.png"] .image-card-actions button:first-child').click();await expect(panel.locator('.meta-grid dd')).toHaveText(Array(6).fill('Not recorded'));
|
||||
await selectFile(page,'control-heavy.png');
|
||||
const explanation='Configuration exceeds inspection limits; download the original file to inspect it locally.';
|
||||
await expect(panel.getByText(explanation,{exact:true})).toHaveCount(1);await expect(panel.getByText(explanation,{exact:true})).toBeVisible();
|
||||
@@ -637,7 +640,7 @@ test('collection viewer follows filtered loaded order beyond mounted cards',asyn
|
||||
// #249 D26: the whole positive's images are in the All images dialog's own embedded gallery.
|
||||
await page.getByRole('button',{name:'All images for this positive',exact:true}).click();
|
||||
const listing=page.getByRole('dialog',{name:'All images for this positive',exact:true});
|
||||
await listing.locator('article[data-image-key="0:inspector/complete.png"] .image-card-actions button').click();
|
||||
await listing.locator('article[data-image-key="0:inspector/complete.png"] .image-card-actions button:first-child').click();
|
||||
await listing.locator('.media-card.selected .image-open').click();await expect(mediaDialog(page).locator('.viewer-canvas img')).toBeVisible();
|
||||
await expect(position(page)).toContainText('of 2 loaded:');
|
||||
});
|
||||
@@ -804,6 +807,7 @@ test('slideshow preference failures leave native playback usable',async({page,re
|
||||
});
|
||||
|
||||
test('full inspector attaches matched theme and mobile evidence with reachable content',async({page,request},testInfo)=>{
|
||||
test.setTimeout(120_000); // 32 evidence captures across four viewports, like the concept comparisons
|
||||
await expect(await request.post('/__test__/reset',{data:{full_inspector:true,include_video:true}})).toBeOK();
|
||||
await page.goto('/prompt_manager/workspace#/images');
|
||||
const measurements=[];
|
||||
@@ -929,7 +933,7 @@ test('collection viewer matches approved responsive frame',async({page,request},
|
||||
await page.screenshot({path:testInfo.outputPath('viewer-mobile-wider-font-2x.png')});
|
||||
await modal.getByRole('button',{name:'Close image viewer',exact:true}).click();await page.evaluate(()=>{document.documentElement.style.zoom='1';});await settleViewport(390);
|
||||
const longFiles=await(await request.get('/prompt_manager/images/output?mode=workspace&q=a-very-long&limit=40')).json();const long=longFiles.images[0];
|
||||
await page.locator(`article[data-image-key="${long.root_index}:${long.path}"] .image-card-actions button`).click();await openCollection(page);
|
||||
await page.locator(`article[data-image-key="${long.root_index}:${long.path}"] .image-card-actions button:first-child`).click();await openCollection(page);
|
||||
await page.evaluate(()=>{document.documentElement.style.zoom='2';});await settleViewport(390);await modal.evaluate(node=>{node.scrollTop=0;});
|
||||
await expect(modal.getByRole('button',{name:'Close image viewer',exact:true})).toBeInViewport();
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { writeFile } from 'node:fs/promises';
|
||||
import { expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { expect } from './fixtures.mjs';
|
||||
|
||||
// Content, row counts, and image bytes come from independent synthetic datasets.
|
||||
// Saved choices and actual executions are separate production sections. The concept's
|
||||
@@ -17,7 +17,6 @@ export const canonicalDifferences = {
|
||||
// Closed by #249: the gallery's checkbox is Images-only (#259) and the role blocks are the concept's (PR D, F); the
|
||||
// exact history comparison is tests/browser/history-concept.spec.mjs.
|
||||
remainingApplicationPolish: [
|
||||
'Gallery card tag button (#249 D9, image-annotations slice) remains outside #209',
|
||||
'Production keeps mobile toolbar inside media panel and readable scrolling details; the concept toolbar collision is not reproduced',
|
||||
],
|
||||
};
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact} from './concept-inventory.mjs';
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow,withoutItems} from './concept-inventory.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConceptAt,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow,withoutItems} from './concept-inventory.mjs';
|
||||
|
||||
// Task C7 of #249: the production Images page against the immutable concept, on the concept-shaped real fixture (24
|
||||
// images of the concept's six prompts, its seeds, sizes and favorites). Heading, toolbar, filter row, caption, grid,
|
||||
@@ -30,9 +30,11 @@ function captionDay(seconds,now=new Date()) {
|
||||
return `${name} · ${date.toLocaleDateString('en-US',options)}`;
|
||||
}
|
||||
|
||||
/** D8: the selection bar's real actions only; Add tags, Add to collection and Remove links… are left out. */
|
||||
const OMITTED_BULK=Object.freeze(['Add tags','Add to collection','Remove links…']);
|
||||
const OMITTED_BULK_SELECTORS=Object.freeze(['bulk-tags','bulk-collection','bulk-remove'].map(action=>`#bulkbar [data-action="${action}"]`));
|
||||
/** D8 (amended by #249 PR D): the selection bar's real actions only; Add to collection and Remove links… are left out. */
|
||||
const OMITTED_BULK=Object.freeze(['Add to collection','Remove links…']);
|
||||
const OMITTED_BULK_SELECTORS=Object.freeze(['bulk-collection','bulk-remove'].map(action=>`#bulkbar [data-action="${action}"]`));
|
||||
/** D85: Auto-tag selected follows the concept's Add tags, a concept button of its own text. */
|
||||
const AUTO_TAG_CLONE=Object.freeze([{selector:'#bulkbar [data-action="bulk-tags"]',text:'Auto-tag selected'}]);
|
||||
|
||||
/**
|
||||
* `withoutItems` for a row with padding and a border (the `.bulkbar`): its items flow again inside the content box,
|
||||
@@ -59,12 +61,12 @@ function mapBulkbar(bulkbar,rise) {
|
||||
}
|
||||
|
||||
/**
|
||||
* A list row's info once D9 narrows its actions to `actionsWidth`, from the concept's boxes:
|
||||
* A list row's info with its actions `actionsWidth` wide, from the concept's boxes:
|
||||
* - D3 (the measurable list row, `.list-grid .media-info` in the overlay): the info fills the row beside the art and may
|
||||
* shrink below a long nowrap title (`min-width: 0`), so it is the card's inner width less the art, and the actions
|
||||
* end at its right padding; it keeps its own height (text or actions plus padding), centred in the row the concept
|
||||
* stretches it across, so the text, centred in either, moves only by half the lines it lost;
|
||||
* - D9: the wider text column fits a subtitle the concept wrapped when its one-line width (`maxContent`) now fits.
|
||||
* - the text column fits a subtitle the concept wrapped when its one-line width (`maxContent`) now fits.
|
||||
*/
|
||||
function listInfo(card,actionsWidth) {
|
||||
const {info}=card,{copy,p}=info;
|
||||
@@ -83,7 +85,7 @@ function listInfo(card,actionsWidth) {
|
||||
/** Recorded Images deviations only (spec §8); everything else in the concept inventory must equal production. */
|
||||
const nocase=(a,b)=>{const x=a.toLowerCase(),y=b.toLowerCase();return x<y?-1:x>y?1:a<b?-1:a>b?1:0;};
|
||||
|
||||
export function mapImagesConcept(reference,{manifest,now=new Date()}) {
|
||||
export function mapImagesConcept(reference,{manifest,now=new Date(manifest.now*1000)}) {
|
||||
const paths=new Map(manifest.images.map(image=>[image.filename,image.path]));
|
||||
// D14: the sort appends production's two sorts. Save view is the concept's own (saved views, #249 D16 retired).
|
||||
const toolbar={...reference.toolbar};
|
||||
@@ -109,21 +111,16 @@ export function mapImagesConcept(reference,{manifest,now=new Date()}) {
|
||||
const cards=reference.cards.map(card=>{
|
||||
const filename=card.open.label.replace(/^Open /,'');
|
||||
const actions=card.info.actions;
|
||||
const tag=actions.buttons.find(button=>button.label?.startsWith('Tag or auto-tag '));
|
||||
// D9: no Tag button until the image-annotations slice; the actions group narrows by its box and gap.
|
||||
const freed=tag?tag.width+parseFloat(actions.columnGap):0;
|
||||
// D3: a list row's info keeps its own height (text or actions plus its padding) so bounded rendering can measure
|
||||
// it, centred in the row the concept stretches it across; the text and the actions keep their place.
|
||||
const actionsWidth=actions.width-freed;
|
||||
const listRow=list?listInfo(card,actionsWidth):{};
|
||||
const listRow=list?listInfo(card,actions.width):{};
|
||||
return {...card,
|
||||
// D29: the concept's alt adds its demo-only " · sample illustration N" to the card title.
|
||||
img:{...card.img,alt:card.img.alt.replace(/ · sample illustration \d+$/,'')},
|
||||
info:{...card.info,...listRow.info,copy:{...card.info.copy,...listRow.copy},p:{...card.info.p,...listRow.p},
|
||||
// D10: the title keeps the file's relative path (the fixture manifest's) in `title`.
|
||||
h3:{...card.info.h3,title:paths.get(filename)??null},
|
||||
actions:{...actions,width:actionsWidth,left:listRow.actionsLeft??actions.left+freed,
|
||||
buttons:actions.buttons.filter(button=>button!==tag)}}};
|
||||
actions:{...actions,left:listRow.actionsLeft??actions.left}}};
|
||||
});
|
||||
// D17: the right-hand date is the newest result's day under Recently added, here the fixture's newest mtime.
|
||||
// D35: the count line uses the singular for one.
|
||||
@@ -321,7 +318,7 @@ async function behaviorsOf(page,scope) {
|
||||
return {hover,opened,viewerTitle,focusAfterViewer,selected,layouts,search,single};
|
||||
}
|
||||
|
||||
test.beforeEach(async({request},testInfo)=>{testInfo.manifest=await seedConcept(request);});
|
||||
test.beforeEach(async({page,request},testInfo)=>{testInfo.manifest=await seedConceptAt(page,request);});
|
||||
|
||||
for(const [width,height] of SIZES)for(const theme of THEMES)test(`images page geometry, tokens, controls and behaviors match the concept ${width}x${height} ${theme}`,async({page,request},testInfo)=>{
|
||||
test.setTimeout(240_000);
|
||||
@@ -351,7 +348,7 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`images page ge
|
||||
importDiff:diffInventory(mapImportConcept(referenceImport),actualImport),referenceBehaviors,actualBehaviors};
|
||||
}
|
||||
// Load more in production: a fixture past one listing page appends one batch and shows the concept's toast.
|
||||
testInfo.manifest=await seedConcept(request,{extra_images:LOAD_MORE_EXTRA});
|
||||
testInfo.manifest=await seedConceptAt(page,request,{extra_images:LOAD_MORE_EXTRA});
|
||||
for(const motion of MOTIONS) {
|
||||
const total=FIXTURE_IMAGES+LOAD_MORE_EXTRA;
|
||||
await openProductionLoaded(page,theme,motion,total);
|
||||
@@ -448,6 +445,10 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`images selecti
|
||||
const restore=await mutate(page,{remove:[...OMITTED_BULK_SELECTORS]});
|
||||
const removed=(await inventoryOf(page,'#main')).bulkbar;
|
||||
await restore();
|
||||
// The oracle for production's bar: the concept page itself without the omitted actions and with D85's button.
|
||||
const restoreProduction=await mutate(page,{remove:[...OMITTED_BULK_SELECTORS],clone:AUTO_TAG_CLONE});
|
||||
const oracle=await inventoryOf(page,'#main');
|
||||
await restoreProduction();
|
||||
await captureCanonicalState(page,testInfo,`concept-images-selected-${width}-${theme}`);
|
||||
await openProductionLoaded(page,theme,'no-preference');
|
||||
const emptyActual=await inventoryOf(page,'#workspace-main');
|
||||
@@ -459,12 +460,13 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`images selecti
|
||||
expect.soft(reference.bulkbar.bar.items.map(item=>item.text),'concept bar').toEqual(['1 selected','Add tags','Set category',
|
||||
'Add to collection','Export metadata','Remove links…','Clear']);
|
||||
expect.soft(diffInventory(stripExact(mapBulkbar(reference.bulkbar,0).bulkbar.bar),stripExact(removed.bar)),'bar without the omitted actions').toEqual([]);
|
||||
expect.soft(actual.bulkbar.bar.items.map(item=>item.text),'production bar').toEqual(['1 selected','Set category','Export metadata','Clear']);
|
||||
expect.soft(actual.bulkbar.bar.items.map(item=>item.text),'production bar').toEqual(['1 selected','Add tags','Auto-tag selected',
|
||||
'Set category','Export metadata','Clear']);
|
||||
expect.soft(compare(empty,emptyActual,manifest,['bulkbar','cards','panel']),'nothing checked').toEqual([]);
|
||||
expect.soft(compare(reference,actual,manifest,['bulkbar','cards','panel']),'one card checked').toEqual([]);
|
||||
expect.soft(compare(oracle,actual,manifest,['bulkbar','cards','panel']),'one card checked').toEqual([]);
|
||||
// The bar comparison notices a change to the production bar.
|
||||
await mutate(page,{css:'.bulkbar { padding: 4px }'});
|
||||
expect(compare(reference,await inventoryOf(page,'#workspace-main'),manifest,['bulkbar']).map(item=>item.path)).toContain('bulkbar.bar.paddingTop');
|
||||
expect(compare(oracle,await inventoryOf(page,'#workspace-main'),manifest,['bulkbar']).map(item=>item.path)).toContain('bulkbar.bar.paddingTop');
|
||||
});
|
||||
|
||||
// The comparison is only worth something if it notices a change (spec §7 mutation check): one CSS mutation (the
|
||||
@@ -512,7 +514,7 @@ test('one appended image reads in the singular',async({page,request})=>{
|
||||
test.setTimeout(120_000);
|
||||
await page.setViewportSize({width:1600,height:1100});
|
||||
const total=PRODUCTION_BATCH+1;
|
||||
await seedConcept(request,{extra_images:total-FIXTURE_IMAGES});
|
||||
await seedConceptAt(page,request,{extra_images:total-FIXTURE_IMAGES});
|
||||
await openProductionLoaded(page,'dark','no-preference',total);
|
||||
await expect(page.locator('.infinite-continuation > p').first()).toHaveText(countCopy(`${PRODUCTION_BATCH} of ${total} images · more load as you scroll`));
|
||||
// A dispatched press: scrolling the button into view lets the page's own observer load the one image first.
|
||||
@@ -528,7 +530,7 @@ test('one image appended by scrolling reads in the singular',async({page,request
|
||||
test.setTimeout(120_000);
|
||||
await page.setViewportSize({width:1600,height:1100});
|
||||
const total=PRODUCTION_BATCH+1;
|
||||
await seedConcept(request,{extra_images:total-FIXTURE_IMAGES});
|
||||
await seedConceptAt(page,request,{extra_images:total-FIXTURE_IMAGES});
|
||||
await openProductionLoaded(page,'dark','no-preference',total);
|
||||
await expect(page.locator('#result-count')).toHaveText(new RegExp(`^${PRODUCTION_BATCH} of ${total} images`));
|
||||
await page.locator('.infinite-continuation').scrollIntoViewIfNeeded();
|
||||
@@ -538,8 +540,9 @@ test('one image appended by scrolling reads in the singular',async({page,request
|
||||
});
|
||||
|
||||
// D3 (one footer height for the collection): bounded rendering cannot measure unrendered cards, so every row reserves
|
||||
// the tallest measured footer. Two seedless files (D11 "W × H · PNG", one line) form the first row at 390px, where
|
||||
// every concept subtitle wraps: their row is as tall as the wrapped rows, and every concept row moves down by one.
|
||||
// the tallest measured footer. Two seedless files (D11 "W × H · PNG", one line: a short "96 × 72" beside Info and Tag)
|
||||
// form the first row at 390px, where every concept subtitle wraps: their row is as tall as the wrapped rows, and every
|
||||
// concept row moves down by one.
|
||||
test('one footer height for the collection (D3)',async({page,request})=>{
|
||||
// A second fixture seed (dozens of PNGs and their thumbnails) runs inside the test.
|
||||
test.setTimeout(120_000);
|
||||
@@ -548,7 +551,7 @@ test('one footer height for the collection (D3)',async({page,request})=>{
|
||||
const reference=await inventoryOf(page,'#main');
|
||||
expect(reference.grid.columns).toBe(2);
|
||||
expect(reference.cards.every(card=>card.info.p.height>parseFloat(card.info.p.lineHeight))).toBe(true);
|
||||
const manifest=await seedConcept(request,{seedless_images:2});
|
||||
const manifest=await seedConceptAt(page,request,{seedless_images:2,short_seedless:true});
|
||||
await openProductionLoaded(page,'dark','no-preference',FIXTURE_IMAGES+2);
|
||||
const actual=await inventoryOf(page,'#workspace-main');
|
||||
const box=card=>({left:card.left,top:card.top,width:card.width,height:card.height});
|
||||
@@ -557,7 +560,10 @@ test('one footer height for the collection (D3)',async({page,request})=>{
|
||||
...reference.cards.slice(0,4).map(card=>({...box(card),top:card.top+pitch}))]);
|
||||
const seedless=actual.cards.slice(0,2);
|
||||
expect(seedless.map(card=>card.open.label)).toEqual(manifest.seedless.map(image=>`Open ${image.filename}`));
|
||||
expect(seedless.map(card=>card.info.p.text)).toEqual(['1536 × 1152 · PNG','1536 × 1152 · PNG']);
|
||||
expect(seedless.map(card=>card.info.p.text)).toEqual(['96 × 72 · PNG','96 × 72 · PNG']);
|
||||
// Their own footer is one line shorter than the row it reserves.
|
||||
for(const card of seedless)expect(card.info.height).toBeLessThan(reference.cards[0].info.height);
|
||||
for(const card of seedless) {
|
||||
expect(card.info.p.height).toBeLessThanOrEqual(parseFloat(card.info.p.lineHeight)+0.5);
|
||||
expect(card.info.height).toBeLessThan(reference.cards[0].info.height);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
import { writeFile } from 'node:fs/promises';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { captureCanonicalState } from './history-canonical-comparison.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
import { parityPng } from './local-inspect-files.mjs';
|
||||
|
||||
// PR L2 Task 7 of the local-Inspect plan (#249): the native "Inspect a file" page against the immutable concept's
|
||||
// `#metadata` (docs/concepts/workspace/app.js renderMetadata 740–748, selection 1715–1729, style.css 709–722,
|
||||
// 1180–1194, 1543–1545, 1559–1564), at 1600x1100 and 390x844 in both themes, in three states: initial, selected (the
|
||||
// same synthetic PNG set into both `#local-file` inputs) and viewer-open. The concept is never edited: the recorded
|
||||
// differences D60–D66 (TESTING.md) are applied to its measurement, and, where they change copy, to its page before it
|
||||
// is measured, so the browser lays the concept out with production's words. After that the diff must be empty.
|
||||
// Paired captures are artifacts only; there are no PNG baselines.
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC', reducedMotion: 'reduce' });
|
||||
|
||||
const FILE = 'f02.png';
|
||||
/** D60 and D61, the maintainer's LI-3 copy, keyed by the concept's own copy. */
|
||||
const COPY = {
|
||||
action: ['Open sample metadata', 'Open file metadata'],
|
||||
intro: ['Keep the existing local-file inspection workflow. Only add the file to your library when you choose to import it.',
|
||||
'Read embedded metadata in this browser. The file is not uploaded or added to your library.'],
|
||||
chooser: ['Choose an image for a local preview', 'Choose an image or video for a local preview'],
|
||||
note: ['This mock previews your file locally. Sample metadata is available with “Open sample metadata”.',
|
||||
'Your file stays in this browser. Its metadata opens with “Open file metadata”.'],
|
||||
toast: ['Local preview only. No file uploaded; sample metadata is available above.',
|
||||
'Local preview only. No file uploaded; metadata is available above.'],
|
||||
};
|
||||
/** D62: the concept's image list plus the videos the local readers support. */
|
||||
const ACCEPT = ['image/png,image/jpeg,image/webp', 'image/png,image/jpeg,image/webp,video/*,.mp4,.mov,.m4v,.webm,.mkv,.avi,.wmv'];
|
||||
|
||||
test.beforeEach(async ({ page, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
page.conceptErrors = [];
|
||||
page.on('pageerror', error => page.conceptErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.conceptErrors).toEqual([]);
|
||||
});
|
||||
|
||||
/** Runs in the page: the Inspect surface below the heading, and the heading's actions. Self-contained. */
|
||||
function measureInspect(scope) {
|
||||
const main = document.querySelector(scope);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const font = node => css(node, 'fontFamily', 'fontSize', 'fontWeight', 'lineHeight', 'color');
|
||||
const origin = main.querySelector(':scope > section.surface');
|
||||
const box = node => {
|
||||
if (!node || !origin) return null;
|
||||
const a = node.getBoundingClientRect(), b = origin.getBoundingClientRect();
|
||||
return { left: Math.round(a.left - b.left), top: Math.round(a.top - b.top), width: Math.round(a.width), height: Math.round(a.height) };
|
||||
};
|
||||
const text = node => node ? node.textContent.replace(/\s+/g, ' ').trim() : null;
|
||||
const heading = main.querySelector(':scope > .heading');
|
||||
const surface = origin;
|
||||
const drop = surface?.querySelector(':scope > .file-drop');
|
||||
const input = drop?.querySelector('input#local-file');
|
||||
const preview = surface?.querySelector(':scope > #local-preview');
|
||||
const media = preview?.querySelector('.local-file-preview');
|
||||
return {
|
||||
heading: {
|
||||
title: text(heading?.querySelector('h1')),
|
||||
description: text(heading?.querySelector('p')),
|
||||
actions: [...(heading?.querySelectorAll('.heading-actions > *') ?? [])].map(node => ({ name: name(node), text: text(node),
|
||||
...css(node, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopLeftRadius'), box: box(node) })),
|
||||
},
|
||||
surface: surface && {
|
||||
children: [...surface.children].map(name), box: box(surface),
|
||||
...css(surface, 'paddingTop', 'paddingRight', 'paddingBottom', 'paddingLeft', 'borderTopWidth', 'borderTopStyle',
|
||||
'borderTopColor', 'borderTopLeftRadius', 'backgroundColor'),
|
||||
h2: { text: text(surface.querySelector(':scope > h2')), ...font(surface.querySelector(':scope > h2')), box: box(surface.querySelector(':scope > h2')) },
|
||||
p: { text: text(surface.querySelector(':scope > p')), ...font(surface.querySelector(':scope > p')),
|
||||
...css(surface.querySelector(':scope > p'), 'marginTop'), box: box(surface.querySelector(':scope > p')) },
|
||||
},
|
||||
drop: drop && {
|
||||
children: [...drop.children].map(name), box: box(drop),
|
||||
...css(drop, 'display', 'placeItems', 'textAlign', 'minHeight', 'rowGap', 'columnGap', 'marginTop', 'paddingTop',
|
||||
'paddingRight', 'paddingBottom', 'paddingLeft', 'borderTopWidth', 'borderTopStyle', 'borderTopColor',
|
||||
'borderBottomStyle', 'borderLeftStyle', 'borderRightStyle', 'borderTopLeftRadius', 'backgroundColor'),
|
||||
icon: { box: box(drop.querySelector(':scope > svg.icon')), viewBox: drop.querySelector(':scope > svg.icon')?.getAttribute('viewBox'),
|
||||
d: drop.querySelector(':scope > svg.icon path')?.getAttribute('d') ?? null,
|
||||
...css(drop.querySelector(':scope > svg.icon'), 'color', 'strokeWidth') },
|
||||
label: { text: text(drop.querySelector(':scope > label')), for: drop.querySelector(':scope > label')?.htmlFor ?? null,
|
||||
...font(drop.querySelector(':scope > label')), box: box(drop.querySelector(':scope > label')) },
|
||||
input: input && { type: input.type, accept: input.accept, multiple: input.multiple, ...css(input, 'maxWidth', 'fontSize', 'color'),
|
||||
box: box(input) },
|
||||
note: { text: text(drop.querySelector(':scope > span.small')), ...font(drop.querySelector(':scope > span.small')),
|
||||
box: box(drop.querySelector(':scope > span.small')) },
|
||||
},
|
||||
preview: preview && {
|
||||
children: [...preview.children].map(name), box: box(preview),
|
||||
media: media && { tag: media.localName, alt: media.getAttribute('alt'), blob: media.src.startsWith('blob:'),
|
||||
...css(media, 'width', 'maxHeight', 'objectFit', 'marginTop', 'display'), box: box(media) },
|
||||
},
|
||||
// The chooser's displayed name: a file input shows the last path segment of its value.
|
||||
chosen: input ? { name: input.value.split(/[\\/]/).pop(), files: [...input.files].map(file => file.name) } : null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Runs in the page: the open viewer's dialog box, its toolbar controls in order and its rail's sections in order. */
|
||||
function measureViewer() {
|
||||
const dialog = document.querySelector('dialog#viewer-dialog[open]');
|
||||
if (!dialog) return null;
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const rect = node => { const b = node.getBoundingClientRect(); return { left: Math.round(b.left), top: Math.round(b.top), width: Math.round(b.width), height: Math.round(b.height) }; };
|
||||
const text = node => node.textContent.replace(/\s+/g, ' ').trim();
|
||||
const rail = dialog.querySelector('#viewer-details');
|
||||
// The rail's sections in document order, whatever wraps them.
|
||||
const SECTIONS = [
|
||||
['.panel-heading', () => 'heading'],
|
||||
['.panel-heading + p.small', () => 'filename'],
|
||||
['.tags', node => node.querySelector('button.chip') ? 'tag chips' : 'badges'],
|
||||
['section.prompt-section.positive', () => 'positive'],
|
||||
['section.prompt-section.negative', () => 'negative'],
|
||||
['dl.meta-grid', node => node.closest('details') ? null : 'facts'],
|
||||
['details > summary', node => text(node)],
|
||||
['button.send', node => node.closest('.prompt-section') ? `send ${node.closest('.prompt-section').matches('.positive') ? 'positive' : 'negative'}` : 'send pair'],
|
||||
['button', node => /Prompt history/.test(node.textContent) ? 'Prompt history' : text(node) === 'Copy settings summary' ? 'settings summary' : null],
|
||||
];
|
||||
const sections = [...rail.querySelectorAll('*')].map(node => {
|
||||
for (const [selector, label] of SECTIONS) if (node.matches(selector)) return label(node);
|
||||
return null;
|
||||
}).filter(Boolean);
|
||||
return {
|
||||
dialog: rect(dialog),
|
||||
// The layout below the header; the header's height follows the title (D66) and the rail's height its content.
|
||||
layout: (() => {
|
||||
const header = dialog.querySelector('.dialog-header').getBoundingClientRect().bottom;
|
||||
const left = rect(dialog.querySelector('.viewer-left')), side = rect(rail);
|
||||
return { left: { ...left, top: Math.round(left.top - header) }, rail: { left: side.left, width: side.width },
|
||||
...css(dialog.querySelector('.viewer-layout'), 'display', 'gridTemplateColumns') };
|
||||
})(),
|
||||
title: text(dialog.querySelector('#viewer-title')),
|
||||
subtitle: text(dialog.querySelector('.dialog-header p')),
|
||||
toolbar: [...dialog.querySelectorAll('#viewer-toolbar > button, #viewer-toolbar > a')].map(text),
|
||||
railTitle: text(rail.querySelector('.panel-heading h2')),
|
||||
sections,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* D60–D62 applied to the open concept page before it is measured: production's action label, copy and accept list,
|
||||
* so the browser lays out what follows them. The concept file itself is never touched.
|
||||
*/
|
||||
async function withProductionCopy(page) {
|
||||
await page.evaluate(({ COPY, ACCEPT }) => {
|
||||
const main = document.querySelector('#main');
|
||||
const swap = (node, [from, to]) => { if (node.textContent !== from) throw new Error(`concept copy changed: ${node.textContent}`); node.textContent = to; };
|
||||
swap(main.querySelector('.heading-actions > button'), COPY.action);
|
||||
swap(main.querySelector('section.surface > p'), COPY.intro);
|
||||
swap(main.querySelector('.file-drop > label'), COPY.chooser);
|
||||
swap(main.querySelector('.file-drop > span.small'), COPY.note);
|
||||
const input = main.querySelector('#local-file');
|
||||
if (input.accept !== ACCEPT[0]) throw new Error(`concept accept changed: ${input.accept}`);
|
||||
input.accept = ACCEPT[1];
|
||||
}, { COPY, ACCEPT });
|
||||
}
|
||||
|
||||
/**
|
||||
* D66: the viewer of a local file is titled by its filename and has no tag chips and no Prompt history. D117 (LI-4):
|
||||
* "Copy settings summary" follows the six facts.
|
||||
*/
|
||||
function mapViewerConcept(reference) {
|
||||
const sections = reference.sections.filter(section => section !== 'tag chips' && section !== 'Prompt history');
|
||||
const facts = sections.indexOf('facts');
|
||||
return {
|
||||
...reference,
|
||||
title: FILE,
|
||||
railTitle: FILE,
|
||||
sections: [...sections.slice(0, facts + 1), 'settings summary', ...sections.slice(facts + 1)],
|
||||
};
|
||||
}
|
||||
|
||||
// Two frames, then every running transition finished.
|
||||
const settle = page => page.evaluate(async () => {
|
||||
await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)));
|
||||
await Promise.all(document.getAnimations().map(animation => animation.finished.catch(() => {})));
|
||||
});
|
||||
|
||||
async function conceptStates(page, theme, testInfo, tag) {
|
||||
await openConcept(page, 'metadata', theme);
|
||||
await page.mouse.move(0, 0);
|
||||
await withProductionCopy(page);
|
||||
await settle(page);
|
||||
const initial = await page.evaluate(measureInspect, '#main');
|
||||
await captureCanonicalState(page, testInfo, `concept-initial-${tag}`);
|
||||
await page.locator('#main #local-file').setInputFiles(parityPng(FILE));
|
||||
await expect(page.locator('#main #local-preview img.local-file-preview')).toBeVisible();
|
||||
await page.locator('#main #local-preview img').evaluate(image => image.decode().catch(() => {}));
|
||||
await expect(page.locator('#toast')).toHaveText(COPY.toast[0]);
|
||||
await settle(page);
|
||||
const selected = await page.evaluate(measureInspect, '#main');
|
||||
await captureCanonicalState(page, testInfo, `concept-selected-${tag}`);
|
||||
await page.locator('#main .heading-actions > button').click();
|
||||
await expect(page.locator('dialog#viewer-dialog[open] #viewer-details .panel-heading')).toBeVisible();
|
||||
await page.mouse.move(0, 0);
|
||||
await settle(page);
|
||||
const viewer = await page.evaluate(measureViewer);
|
||||
await captureCanonicalState(page, testInfo, `concept-viewer-${tag}`);
|
||||
return { initial, selected, viewer };
|
||||
}
|
||||
|
||||
async function productionStates(page, theme, testInfo, tag) {
|
||||
await openProduction(page, 'inspect', theme);
|
||||
await page.mouse.move(0, 0);
|
||||
const action = page.locator('#workspace-main .heading-actions').getByRole('button', { name: COPY.action[1], exact: true });
|
||||
// D60: unavailable, and described by the drop note, until a classified file is chosen.
|
||||
await expect(action).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(action).toHaveAttribute('aria-describedby', 'local-file-note');
|
||||
await settle(page);
|
||||
const initial = await page.evaluate(measureInspect, '#workspace-main');
|
||||
await captureCanonicalState(page, testInfo, `production-initial-${tag}`);
|
||||
await page.locator('#workspace-main #local-file').setInputFiles(parityPng(FILE));
|
||||
await expect(page.locator('#workspace-main #local-preview img.local-file-preview')).toBeVisible();
|
||||
await page.locator('#workspace-main #local-preview img').evaluate(image => image.decode().catch(() => {}));
|
||||
await expect(page.locator('#workspace-toast')).toHaveText(COPY.toast[1]);
|
||||
await expect(action).toHaveAttribute('aria-disabled', 'false');
|
||||
// D63: a readable PNG shows no status line.
|
||||
await expect(page.locator('#workspace-main #local-preview p.small')).toHaveCount(0);
|
||||
await settle(page);
|
||||
const selected = await page.evaluate(measureInspect, '#workspace-main');
|
||||
await captureCanonicalState(page, testInfo, `production-selected-${tag}`);
|
||||
await action.click();
|
||||
await expect(page.locator('dialog#viewer-dialog[open] #viewer-details .panel-heading')).toBeVisible();
|
||||
await expect(page.locator('dialog#viewer-dialog[open] #viewer-details .meta-grid').first()).toBeVisible();
|
||||
await page.mouse.move(0, 0);
|
||||
await settle(page);
|
||||
const viewer = await page.evaluate(measureViewer);
|
||||
await captureCanonicalState(page, testInfo, `production-viewer-${tag}`);
|
||||
return { initial, selected, viewer };
|
||||
}
|
||||
|
||||
/** The concept's fixed CSS values (style.css 709–722, 1180–1194, 1543–1545, 1559–1564), restated as the plan pins them. */
|
||||
function expectPinnedValues(state) {
|
||||
expect(state.surface).toMatchObject({ paddingTop: '22px', paddingLeft: '22px', borderTopLeftRadius: '7px', borderTopStyle: 'solid' });
|
||||
expect(state.surface.h2).toMatchObject({ fontSize: '17px', fontWeight: '600' });
|
||||
expect(state.surface.p).toMatchObject({ fontSize: '12px' });
|
||||
expect(state.drop).toMatchObject({ display: 'grid', textAlign: 'center', minHeight: '170px', rowGap: '12px', columnGap: '12px',
|
||||
marginTop: '20px', paddingTop: '22px', paddingLeft: '22px', borderTopStyle: 'dashed', borderBottomStyle: 'dashed',
|
||||
borderLeftStyle: 'dashed', borderRightStyle: 'dashed', borderTopWidth: '1px', borderTopLeftRadius: '7px' });
|
||||
expect(state.drop.input).toMatchObject({ maxWidth: '100%', fontSize: '12px' });
|
||||
}
|
||||
|
||||
for (const [width, height] of SIZES) for (const theme of THEMES) {
|
||||
test(`inspect surface matches the concept ${width}x${height} ${theme}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
await seedConcept(request);
|
||||
const tag = `${theme}-${width}`;
|
||||
const concept = await conceptStates(page, theme, testInfo, tag);
|
||||
const production = await productionStates(page, theme, testInfo, tag);
|
||||
const mappedViewer = mapViewerConcept(concept.viewer);
|
||||
const diff = {
|
||||
initial: diffInventory(concept.initial, production.initial),
|
||||
selected: diffInventory(concept.selected, production.selected),
|
||||
viewer: diffInventory(mappedViewer, production.viewer),
|
||||
};
|
||||
const path = testInfo.outputPath('inspect-geometry.json');
|
||||
await writeFile(path, JSON.stringify({ concept, production, diff }, null, 2));
|
||||
await testInfo.attach('inspect-geometry', { path, contentType: 'application/json' });
|
||||
|
||||
// The concept is what this comparison assumes: its structure, its own copy (swapped above) and its chooser.
|
||||
expect(concept.initial.surface.children).toEqual(['h2', 'p', 'div.file-drop.spaced-20', 'div']);
|
||||
expect(concept.initial.drop.children).toEqual(['svg.icon', 'label', 'input', 'span.small']);
|
||||
expect(concept.selected.preview.children).toEqual(['img.local-file-preview']);
|
||||
expect(concept.selected.chosen).toEqual({ name: FILE, files: [FILE] });
|
||||
expect(concept.viewer.toolbar).toEqual(['← Previous', '−', '+', 'Fit', 'Rotate', 'Flip', 'Play', 'Download', 'Next →']);
|
||||
|
||||
expect(diff).toEqual({ initial: [], selected: [], viewer: [] });
|
||||
for (const state of [production.initial, production.selected]) expectPinnedValues(state);
|
||||
expect(production.selected.preview.media).toMatchObject({ tag: 'img', alt: 'Local file preview', blob: true,
|
||||
maxHeight: '400px', objectFit: 'contain', marginTop: '20px' });
|
||||
// The chooser names the chosen file after a real setInputFiles, as the concept's does.
|
||||
expect(production.initial.chosen).toEqual({ name: '', files: [] });
|
||||
expect(production.selected.chosen).toEqual({ name: FILE, files: [FILE] });
|
||||
// D66 in the rail: no tag chips and no Prompt history; the title is the filename. D117 follows the facts.
|
||||
expect(production.viewer.sections).toEqual(['heading', 'filename', 'badges', 'positive', 'send positive', 'negative',
|
||||
'send negative', 'facts', 'settings summary', 'File & provenance', 'Workflow & raw metadata', 'send pair']);
|
||||
});
|
||||
}
|
||||
|
||||
test('mutation is noticed', async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await seedConcept(request);
|
||||
await openConcept(page, 'metadata', 'dark');
|
||||
await withProductionCopy(page);
|
||||
const concept = await page.evaluate(measureInspect, '#main');
|
||||
await openProduction(page, 'inspect', 'dark');
|
||||
await page.mouse.move(0, 0);
|
||||
expect(diffInventory(concept, await page.evaluate(measureInspect, '#workspace-main'))).toEqual([]);
|
||||
|
||||
// A solid border on the drop area is noticed on every side and nowhere else.
|
||||
const undoCss = await mutate(page, { css: '#workspace-main .file-drop { border-style: solid; }' });
|
||||
expect(diffInventory(concept, await page.evaluate(measureInspect, '#workspace-main')).map(item => item.path))
|
||||
.toEqual(['drop.borderTopStyle', 'drop.borderBottomStyle', 'drop.borderLeftStyle', 'drop.borderRightStyle']);
|
||||
await undoCss();
|
||||
// A renamed label is noticed (its text; its box only if the new words lay out differently).
|
||||
const label = page.locator('#workspace-main .file-drop > label');
|
||||
await label.evaluate(node => { node.dataset.original = node.textContent; node.textContent = 'Choose a file'; });
|
||||
const renamed = diffInventory(concept, await page.evaluate(measureInspect, '#workspace-main')).map(item => item.path);
|
||||
expect(renamed).toContain('drop.label.text');
|
||||
expect(renamed.every(path => path.startsWith('drop.label.'))).toBe(true);
|
||||
await label.evaluate(node => { node.textContent = node.dataset.original; delete node.dataset.original; });
|
||||
// A removed note is noticed by its path.
|
||||
const undoDom = await mutate(page, { remove: ['#workspace-main .file-drop > span.small'] });
|
||||
expect(diffInventory(concept, await page.evaluate(measureInspect, '#workspace-main')).map(item => item.path))
|
||||
.toEqual(expect.arrayContaining(['drop.children.3', 'drop.note.text']));
|
||||
await undoDom();
|
||||
expect(diffInventory(concept, await page.evaluate(measureInspect, '#workspace-main'))).toEqual([]);
|
||||
await captureCanonicalState(page, testInfo, 'production-after-mutations');
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,seedConcept,openConcept,openProduction,inspectorOf,diffInventory,mutate,countCopy} from './concept-inventory.mjs';
|
||||
import {SIZES,THEMES,seedConcept,openConcept,openProduction,inspectorOf,diffInventory,mutate,countCopy,saveImageTags} from './concept-inventory.mjs';
|
||||
|
||||
// Task D4 of #249: the Images side panel (app.js imageDetails a400–414) and the viewer's full details (imageDetails(i,
|
||||
// true)) against the immutable concept, on the concept-shaped real fixture, at 1600x1100 and 390x844 in both themes.
|
||||
@@ -45,10 +45,10 @@ function withCategoryScope(inventory) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Recorded inspector deviations only (spec §8, TESTING.md D3, D20–D22, D34 and D38); everything else must equal
|
||||
* production. `reference.viewerPlain` is the concept's own full details
|
||||
* with its tag chips made plain (D20) and, under a coarse pointer, D34's targets, both at runtime: the browser lays out
|
||||
* what follows them, as the Images spec's removed toolbar items do.
|
||||
* Recorded inspector deviations only (spec §8, TESTING.md D3, D20–D22, D34, D38 and D117); everything else must
|
||||
* equal production. `reference.viewerPlain` is the concept's own full details
|
||||
* with its tag chips made plain (D20), D117's summary button added and, under a coarse pointer, D34's targets, all at
|
||||
* runtime: the browser lays out what follows them, as the Images spec's removed toolbar items do.
|
||||
*/
|
||||
export function mapInspectorConcept(reference,{viewport}) {
|
||||
const {panel,viewerPlain,fileControls}=reference;
|
||||
@@ -105,7 +105,11 @@ async function openProductionPanel(page,theme) {
|
||||
await settle(page);
|
||||
}
|
||||
|
||||
/** The concept's tag chips in the open viewer as plain `span.chip`s (D20) and D34's coarse targets; returns an undo. */
|
||||
/**
|
||||
* The concept's tag chips in the open viewer as plain `span.chip`s (D20), D34's coarse targets and D117's "Copy
|
||||
* settings summary" after the six facts (LI-4), drawn as the concept's own plain button (its "Download sample JSON",
|
||||
* without its action); returns an undo.
|
||||
*/
|
||||
async function plainViewerChips(page) {
|
||||
const restoreTargets=await mutate(page,{css:D34_COARSE_TARGETS});
|
||||
await page.evaluate(()=>{
|
||||
@@ -113,10 +117,15 @@ async function plainViewerChips(page) {
|
||||
const span=document.createElement('span');span.className='chip';span.textContent=button.textContent;
|
||||
button.replaceWith(span);return [span,button];
|
||||
});
|
||||
const summary=document.querySelector('#viewer-details [data-action="download-workflow"]').cloneNode(false);
|
||||
for(const key of Object.keys(summary.dataset))delete summary.dataset[key];
|
||||
summary.textContent='Copy settings summary';summary.classList.add('settings-summary');
|
||||
document.querySelector('#viewer-details > div > dl.meta-grid').after(summary);window.__summaryButton=summary;
|
||||
});
|
||||
await settle(page);
|
||||
return async()=>{
|
||||
await page.evaluate(()=>{for(const [span,button] of window.__plainChips)span.replaceWith(button);delete window.__plainChips;});
|
||||
await page.evaluate(()=>{for(const [span,button] of window.__plainChips)span.replaceWith(button);delete window.__plainChips;
|
||||
window.__summaryButton.remove();delete window.__summaryButton;});
|
||||
await restoreTargets();
|
||||
};
|
||||
}
|
||||
@@ -319,9 +328,30 @@ test('a recipe-linked image without recipe metadata shows its positive prompt me
|
||||
await expect(panelOf(page).locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Prompt history']);
|
||||
});
|
||||
|
||||
// Task C3 (D88, #249 PR C): the concept's chip row is the image's own tags (`i.tags`, app.js a402). With those names
|
||||
// saved as the file's current image tags, production draws the row from them (Tags filter buttons in the panel, plain
|
||||
// in the viewer, the linked positive's equal tags not repeated) and still equals the concept, in both sizes.
|
||||
for(const [width,height] of SIZES)test(`image tags saved on the concept image keep the inspector equal to the concept ${width}x${height} (D88)`,async({page,request},testInfo)=>{
|
||||
test.setTimeout(120_000);
|
||||
const viewport={width,height};await page.setViewportSize(viewport);
|
||||
await openConceptPanel(page,'dark');
|
||||
const names=await panelOf(page).locator('.detail-inner > .tags').last().locator('button.chip').allTextContents();
|
||||
expect(names).toEqual(['cinematic','blue_hour','architecture']);
|
||||
await captureCanonicalState(page,testInfo,`concept-image-tags-${width}`,'#image-inspector');
|
||||
const reference=await inventory(page,true);
|
||||
const {path}=testInfo.manifest.images.find(item=>item.filename==='alpine_00001.png');
|
||||
expect(await saveImageTags(request,path,names)).toEqual(names);
|
||||
const read=page.waitForResponse(item=>new URL(item.url()).pathname===`/prompt_manager/images/inspect/${path}`);
|
||||
await openProductionPanel(page,'dark');
|
||||
expect((await (await read).json()).image_tags).toEqual({state:'current',tags:names});
|
||||
await expect(panelOf(page).locator('.detail-inner > .tags').last().locator('.chip')).toHaveText(names);
|
||||
await captureCanonicalState(page,testInfo,`production-image-tags-${width}`,'#image-inspector');
|
||||
assertInspectorMatch(reference,await inventory(page),viewport);
|
||||
});
|
||||
|
||||
// The comparison is only worth something if it notices a change (spec §7 mutation check): on the panel one CSS
|
||||
// mutation (the role label's letter spacing) and one DOM mutation (the footer removed); in the viewer (final review m5)
|
||||
// its pair actions removed and D38's Show in folder removed; each applied after an exact match.
|
||||
// its pair actions removed, D38's Show in folder removed and D117's summary removed; each applied after an exact match.
|
||||
test('inspector comparison detects mutations',async({page})=>{
|
||||
test.setTimeout(120_000);
|
||||
const viewport={width:1600,height:1100};
|
||||
@@ -345,6 +375,9 @@ test('inspector comparison detects mutations',async({page})=>{
|
||||
expect(()=>assertInspectorMatch(reference,withoutActions,viewport)).toThrow(/viewerDetails\.actions/);
|
||||
const withoutReveal=await viewerMutated({remove:['#viewer-details button[aria-label$=" in folder"]']});
|
||||
expect(()=>assertInspectorMatch(reference,withoutReveal,viewport)).toThrow(/fileControls/);
|
||||
// D117: the summary action after the facts is part of the mapped concept, so losing it is a difference.
|
||||
const withoutSummary=await viewerMutated({remove:['#viewer-details > div > button']});
|
||||
expect(()=>assertInspectorMatch(reference,withoutSummary,viewport)).toThrow(/viewerDetails\.structure\.inner/);
|
||||
assertInspectorMatch(reference,await viewerMutated(),viewport);
|
||||
|
||||
await mutate(page,{remove:['#image-inspector > .detail-footer']});
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow} from './concept-inventory.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConceptAt,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,countCopy,reflow} from './concept-inventory.mjs';
|
||||
|
||||
// Task E3 of #249: the production Prompt library shelf against the immutable concept (app.js renderLibrary a572–586,
|
||||
// style.css s648–708), on the concept-shaped real fixture (the concept's six prompts with their negatives, categories,
|
||||
@@ -37,7 +37,8 @@ export function mapLibraryConcept(reference) {
|
||||
// D43: tag chips come in the store's alphabetical order; the row flows again from the concept's own chips.
|
||||
tags:sortedTags(card.tags),
|
||||
// D23: the stars are aria-hidden, with a visually hidden "Rated N out of 5" spoken in their place. The fixture's
|
||||
// run_count (4) and last_used_at (now) give the concept's own "4 runs" and "Used today". The two-line clamp is
|
||||
// run_count (4) and last_used_at (its own "now", where the page's clock stands) give the concept's own "4 runs"
|
||||
// and "Used today". The two-line clamp is
|
||||
// applied below through `clampedCards`, measured on the concept with `CLAMP_CSS` (its samples wrap at 390px).
|
||||
footer:{...card.footer,facts:{...card.footer.facts,spoken:card.footer.facts.spoken.replace(/^(★+)/,stars=>`Rated ${stars.length} out of 5`)}}}));
|
||||
return {...reference,library:{...library,toolbar,
|
||||
@@ -238,7 +239,8 @@ async function openHost(context) {
|
||||
}
|
||||
|
||||
test.beforeEach(async({page,request},testInfo)=>{
|
||||
testInfo.manifest=await seedConcept(request);
|
||||
// The page's clock stands at the fixture's own "now", so its last use reads "Used today" on either side of midnight.
|
||||
testInfo.manifest=await seedConceptAt(page,request);
|
||||
// Both documents copy through the clipboard; the stub records what they wrote.
|
||||
await page.addInitScript(()=>{window.copied=[];Object.defineProperty(navigator,'clipboard',{configurable:true,value:{writeText:async text=>{window.copied.push(text);}}});});
|
||||
});
|
||||
@@ -322,6 +324,27 @@ for(const theme of THEMES)test(`library grid and cards match the concept 1800x11
|
||||
expect(compare(reference,actual,{sections:['library'],library:['grid','cards','trailing']})).toEqual([]);
|
||||
});
|
||||
|
||||
// PR #284's CI run: the fixture recorded the last use at 23:59:5x UTC and the shelf rendered at 00:00:05, so production
|
||||
// read "Used yesterday" against the concept's "Used today". Here the browser's wall clock starts at 23:59:58 UTC, the
|
||||
// fixture records its use at that instant, and midnight passes before either document opens.
|
||||
const BEFORE_MIDNIGHT=Date.UTC(2026,9,22,23,59,58);
|
||||
const PAST_MIDNIGHT_MS=5_000;
|
||||
test('a run that crosses midnight still reads the concept fixture\'s day',async({page,request})=>{
|
||||
await page.setViewportSize({width:1800,height:1100});
|
||||
await page.clock.install({time:BEFORE_MIDNIGHT});
|
||||
const manifest=await seedConceptAt(page,request,{now:BEFORE_MIDNIGHT/1000});
|
||||
expect(manifest.now*1000,'the fixture\'s now').toBe(BEFORE_MIDNIGHT);
|
||||
const used=(await (await request.get('/prompt_manager/library')).json()).items.map(item=>Date.parse(item.last_used_at));
|
||||
expect(used,'every last use at the fixture\'s now').toEqual(Array(CONCEPT_PROMPTS).fill(BEFORE_MIDNIGHT));
|
||||
await page.clock.fastForward(PAST_MIDNIGHT_MS);
|
||||
await openConceptLibrary(page,'dark','no-preference');
|
||||
const reference=await conceptInventory(page);
|
||||
await openProductionLibrary(page,'dark','no-preference');
|
||||
const actual=await inventoryOf(page,'#workspace-main');
|
||||
expect(actual.library.cards.map(card=>card.footer.facts.text.replace(/^★+ · /,''))).toEqual(Array(CONCEPT_PROMPTS).fill('Used today'));
|
||||
expect(compare(reference,actual,{sections:['library'],library:['grid','cards','trailing']})).toEqual([]);
|
||||
});
|
||||
|
||||
// The comparison is only worth something if it notices a change (spec §7 mutation check): one CSS mutation (the footer
|
||||
// padding) and one DOM mutation (the negative-variants chip removed), each applied after an exact match.
|
||||
test('library comparison detects mutations',async({page})=>{
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
|
||||
// Artifact-only captures of the editor states that have no concept counterpart (archive confirmation, Show archived,
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
// Files for the Inspect page workflows, as Playwright `setInputFiles` payloads ({name, mimeType, buffer}) or as the
|
||||
// bytes a page builds a File or DataTransfer from. Built from the committed fixtures and the byte-exact PNG builders
|
||||
// the node tests use (tests/js/helpers/png-bytes.mjs), so the browser reads exactly what the unit tests pin.
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import zlib from 'node:zlib';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { raw, ihdr, idat, iend, tEXt } from '../js/helpers/png-bytes.mjs';
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const FIXTURES = path.join(here, '..', 'fixtures');
|
||||
const read = (...parts) => fs.readFileSync(path.join(FIXTURES, ...parts));
|
||||
const file = (name, mimeType, buffer) => ({ name, mimeType, buffer: Buffer.from(buffer) });
|
||||
|
||||
/** One committed parity PNG (tests/fixtures/inspection_parity), e.g. f02.png. */
|
||||
export const parityPng = (name, as = name) => file(as, 'image/png', read('inspection_parity', name));
|
||||
/** One committed local-inspect fixture (tests/fixtures/local-inspect). */
|
||||
export const localFixture = (name, mimeType, as = name) => file(as, mimeType, read('local-inspect', name));
|
||||
/** The committed CC0 VP8 clip with no generation metadata. */
|
||||
export const tinyWebm = () => file('tiny.webm', 'video/webm', read('tiny.webm'));
|
||||
|
||||
/** One decodable 2×2 opaque pixel row set, so a synthetic PNG previews. */
|
||||
function pixels(width = 2, height = 2) {
|
||||
const row = [0, ...Array.from({ length: width }, () => [40, 120, 200, 255]).flat()];
|
||||
return zlib.deflateSync(Buffer.from(Array.from({ length: height }, () => row).flat()));
|
||||
}
|
||||
|
||||
/**
|
||||
* A previewable PNG whose positive spans lines and whose negative is a ConditioningZeroOut: the legacy fill shows the
|
||||
* positive exactly and the ZeroOut guard makes the negative explicitly empty.
|
||||
*/
|
||||
export const POSITIVE_LINES = 'a lighthouse at dusk\nsoft rain, long exposure';
|
||||
export function sendablePng(name = 'send-me.png') {
|
||||
const prompt = JSON.stringify({
|
||||
3: { class_type: 'KSampler', inputs: { seed: 11, steps: 12, cfg: 5, sampler_name: 'euler', scheduler: 'normal', denoise: 1, model: ['4', 0], positive: ['6', 0], negative: ['8', 0], latent_image: ['5', 0] } },
|
||||
4: { class_type: 'CheckpointLoaderSimple', inputs: { ckpt_name: 'local.safetensors' } },
|
||||
5: { class_type: 'EmptyLatentImage', inputs: { width: 512, height: 512, batch_size: 1 } },
|
||||
6: { class_type: 'CLIPTextEncode', inputs: { text: POSITIVE_LINES, clip: ['4', 1] } },
|
||||
8: { class_type: 'ConditioningZeroOut', inputs: { conditioning: ['6', 0] } },
|
||||
});
|
||||
return file(name, 'image/png', raw(ihdr(2, 2), idat(pixels()), tEXt('prompt', prompt), iend()));
|
||||
}
|
||||
|
||||
/** A PNG whose text chunk fails its CRC: the reader reports the metadata unreadable; the bytes stay downloadable. */
|
||||
export const corruptPng = (name = 'corrupt.png') =>
|
||||
file(name, 'image/png', raw(ihdr(2, 2), idat(pixels()), tEXt('prompt', '{"3":{}}', { crc: 1 }), iend()));
|
||||
|
||||
/** Bytes no reader claims, under an image or video name. */
|
||||
export const disguised = () => [
|
||||
file('notes.png', 'image/png', Buffer.from('plain text, not an image')),
|
||||
file('page.png', 'image/png', Buffer.from('<!doctype html><title>x</title><p>hi')),
|
||||
file('vector.png', 'image/png', Buffer.from('<svg xmlns="http://www.w3.org/2000/svg" width="4" height="4"/>')),
|
||||
file('stream.mp4', 'video/mp4', Buffer.from('#EXTM3U\n#EXT-X-VERSION:3\n')),
|
||||
];
|
||||
|
||||
/**
|
||||
* core-faststart.mp4 with its video sample entry renamed from `avc1` to `zzzz`: the container and its tags read as
|
||||
* before, but no browser has a decoder for the stream, so the preview fails deterministically on every engine.
|
||||
*/
|
||||
export function undecodableMp4(name = 'undecodable.mp4') {
|
||||
const bytes = Buffer.from(read('local-inspect', 'core-faststart.mp4'));
|
||||
const at = bytes.indexOf(Buffer.from('avc1', 'latin1'), bytes.indexOf(Buffer.from('stsd', 'latin1')));
|
||||
if (at < 0) throw new Error('core-faststart.mp4 has no avc1 sample entry');
|
||||
bytes.write('zzzz', at, 'latin1');
|
||||
return file(name, 'video/mp4', bytes);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import {readFileSync} from 'node:fs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
const pair={id:4,positive_text:'Library positive',negative_text:'Library negative',category:'Portrait',tags:['studio']};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
|
||||
const moduleUrl = '/prompt_manager/js/shared/toolkit/prompt-bridge.js';
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
|
||||
const workspace='/prompt_manager/workspace#/images';
|
||||
const details=page=>page.getByRole('complementary',{name:'Selected image details',exact:true});
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Hold real requests at this test server, so a page can leave while one is in flight.
|
||||
|
||||
A Playwright ``page.route`` hold answers inside the browser's interception layer,
|
||||
which WebKit cancels differently from a request the server has not answered yet.
|
||||
These commands hold the request at the server itself:
|
||||
|
||||
- ``POST /__test__/hold`` with ``{"path": "/prompt_manager/..."}`` holds every
|
||||
later request for that path until it is released. With ``"stage": "body"``
|
||||
the server answers first and holds after sending the headers and the first
|
||||
half of the body, so the browser has a response whose body is still loading.
|
||||
- ``GET /__test__/hold`` answers ``{"held": {path: count}}``: how many requests
|
||||
for each held path have arrived and are waiting.
|
||||
- ``POST /__test__/release`` lets every held request go on, and stops holding.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
HOLDABLE_PREFIX = "/prompt_manager/"
|
||||
|
||||
|
||||
class RequestHolds:
|
||||
def __init__(self) -> None:
|
||||
self._gates: dict[str, asyncio.Event] = {}
|
||||
self._waiting: dict[str, int] = {}
|
||||
self._stages: dict[str, str] = {}
|
||||
|
||||
def hold(self, path: str, stage: str = "request") -> None:
|
||||
self._gates.setdefault(path, asyncio.Event())
|
||||
self._waiting.setdefault(path, 0)
|
||||
self._stages[path] = stage
|
||||
|
||||
def stage(self, path: str) -> str | None:
|
||||
return self._stages.get(path) if path in self._gates else None
|
||||
|
||||
def held(self) -> dict[str, int]:
|
||||
return dict(self._waiting)
|
||||
|
||||
def release(self) -> None:
|
||||
for gate in self._gates.values():
|
||||
gate.set()
|
||||
self._gates.clear()
|
||||
self._waiting.clear()
|
||||
self._stages.clear()
|
||||
|
||||
async def wait(self, path: str) -> None:
|
||||
gate = self._gates.get(path)
|
||||
if gate is None:
|
||||
return
|
||||
self._waiting[path] = self._waiting.get(path, 0) + 1
|
||||
try:
|
||||
await gate.wait()
|
||||
finally:
|
||||
if path in self._waiting:
|
||||
self._waiting[path] -= 1
|
||||
|
||||
|
||||
def register_request_holds(app: web.Application, routes) -> RequestHolds:
|
||||
holds = RequestHolds()
|
||||
|
||||
@web.middleware
|
||||
async def hold_middleware(request: web.Request, handler):
|
||||
stage = holds.stage(request.path)
|
||||
if stage == "request":
|
||||
await holds.wait(request.path)
|
||||
response = await handler(request)
|
||||
if stage != "body" or not isinstance(response, web.Response):
|
||||
return response
|
||||
body = response.body if isinstance(response.body, bytes) else b""
|
||||
streamed = web.StreamResponse(status=response.status, headers=response.headers)
|
||||
streamed.content_length = len(body)
|
||||
await streamed.prepare(request)
|
||||
await streamed.write(body[: len(body) // 2])
|
||||
await holds.wait(request.path)
|
||||
await streamed.write(body[len(body) // 2 :])
|
||||
await streamed.write_eof()
|
||||
return streamed
|
||||
|
||||
app.middlewares.append(hold_middleware)
|
||||
|
||||
@routes.post("/__test__/hold")
|
||||
async def hold(request: web.Request) -> web.Response:
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
path = data.get("path") if isinstance(data, dict) else None
|
||||
stage = data.get("stage", "request") if isinstance(data, dict) else None
|
||||
if (
|
||||
not isinstance(path, str)
|
||||
or not path.startswith(HOLDABLE_PREFIX)
|
||||
or set(data) - {"path", "stage"}
|
||||
or stage not in ("request", "body")
|
||||
):
|
||||
raise web.HTTPBadRequest()
|
||||
holds.hold(path, stage)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.get("/__test__/hold")
|
||||
async def held(request: web.Request) -> web.Response:
|
||||
return web.json_response({"success": True, "held": holds.held()})
|
||||
|
||||
@routes.post("/__test__/release")
|
||||
async def release(request: web.Request) -> web.Response:
|
||||
holds.release()
|
||||
return web.json_response({"success": True})
|
||||
|
||||
return holds
|
||||
@@ -0,0 +1,212 @@
|
||||
// Needs review against the approved concept (#249 PR F, plan Task F3, Mock inventory M4: app.js renderReview
|
||||
// a749–760, style.css s407–415, s796–814 and their breakpoints, screenshot needs-review-tagging.png). Production runs
|
||||
// on the review fixture's concept-shaped queue (workspace_review_fixture.py `concept: true`: the concept's first three
|
||||
// image names and sizes, and the concept's first prompt on the open item), so every remaining difference is a recorded
|
||||
// deviation applied by mapConceptReview (D3, D29, D119) or a test failure. Paired captures of every state are attached
|
||||
// as artifacts; no screenshot baseline is added.
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { openConcept, openProduction, inventoryOf, mutate, saveImageTags, diffInventory, SIZES, THEMES } from './concept-inventory.mjs';
|
||||
import { reviewInventory, mapConceptReview, taggingInventory, mapConceptTagging, diffTaggingInventory } from './tagging-canonical-comparison.mjs';
|
||||
|
||||
const CONCEPT_FILE = 'alpine_00001.png';
|
||||
const CONCEPT_TAGS = ['cinematic', 'blue_hour', 'architecture'];
|
||||
const SURFACE = ['h2', 'p.small', 'div.row-actions.spaced-15', 'div.field', 'section.positive.prompt-section',
|
||||
'section.negative.prompt-section', 'div.dialog-actions'];
|
||||
// The fake model's server default instructions (tests/browser/server.py TestModel.default_prompt), D84.
|
||||
const INSTRUCTIONS = 'Describe the image';
|
||||
|
||||
const panel = page => page.locator('#workspace-main section.review-panel');
|
||||
const tagDialog = page => page.getByRole('dialog', { name: 'Tag image', exact: true });
|
||||
|
||||
async function seed(request, data = {}) {
|
||||
await expect(await request.post('/__test__/review/reset', { data: { items: 3, concept: true, ...data } })).toBeOK();
|
||||
}
|
||||
/** Production's Needs review page with its first item's sampler paths loaded. */
|
||||
async function openReview(page, theme) {
|
||||
await openProduction(page, 'review', theme);
|
||||
await expect(panel(page).getByRole('combobox', { name: 'Sampler context', exact: true })).toBeEnabled();
|
||||
await settle(page);
|
||||
}
|
||||
async function settle(page) {
|
||||
await page.mouse.move(0, 0);
|
||||
await page.evaluate(async () => {
|
||||
await document.fonts.ready;
|
||||
await Promise.all([...document.images].map(image => image.decode().catch(() => {})));
|
||||
});
|
||||
}
|
||||
const conceptInventory = page => page.evaluate(reviewInventory, '#main');
|
||||
const productionInventory = page => page.evaluate(reviewInventory, '#workspace-main');
|
||||
const node = (tree, prefix) => tree.tag?.startsWith(prefix) ? tree : tree.children?.map(child => node(child, prefix)).find(Boolean) ?? null;
|
||||
/** Computed values of CSS tokens, read through a probe element in the open page. */
|
||||
const tokens = (page, declarations) => page.evaluate(text => {
|
||||
const probe = document.createElement('i');
|
||||
probe.style.cssText = text;
|
||||
document.body.append(probe);
|
||||
const style = getComputedStyle(probe), values = [style.borderTopColor, style.backgroundColor];
|
||||
probe.remove();
|
||||
return values;
|
||||
}, declarations);
|
||||
|
||||
async function compareAt(page, request, width, height, theme) {
|
||||
await seed(request);
|
||||
await page.setViewportSize({ width, height });
|
||||
await openConcept(page, 'review', theme);
|
||||
await settle(page);
|
||||
const concept = await conceptInventory(page);
|
||||
await openReview(page, theme);
|
||||
const actual = await productionInventory(page);
|
||||
return { concept, actual, reference: mapConceptReview(concept, actual) };
|
||||
}
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
page.errors = [];
|
||||
page.on('pageerror', error => page.errors.push(error.message));
|
||||
});
|
||||
test.afterEach(async ({ page, request }) => {
|
||||
expect(page.errors).toEqual([]);
|
||||
await request.post('/__test__/jobs/restart');
|
||||
});
|
||||
|
||||
for (const [width, height] of SIZES) for (const theme of THEMES) {
|
||||
test(`concept and production Metadata review match at ${width}x${height} ${theme}`, async ({ page, request }) => {
|
||||
const { actual, reference } = await compareAt(page, request, width, height, theme);
|
||||
expect(diffInventory(reference, actual), `${width}x${height} ${theme}`).toEqual([]);
|
||||
// Pinned concept values (M4), so a mapping mistake on both sides cannot pass unseen.
|
||||
const main = actual.main;
|
||||
expect(main.children.map(child => child.tag)).toEqual(['div.heading', 'div.notice', 'div.gallery-layout']);
|
||||
const notice = node(main, 'div.notice');
|
||||
const [accent, soft] = await tokens(page, 'border-top:1px solid var(--ca-accent);background:var(--soft-blue)');
|
||||
expect([notice.borderTopColor, notice.borderLeftColor, notice.backgroundColor]).toEqual([accent, accent, soft]);
|
||||
expect([notice.borderTopWidth, notice.borderTopLeftRadius, notice.paddingTop, notice.paddingLeft, notice.marginBottom])
|
||||
.toEqual(['1px', '6px', '15px', '18px', '22px']);
|
||||
expect(notice.flexDirection).toBe(width <= 700 ? 'column' : 'row');
|
||||
expect(notice.children.map(child => child.children.map(item => [item.tag, item.text])))
|
||||
.toEqual([[['h2', '3 images need a decision'], ['p', 'Multiple sampler paths were found. The existing image links were preserved. Choose the correct context or leave them unresolved.']]]);
|
||||
const layout = node(main, 'div.gallery-layout');
|
||||
expect(layout.gridTemplateColumns).toBe(width <= 700 ? `${layout.width}px` : `${layout.width - 22 - 310}px 310px`);
|
||||
const surface = node(main, 'section.review-panel');
|
||||
expect(surface.children.map(child => child.tag)).toEqual(SURFACE);
|
||||
expect(surface.children[0].text).toBe('Resolve the association');
|
||||
expect(surface.children[1].text).toBe(CONCEPT_FILE);
|
||||
const actions = surface.children.at(-1);
|
||||
expect(actions.children.map(button => [button.text, button.primary])).toEqual([['Leave for later', false], ['Apply association', true]]);
|
||||
expect(node(surface, 'select').options.at(-1)).toEqual(['Leave unresolved', false]);
|
||||
expect(node(main, 'div.image-grid').children.map(card => node(card, 'h3').text))
|
||||
.toEqual(['alpine_00001.png', 'dunes_00001.png', 'forest_00001.png']);
|
||||
expect(actual.noPageScroll).toBe(true);
|
||||
});
|
||||
}
|
||||
|
||||
test('the gallery layout follows the concept at every breakpoint', async ({ page, request }) => {
|
||||
await seed(request);
|
||||
// 310px beside the queue, 275px and a 15px gap at ≤1250px, 280px at ≤950px, one column at ≤700px.
|
||||
for (const [width, columns, gap] of [[1600, '310px', '22px'], [1200, '275px', '15px'], [900, '280px', '15px'], [390, null, '15px']]) {
|
||||
await page.setViewportSize({ width, height: 1000 });
|
||||
await openConcept(page, 'review', 'dark');
|
||||
const concept = node((await conceptInventory(page)).main, 'div.gallery-layout');
|
||||
await openReview(page, 'dark');
|
||||
const actual = node((await productionInventory(page)).main, 'div.gallery-layout');
|
||||
expect([actual.gridTemplateColumns, actual.columnGap], `${width}px`).toEqual([concept.gridTemplateColumns, concept.columnGap]);
|
||||
expect([actual.gridTemplateColumns.split(' ').at(-1), actual.columnGap], `${width}px`).toEqual([columns ?? `${actual.width}px`, gap]);
|
||||
expect(actual.gridTemplateColumns.split(' ').length, `${width}px`).toBe(width <= 700 ? 1 : 2);
|
||||
}
|
||||
});
|
||||
|
||||
test('SAVED VIEWS rows match the concept with Needs review after Favorites', async ({ page, request }) => {
|
||||
await seed(request);
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await openConcept(page, 'review', 'dark');
|
||||
const concept = (await inventoryOf(page, '#main')).chrome.navs.find(nav => nav.id === 'saved-nav');
|
||||
await openReview(page, 'dark');
|
||||
const actual = (await inventoryOf(page, '#workspace-main')).chrome.navs.find(nav => nav.id === 'saved-nav');
|
||||
// Sample data: the concept stars three images; this queue's fixture stars none. Needs review counts the queue (3).
|
||||
const reference = { ...concept, rows: concept.rows.map(row => row.text === 'Favorites' ? { ...row, count: { ...row.count, text: '0' } } : row) };
|
||||
expect(diffInventory(reference, actual)).toEqual([]);
|
||||
expect(actual.rows.map(row => [row.text, row.count?.text, row.current])).toEqual([['Favorites', '0', null], ['Needs review', '3', 'page']]);
|
||||
// The count's auto start margin follows the row's own text; every other count token is shared.
|
||||
const { text: favoriteCount, marginInlineStart: favoriteMargin, ...favoriteStyle } = actual.rows[0].count;
|
||||
const { text: reviewCount, marginInlineStart: reviewMargin, ...reviewStyle } = actual.rows[1].count;
|
||||
expect(reviewStyle).toEqual(favoriteStyle);
|
||||
expect(actual.rows[1].icon.d).toBe('M4 3h16v18H4zM8 7h8M8 12h8M8 17h5');
|
||||
});
|
||||
|
||||
test('review Tag dialog matches the needs-review-tagging reference', async ({ page, request }) => {
|
||||
await seed(request);
|
||||
await saveImageTags(request, `review/${CONCEPT_FILE}`, CONCEPT_TAGS);
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await openConcept(page, 'review', 'dark');
|
||||
await page.locator('#main section.surface').getByRole('button', { name: 'Tag image', exact: true }).click();
|
||||
await expect(tagDialog(page)).toBeVisible();
|
||||
await settle(page);
|
||||
const opened = await page.locator('dialog#modal').evaluate(taggingInventory);
|
||||
await page.keyboard.press('Escape');
|
||||
await openReview(page, 'dark');
|
||||
await panel(page).getByRole('button', { name: 'Tag image', exact: true }).click();
|
||||
await expect(tagDialog(page).locator('img.tag-image-preview')).toHaveAttribute('src', /\/image-annotations\/capture\//);
|
||||
await expect(tagDialog(page).locator('#manual-image-tags')).toHaveValue(CONCEPT_TAGS.join(', '));
|
||||
await settle(page);
|
||||
const actual = await page.locator('dialog#modal').evaluate(taggingInventory);
|
||||
const numberField = opened.modelPanel.fields.find(field => field.control.id === 'image-general-threshold');
|
||||
const mapped = mapConceptTagging(opened, { instructions: INSTRUCTIONS, chip: opened.heading.size, numberField });
|
||||
// D119: an undecided queue item names no prompt, so the heading and the preview's alt are the file name.
|
||||
const reference = { ...mapped, heading: { ...mapped.heading, h3: { ...mapped.heading.h3, text: CONCEPT_FILE },
|
||||
preview: { ...mapped.heading.preview, alt: CONCEPT_FILE } } };
|
||||
expect(actual.scopeNote.text).toBe('Tags belong to this image. Resolve its prompt association separately in Needs review.');
|
||||
expect(diffTaggingInventory(reference, actual)).toEqual([]);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(tagDialog(page)).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('the comparison notices a dropped notice border and reordered actions', async ({ page, request }) => {
|
||||
const { actual, reference } = await compareAt(page, request, 1600, 1100, 'dark');
|
||||
expect(diffInventory(reference, actual)).toEqual([]);
|
||||
const undoBorder = await mutate(page, { css: '#workspace-main .notice { border: 0 !important; }' });
|
||||
const borderless = await productionInventory(page);
|
||||
expect(diffInventory(reference, borderless).map(item => item.path)).toEqual(expect.arrayContaining([
|
||||
'main.children.1.borderTopWidth', 'main.children.1.borderTopStyle']));
|
||||
await undoBorder();
|
||||
await panel(page).locator('.dialog-actions').evaluate(row => row.prepend(row.lastElementChild));
|
||||
const reordered = await productionInventory(page);
|
||||
expect(diffInventory(reference, reordered).map(item => item.path)).toEqual(expect.arrayContaining([
|
||||
'main.children.2.children.1.children.6.children.0.text', 'main.children.2.children.1.children.6.children.0.primary']));
|
||||
await panel(page).locator('.dialog-actions').evaluate(row => row.prepend(row.lastElementChild));
|
||||
expect(diffInventory(reference, await productionInventory(page))).toEqual([]);
|
||||
});
|
||||
|
||||
// Paired captures for the PR (no baselines): the populated queue beside the concept in both themes and widths, then
|
||||
// the states the concept lacks (conflict, limited item, unavailable file, empty queue) as production captures.
|
||||
test('paired Needs review captures', async ({ page, request }, testInfo) => {
|
||||
test.setTimeout(120_000);
|
||||
const shot = async name => {
|
||||
await settle(page);
|
||||
const file = testInfo.outputPath(`${name}.png`);
|
||||
await page.screenshot({ path: file, fullPage: true });
|
||||
await testInfo.attach(name, { path: file, contentType: 'image/png' });
|
||||
};
|
||||
await seed(request);
|
||||
for (const [width, height] of SIZES) for (const theme of THEMES) {
|
||||
await page.setViewportSize({ width, height });
|
||||
await openConcept(page, 'review', theme);
|
||||
await shot(`concept-review-${width}-${theme}`);
|
||||
await openReview(page, theme);
|
||||
await shot(`production-review-${width}-${theme}`);
|
||||
}
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await seed(request, { items: 0, conflict: true, limited: true, unavailable: true });
|
||||
await openReview(page, 'dark');
|
||||
await panel(page).getByRole('combobox', { name: 'Sampler context', exact: true }).selectOption({ index: 0 });
|
||||
const decided = page.waitForResponse(item => /\/decision$/.test(item.url()));
|
||||
await panel(page).getByRole('button', { name: 'Apply association', exact: true }).click();
|
||||
expect((await decided).status()).toBe(409);
|
||||
await expect(panel(page).locator('.warning-note')).toBeVisible();
|
||||
await shot('production-review-conflict');
|
||||
for (const [index, name] of [[1, 'limited'], [2, 'unavailable']]) {
|
||||
await page.locator('#workspace-main .review-queue article').nth(index).getByRole('button', { name: /^Inspect / }).click();
|
||||
await expect(panel(page).locator('.warning-note')).toBeVisible();
|
||||
await shot(`production-review-${name}`);
|
||||
}
|
||||
await seed(request, { items: 0 });
|
||||
await openProduction(page, 'review', 'dark');
|
||||
await expect(page.locator('#workspace-main .notice h2')).toHaveText('No images need a decision');
|
||||
await shot('production-review-empty');
|
||||
});
|
||||
+180
-9
@@ -1,23 +1,32 @@
|
||||
"""Serve production pages and API routes with disposable state for browser tests."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
|
||||
from aiohttp import web
|
||||
from aiohttp.web_runner import GracefulExit
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
from tests.browser.request_hold import register_request_holds
|
||||
from tests.browser import slow_reset_watch
|
||||
from tests.browser.workspace_activity_fixture import register_activity_commands
|
||||
from tests.browser.workspace_system_fixture import register_system_commands
|
||||
from tests.browser.workspace_concept_fixture import (
|
||||
MAX_EXTRA_IMAGES as CONCEPT_MAX_EXTRA_IMAGES,
|
||||
MAX_SEEDLESS_IMAGES as CONCEPT_MAX_SEEDLESS_IMAGES,
|
||||
MAX_EPOCH_SECONDS as CONCEPT_MAX_EPOCH_SECONDS,
|
||||
seed_workspace_concept,
|
||||
)
|
||||
from tests.browser.workspace_visual_fixture import seed_workspace_visual
|
||||
@@ -31,6 +40,8 @@ from tests.browser.workspace_library_editor_fixture import (
|
||||
register_library_editor_commands,
|
||||
)
|
||||
from tests.browser.workspace_vocabulary_fixture import register_vocabulary_commands
|
||||
from tests.browser.workspace_tagging_fixture import register_tagging_commands
|
||||
from tests.browser.workspace_review_fixture import register_review_commands
|
||||
|
||||
CANONICAL_ROOT = Path(__file__).resolve().parents[2]
|
||||
CANONICAL_ASSETS = {
|
||||
@@ -53,11 +64,19 @@ class TestModel:
|
||||
wd14_character_threshold = 0.85
|
||||
custom_prompt = ""
|
||||
loaded = None
|
||||
# Models the Tag image dialog workflows mark as not downloaded.
|
||||
unavailable: set = set()
|
||||
# Scored names the concept comparisons answer instead of the fixed list
|
||||
# (``/__test__/tagging/suggestions``); None keeps the fixed list.
|
||||
suggestions: list | None = None
|
||||
|
||||
def get_models_status(self) -> dict:
|
||||
return {
|
||||
name: {"downloaded": True, "loaded": self.loaded == name}
|
||||
for name in ("gguf", "wd14-vit", "wd14-convnext", "wd14-swinv2")
|
||||
name: {
|
||||
"downloaded": name not in self.unavailable,
|
||||
"loaded": self.loaded == name,
|
||||
}
|
||||
for name in ("gguf", "hf", "wd14-vit", "wd14-convnext", "wd14-swinv2")
|
||||
}
|
||||
|
||||
def is_model_loaded(self) -> bool:
|
||||
@@ -76,6 +95,26 @@ class TestModel:
|
||||
def generate_tags(self, image_path: str, **kwargs: object) -> list[str]:
|
||||
return ["generated", Path(image_path).parent.parent.name]
|
||||
|
||||
def generate_tag_suggestions(
|
||||
self, image_path: str, *, limit: int, **kwargs: object
|
||||
) -> list[tuple[str, float | None]]:
|
||||
"""Fixed scored suggestions (WD14) or unscored ones (language models)."""
|
||||
scored = self.loaded is not None and self.loaded.startswith("wd14")
|
||||
if self.suggestions is not None:
|
||||
return [
|
||||
(name, confidence if scored else None)
|
||||
for name, confidence in self.suggestions[:limit]
|
||||
]
|
||||
names = ["sandstone", "desert", "arch", "sky", "rock", "dune", "canyon"]
|
||||
names += [f"tag_{index:02d}" for index in range(100)]
|
||||
return [
|
||||
(name, round(0.98 - index * 0.008, 3) if scored else None)
|
||||
for index, name in enumerate(names[:limit])
|
||||
]
|
||||
|
||||
def download_model(self, model_type: str, progress_callback=None) -> bool:
|
||||
raise AssertionError("browser tests never download a model")
|
||||
|
||||
|
||||
REVEAL_OUTCOMES = ("opened", "unavailable", "fail")
|
||||
|
||||
@@ -159,8 +198,13 @@ def create_app(root: Path) -> web.Application:
|
||||
history_commands = register_workspace_history_commands(routes, api, root)
|
||||
library_editor_commands = register_library_editor_commands(routes, api, root)
|
||||
register_vocabulary_commands(routes, api, root)
|
||||
register_tagging_commands(routes, api, root, model)
|
||||
register_review_commands(routes, api, root)
|
||||
reveal_stub = register_reveal_stub(routes, api, root)
|
||||
register_activity_commands(app, routes, api)
|
||||
register_system_commands(app, routes, api, root)
|
||||
request_holds = register_request_holds(app, routes)
|
||||
slow_reset_watch.install(app, api)
|
||||
|
||||
async def canonical_asset(request: web.Request) -> web.Response:
|
||||
filename, content_type = CANONICAL_ASSETS[request.path]
|
||||
@@ -185,6 +229,11 @@ def create_app(root: Path) -> web.Application:
|
||||
for table in (
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
"media_review_decisions",
|
||||
"media_review_items",
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
@@ -337,6 +386,9 @@ def create_app(root: Path) -> web.Application:
|
||||
@routes.post("/__test__/reset")
|
||||
async def reset(request: web.Request) -> web.Response:
|
||||
nonlocal history_fixture
|
||||
request_holds.release()
|
||||
# No preview queued by the last test may land in the new fixture.
|
||||
await api._run_in_executor(api.close_auto_thumbnails)
|
||||
library_editor_commands.reset()
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
if not isinstance(data, dict):
|
||||
@@ -449,7 +501,17 @@ def create_app(root: Path) -> web.Application:
|
||||
if (
|
||||
request.query
|
||||
or not isinstance(data, dict)
|
||||
or set(data) - set(limits)
|
||||
or set(data) - set(limits) - {"short_seedless", "now"}
|
||||
or not isinstance(data.get("short_seedless", False), bool)
|
||||
# A chosen "now" (epoch seconds) pins every time the fixture writes.
|
||||
or (
|
||||
"now" in data
|
||||
and (
|
||||
isinstance(data["now"], bool)
|
||||
or not isinstance(data["now"], (int, float))
|
||||
or not 0 < data["now"] < CONCEPT_MAX_EPOCH_SECONDS
|
||||
)
|
||||
)
|
||||
or any(
|
||||
isinstance(data.get(key, 0), bool)
|
||||
or not isinstance(data.get(key, 0), int)
|
||||
@@ -468,6 +530,8 @@ def create_app(root: Path) -> web.Application:
|
||||
root,
|
||||
extra_images=data.get("extra_images", 0),
|
||||
seedless_images=data.get("seedless_images", 0),
|
||||
short_seedless=data.get("short_seedless", False),
|
||||
now=data.get("now"),
|
||||
)
|
||||
)
|
||||
return web.json_response({"success": True, **manifest})
|
||||
@@ -502,27 +566,134 @@ def create_app(root: Path) -> web.Application:
|
||||
shutil.copyfile(root / "output" / "shared" / "same.png", destination)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/add-local-inspect-file")
|
||||
async def add_local_inspect_file(request: web.Request) -> web.Response:
|
||||
"""Serve one committed Inspect fixture from the output folder.
|
||||
|
||||
The local Inspect page reads a chosen file in the browser; its page
|
||||
workflow compares that rail with the Images viewer for the same bytes.
|
||||
Only a committed fixture name is accepted.
|
||||
"""
|
||||
data = await request.json() if request.can_read_body else None
|
||||
fixtures = CANONICAL_ROOT / "tests" / "fixtures"
|
||||
allowed = {
|
||||
path.name: path
|
||||
for folder in ("inspection_parity", "local-inspect")
|
||||
for path in (fixtures / folder).iterdir()
|
||||
if path.suffix.lower() in {".png", ".jpg", ".webp", ".webm", ".mp4"}
|
||||
}
|
||||
if (
|
||||
not isinstance(data, dict)
|
||||
or set(data) != {"name"}
|
||||
or data["name"] not in allowed
|
||||
):
|
||||
raise web.HTTPBadRequest()
|
||||
destination = root / "output" / "local-inspect" / data["name"]
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(allowed[data["name"]], destination)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.get("/api/settings/Comfy.ColorPalette")
|
||||
async def palette(request: web.Request) -> web.Response:
|
||||
return web.json_response("dark")
|
||||
|
||||
async def cleanup(application: web.Application) -> None:
|
||||
# A scan of an ambiguous fixture file captures it for Needs review.
|
||||
api.close_annotation_captures()
|
||||
api.db.close_all()
|
||||
|
||||
seed()
|
||||
# Open Needs review change streams end first, as on the real host.
|
||||
app.on_shutdown.append(api.close_review_events)
|
||||
app.on_cleanup.append(cleanup)
|
||||
app.add_routes(routes)
|
||||
return app
|
||||
|
||||
|
||||
def main() -> None:
|
||||
HOST = "127.0.0.1"
|
||||
PORT_ANNOUNCEMENT = "PM_TEST_SERVER_PORT="
|
||||
|
||||
|
||||
def bind_listener(port: int) -> socket.socket:
|
||||
"""Bind the server's socket before the app is built.
|
||||
|
||||
``port`` 0 lets the OS choose a free port, which this process then holds,
|
||||
so parallel launchers never race for one between choosing and binding.
|
||||
"""
|
||||
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
if port and os.name != "nt":
|
||||
# aiohttp's own default on POSIX; on Windows it would let a
|
||||
# second process share the port.
|
||||
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
listener.bind((HOST, port))
|
||||
listener.listen(128)
|
||||
except OSError:
|
||||
listener.close()
|
||||
raise
|
||||
return listener
|
||||
|
||||
|
||||
def _graceful_exit() -> None:
|
||||
raise GracefulExit()
|
||||
|
||||
|
||||
def exit_when_stdin_closes(loop: asyncio.AbstractEventLoop) -> threading.Thread:
|
||||
"""Stop the app gracefully once stdin reaches end of file.
|
||||
|
||||
The launcher keeps the write end of this process's stdin pipe, and the OS
|
||||
closes it however the launcher dies, SIGKILL included. GracefulExit runs
|
||||
aiohttp's cleanup (closing the database) and lets the temp root be removed,
|
||||
on every OS, where a signal from a dead parent would not.
|
||||
"""
|
||||
|
||||
def watch() -> None:
|
||||
while sys.stdin.buffer.read(65536):
|
||||
pass
|
||||
loop.call_soon_threadsafe(_graceful_exit)
|
||||
|
||||
thread = threading.Thread(target=watch, name="stdin-eof", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
|
||||
def close_owned_log_files() -> None:
|
||||
"""Close Prompt Manager's own log handlers before the temp root goes.
|
||||
|
||||
The logs live in the library's data folder (#249 S6b), which for this
|
||||
server is <root>/user/__promptmanager; Windows cannot remove a file
|
||||
this process still has open.
|
||||
"""
|
||||
for handler in logging.getLogger("prompt_manager").handlers[:]:
|
||||
if getattr(handler, "_prompt_manager_owned", False) is True:
|
||||
handler.close()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--port", type=int, default=8765)
|
||||
args = parser.parse_args()
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=8765, help="0 binds an OS-chosen free port"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--exit-on-stdin-eof",
|
||||
action="store_true",
|
||||
help="shut down when stdin closes (the launcher's parent-death guard)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
listener = bind_listener(args.port)
|
||||
# One machine-readable line tells the launcher which port this process holds.
|
||||
print(f"{PORT_ANNOUNCEMENT}{listener.getsockname()[1]}", flush=True)
|
||||
loop = asyncio.new_event_loop()
|
||||
if args.exit_on_stdin_eof:
|
||||
exit_when_stdin_closes(loop)
|
||||
original_cwd = os.getcwd()
|
||||
with tempfile.TemporaryDirectory(prefix="pm-browser-") as directory:
|
||||
web.run_app(
|
||||
create_app(Path(directory).resolve()), host="127.0.0.1", port=args.port
|
||||
)
|
||||
try:
|
||||
web.run_app(create_app(Path(directory).resolve()), sock=listener, loop=loop)
|
||||
finally:
|
||||
# create_app works inside the root; Windows cannot remove a process's cwd.
|
||||
os.chdir(original_cwd)
|
||||
close_owned_log_files()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept as openConceptPage,openProduction,inventoryOf,diffInventory,focusRingOf,hoverStateOf,mutate} from './concept-inventory.mjs';
|
||||
|
||||
@@ -68,12 +68,12 @@ function remainingFooter(footer,spans) {
|
||||
const OVERVIEW_GLYPH='M3 11 12 4l9 7M5 10v10h14V10M10 20v-6h4v6';
|
||||
|
||||
/**
|
||||
* The concept's SAVED VIEWS rows (app.js a331–335) as production lists them: Favorites with its count (D51), then the
|
||||
* saved views. D50: no "Needs review" row until the tag-review slice (maintainer ruling 2026-10-21).
|
||||
* The concept's SAVED VIEWS rows (app.js a331–335) as production lists them: Favorites with its count (D51), Needs
|
||||
* review with the exact pending count (ruling R5), then the saved views. The concept's "3" is sample data: the concept
|
||||
* fixture records no review items, so production's count reads "0".
|
||||
*/
|
||||
function savedNav(nav) {
|
||||
const rows=nav.rows.filter(row=>row.text!=='Needs review');
|
||||
return {...nav,rows:rows.map((row,index)=>index===rows.length-1?{...row,gapAfter:null}:row)};
|
||||
return {...nav,rows:nav.rows.map(row=>row.text==='Needs review'?{...row,count:{...row.count,text:'0'}}:row)};
|
||||
}
|
||||
|
||||
/** Recorded shell deviations only (spec §8). Everything else in the concept inventory must equal production. */
|
||||
@@ -119,10 +119,7 @@ function mapShellConcept(reference) {
|
||||
const spans=chrome.footer.spans.map(span=>({...span,text:'ComfyAssets · Local tools. Your library.'}));
|
||||
return {...chrome.footer,spans,buttons:[],height:remainingFooter(chrome.footer,spans)};
|
||||
})(),
|
||||
},
|
||||
heading:reference.heading&&{...reference.heading,actions:reference.heading.actions&&{...reference.heading.actions,
|
||||
// #245 vocabulary ruling D2 (mapVocabularyConcept): Auto-tag images is omitted until the image-annotation slice.
|
||||
buttons:reference.heading.actions.buttons.filter(item=>item.text!=='Auto-tag images')}}};
|
||||
}};
|
||||
}
|
||||
|
||||
/** The heading without its page-owned description copy and actions; the h1 text is the shell's and stays compared. */
|
||||
@@ -181,27 +178,28 @@ async function captureFooter(page,testInfo,name) {
|
||||
|
||||
test.beforeEach(async({request})=>{await seedConcept(request);});
|
||||
|
||||
for(const [width,height] of SIZES)for(const theme of THEMES)test(`shell geometry, tokens and controls match the concept ${width}x${height} ${theme}`,async({page},testInfo)=>{
|
||||
// One test per route, size, theme and motion. All of them in one test per size and theme (12 page loads, 12 captures,
|
||||
// 36 hovers) steadily took 22 s median and up to 33 s on CI WebKit, past the 30 s budget with no step stalled; one
|
||||
// route in one motion is a sixth of that work under the same budget.
|
||||
for(const [route,breadcrumb] of ROUTES)for(const [width,height] of SIZES)for(const theme of THEMES)for(const motion of MOTIONS)test(`shell geometry, tokens and controls match the concept on ${route} ${width}x${height} ${theme} ${motion}`,async({page},testInfo)=>{
|
||||
await page.setViewportSize({width,height});const result={};
|
||||
for(const motion of MOTIONS)for(const [route,breadcrumb] of ROUTES) {
|
||||
await openConcept(page,route,theme,motion);await page.mouse.move(0,0);
|
||||
const reference=await inventoryOf(page,'#main');
|
||||
if(motion==='no-preference')await captureCanonicalState(page,testInfo,`concept-shell-${route}-${width}-${theme}`);
|
||||
const referenceRings=FULL_HEADING.has(route)?await ringsOf(page,'#main'):null;
|
||||
const referenceHovers=await hoversOf(page);
|
||||
if(motion==='no-preference')await captureFooter(page,testInfo,`concept-footer-${route}-${width}-${theme}`);
|
||||
await openProduction(page,route,theme,motion);await page.mouse.move(0,0);
|
||||
const actual=await inventoryOf(page,'#workspace-main');
|
||||
if(motion==='no-preference')await captureCanonicalState(page,testInfo,`production-shell-${route}-${width}-${theme}`);
|
||||
const actualRings=FULL_HEADING.has(route)?await ringsOf(page,'#workspace-main'):null;
|
||||
const actualHovers=await hoversOf(page);
|
||||
if(motion==='no-preference')await captureFooter(page,testInfo,`production-footer-${route}-${width}-${theme}`);
|
||||
result[`${motion} ${route}`]={route,breadcrumb,reference,actual,diff:compare(route,reference,actual),referenceRings,actualRings,
|
||||
referenceHovers,actualHovers};
|
||||
}
|
||||
const path=testInfo.outputPath('shell-geometry.json');await writeFile(path,JSON.stringify(result,null,2));
|
||||
await testInfo.attach('shell-geometry',{path,contentType:'application/json'});
|
||||
for(const [key,{route,breadcrumb,reference,actual,diff,referenceRings,actualRings,referenceHovers,actualHovers}] of Object.entries(result)) {
|
||||
await openConcept(page,route,theme,motion);await page.mouse.move(0,0);
|
||||
const reference=await inventoryOf(page,'#main');
|
||||
if(motion==='no-preference')await captureCanonicalState(page,testInfo,`concept-shell-${route}-${width}-${theme}`);
|
||||
const referenceRings=FULL_HEADING.has(route)?await ringsOf(page,'#main'):null;
|
||||
const referenceHovers=await hoversOf(page);
|
||||
if(motion==='no-preference')await captureFooter(page,testInfo,`concept-footer-${route}-${width}-${theme}`);
|
||||
await openProduction(page,route,theme,motion);await page.mouse.move(0,0);
|
||||
const actual=await inventoryOf(page,'#workspace-main');
|
||||
if(motion==='no-preference')await captureCanonicalState(page,testInfo,`production-shell-${route}-${width}-${theme}`);
|
||||
const actualRings=FULL_HEADING.has(route)?await ringsOf(page,'#workspace-main'):null;
|
||||
const actualHovers=await hoversOf(page);
|
||||
if(motion==='no-preference')await captureFooter(page,testInfo,`production-footer-${route}-${width}-${theme}`);
|
||||
result[`${motion} ${route}`]={reference,actual,diff:compare(route,reference,actual),referenceRings,actualRings,
|
||||
referenceHovers,actualHovers};
|
||||
const path=testInfo.outputPath(`shell-geometry-${route}-${motion}.json`);await writeFile(path,JSON.stringify(result,null,2));
|
||||
await testInfo.attach(`shell-geometry-${route}-${motion}`,{path,contentType:'application/json'});
|
||||
for(const [key,{reference,actual,diff,referenceRings,actualRings,referenceHovers,actualHovers}] of Object.entries(result)) {
|
||||
// The concept itself is what this comparison assumes: its sample counts and its shell controls.
|
||||
expect.soft(reference.chrome.navs[0].rows.map(row=>[row.text,row.count?.text??null]),`${key} concept WORKSPACE rows`)
|
||||
.toEqual([['Images','24'],['Prompt library','6'],['Tags & categories','15'],['Inspect a file',null]]);
|
||||
@@ -245,8 +243,9 @@ for(const [width,height] of SIZES)test(`a saved view row matches the concept's $
|
||||
await openProduction(page,'images','dark');
|
||||
const actual=await save('Save','#workspace-main');
|
||||
expect(diffInventory(mapShellConcept(reference).chrome.navs,actual.chrome.navs)).toEqual([]);
|
||||
const referenceRows=reference.chrome.navs.find(nav=>nav.id==='saved-nav').rows;
|
||||
expect(actual.chrome.navs.find(nav=>nav.id==='saved-nav').rows.map(row=>[row.text,row.count]))
|
||||
.toEqual([['Favorites',reference.chrome.navs.find(nav=>nav.id==='saved-nav').rows[0].count],['Golden hour',null]]);
|
||||
.toEqual([['Favorites',referenceRows[0].count],['Needs review',{...referenceRows[1].count,text:'0'}],['Golden hour',null]]);
|
||||
});
|
||||
|
||||
test('the inventory reports a missing shell element by its path instead of failing',async({page})=>{
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
"""Report a slow test reset with what the server was doing while it waited.
|
||||
|
||||
A reset normally answers within about a second. Once one runs past
|
||||
SLOW_RESET_SECONDS, this writes to stderr, while the reset is still waiting:
|
||||
|
||||
- every thread's stack, so a reset waiting on a lock, a full executor or a
|
||||
slow disk shows where it waits and who holds it;
|
||||
- the default executor's threads and queued work;
|
||||
- how many database connections are open, and the WAL size;
|
||||
- on Linux, each thread's scheduler state and the kernel function it sleeps
|
||||
in (``D`` with a journal or writeback wait is a slow disk flush, a nanosleep
|
||||
is SQLite's busy handler, a futex is a Python lock or an idle worker).
|
||||
|
||||
When the reset ends it writes how long it took and, for each executor call it
|
||||
made, how long the call queued and how long it ran.
|
||||
|
||||
Every report line starts with REPORT_PREFIX. The worker fixture
|
||||
(tools/vendor/start-browser-server.mjs) copies those lines to the job log,
|
||||
so a slow reset is visible even in a test that still passes.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextvars
|
||||
import faulthandler
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from typing import Callable, TextIO
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
SLOW_RESET_SECONDS = 2.0
|
||||
REPORT_PREFIX = "pm-slow-reset| "
|
||||
|
||||
_current: contextvars.ContextVar["ResetWatch | None"] = contextvars.ContextVar(
|
||||
"pm_reset_watch", default=None
|
||||
)
|
||||
|
||||
|
||||
def is_reset_path(path: str) -> bool:
|
||||
return path.startswith("/__test__/") and path.endswith("/reset")
|
||||
|
||||
|
||||
class ResetWatch:
|
||||
"""Time one reset; dump the server's state if it passes the threshold."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
label: str,
|
||||
describe: Callable[[], list[str]],
|
||||
*,
|
||||
threshold: float = SLOW_RESET_SECONDS,
|
||||
stream: TextIO | None = None,
|
||||
) -> None:
|
||||
self.label = label
|
||||
self.threshold = threshold
|
||||
self._describe = describe
|
||||
self._stream = stream
|
||||
self._start = time.monotonic()
|
||||
self._calls: list[str] = []
|
||||
self._calls_lock = threading.Lock()
|
||||
self._stalled = threading.Event()
|
||||
self._timer = threading.Timer(threshold, self._report_stall)
|
||||
self._timer.daemon = True
|
||||
self._timer.name = "pm-slow-reset-watch"
|
||||
|
||||
def start(self) -> "ResetWatch":
|
||||
self._timer.start()
|
||||
return self
|
||||
|
||||
def elapsed(self) -> float:
|
||||
return time.monotonic() - self._start
|
||||
|
||||
def record_call(self, name: str, queued: float, ran: float) -> None:
|
||||
with self._calls_lock:
|
||||
self._calls.append(
|
||||
f"executor call {name}: queued {queued:.3f}s, ran {ran:.3f}s"
|
||||
)
|
||||
|
||||
def finish(self, outcome: str) -> None:
|
||||
self._timer.cancel()
|
||||
elapsed = self.elapsed()
|
||||
# A coarse clock (Windows) can fire the timer just short of the
|
||||
# threshold; a reset that was reported always gets its summary.
|
||||
if elapsed < self.threshold and not self._stalled.is_set():
|
||||
return
|
||||
with self._calls_lock:
|
||||
calls = list(self._calls)
|
||||
self._write([f"{self.label} {outcome} after {elapsed:.3f}s", *calls])
|
||||
|
||||
def _report_stall(self) -> None:
|
||||
self._stalled.set()
|
||||
lines = [f"{self.label} still running after {self.elapsed():.3f}s"]
|
||||
try:
|
||||
lines.extend(self._describe())
|
||||
except Exception as error: # a diagnostic never breaks the reset
|
||||
lines.append(f"state unavailable: {error!r}")
|
||||
lines.extend(thread_stacks())
|
||||
self._write(lines)
|
||||
|
||||
def _write(self, lines: list[str]) -> None:
|
||||
stream = self._stream or sys.stderr
|
||||
text = "".join(
|
||||
f"{REPORT_PREFIX}{part}\n" for line in lines for part in line.splitlines()
|
||||
)
|
||||
try:
|
||||
stream.write(text)
|
||||
stream.flush()
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def thread_stacks() -> list[str]:
|
||||
"""Every thread's stack, from faulthandler.
|
||||
|
||||
Formatting sys._current_frames() showed a thread blocked in SQLite as idle
|
||||
(Python 3.14); faulthandler reports where each thread really is.
|
||||
"""
|
||||
# Before Python 3.14 faulthandler prints ids only; name each id here.
|
||||
names = [
|
||||
f"thread 0x{thread.ident:016x} is {thread.name}"
|
||||
for thread in threading.enumerate()
|
||||
if thread.ident is not None
|
||||
]
|
||||
with tempfile.TemporaryFile() as dump:
|
||||
faulthandler.dump_traceback(file=dump, all_threads=True)
|
||||
dump.seek(0)
|
||||
return [*names, *dump.read().decode("utf-8", "replace").splitlines()]
|
||||
|
||||
|
||||
def executor_state(loop: asyncio.AbstractEventLoop) -> list[str]:
|
||||
executor = getattr(loop, "_default_executor", None)
|
||||
if executor is None:
|
||||
return ["default executor: not started"]
|
||||
threads = getattr(executor, "_threads", ())
|
||||
queue = getattr(executor, "_work_queue", None)
|
||||
queued = queue.qsize() if queue is not None else "?"
|
||||
return [
|
||||
f"default executor: {len(threads)} of "
|
||||
f"{getattr(executor, '_max_workers', '?')} threads, {queued} queued"
|
||||
]
|
||||
|
||||
|
||||
def database_state(model) -> list[str]:
|
||||
"""How many connections are open, and the WAL size.
|
||||
|
||||
Never touches another thread's connection: a connection busy in SQLite
|
||||
holds its mutex, and reading even ``in_transaction`` from here would block
|
||||
on it while holding the GIL, freezing every thread until that query ends.
|
||||
"""
|
||||
with model._conn_lock:
|
||||
count = len(model._connections)
|
||||
wal = model.db_path + "-wal"
|
||||
size = os.path.getsize(wal) if os.path.exists(wal) else 0
|
||||
return [f"database: {count} connections, WAL {size} bytes"]
|
||||
|
||||
|
||||
def kernel_wait_state() -> list[str]:
|
||||
"""Each thread's state and kernel wait channel, where /proc has them."""
|
||||
lines = []
|
||||
for thread in threading.enumerate():
|
||||
task = f"/proc/self/task/{thread.native_id}"
|
||||
try:
|
||||
with open(f"{task}/stat", encoding="ascii") as stat:
|
||||
state = stat.read().rsplit(")", 1)[1].split()[0]
|
||||
with open(f"{task}/wchan", encoding="ascii") as wchan:
|
||||
channel = wchan.read().strip() or "0"
|
||||
except (OSError, IndexError):
|
||||
continue
|
||||
# The id matches the "Thread 0x..." headers of the stacks below.
|
||||
lines.append(
|
||||
f"kernel: {thread.name} (0x{thread.ident:016x}) state {state}, "
|
||||
f"waiting in {channel}"
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def install(app: web.Application, api) -> None:
|
||||
"""Watch every reset route, and time the executor calls each one makes."""
|
||||
run_in_executor = api._run_in_executor
|
||||
|
||||
async def timed_run_in_executor(func, *args, **kwargs):
|
||||
watch = _current.get()
|
||||
if watch is None:
|
||||
return await run_in_executor(func, *args, **kwargs)
|
||||
queued_at = time.monotonic()
|
||||
started: list[float] = []
|
||||
|
||||
def timed(*call_args, **call_kwargs):
|
||||
started.append(time.monotonic())
|
||||
return func(*call_args, **call_kwargs)
|
||||
|
||||
try:
|
||||
return await run_in_executor(timed, *args, **kwargs)
|
||||
finally:
|
||||
end = time.monotonic()
|
||||
begin = started[0] if started else end
|
||||
watch.record_call(
|
||||
getattr(func, "__qualname__", repr(func)),
|
||||
begin - queued_at,
|
||||
end - begin,
|
||||
)
|
||||
|
||||
api._run_in_executor = timed_run_in_executor
|
||||
|
||||
@web.middleware
|
||||
async def watch_resets(request: web.Request, handler):
|
||||
if not is_reset_path(request.path):
|
||||
return await handler(request)
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def describe() -> list[str]:
|
||||
return [
|
||||
*executor_state(loop),
|
||||
*database_state(api.db.model),
|
||||
*kernel_wait_state(),
|
||||
]
|
||||
|
||||
watch = ResetWatch(
|
||||
f"{request.method} {request.path}", describe, threshold=SLOW_RESET_SECONDS
|
||||
).start()
|
||||
token = _current.set(watch)
|
||||
outcome = "failed"
|
||||
try:
|
||||
response = await handler(request)
|
||||
outcome = f"answered {response.status}"
|
||||
return response
|
||||
except web.HTTPException as error:
|
||||
outcome = f"answered {error.status}"
|
||||
raise
|
||||
finally:
|
||||
_current.reset(token)
|
||||
watch.finish(outcome)
|
||||
|
||||
app.middlewares.append(watch_resets)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,291 @@
|
||||
// Concept-vs-production inventory of the Tag image dialog (plan Mock inventory M2: docs/concepts/workspace/app.js
|
||||
// tagReview a1037–1061, renderTagModelOptions a1062–1076, generateTags a1077–1135; style.css s1598–1694) and of the
|
||||
// card's Info and Tag group (M1: app.js mediaCard a398, style.css s1589–1597). The *Of functions run in the page.
|
||||
// The concept is never rewritten: mapConceptTagging applies only the recorded deviations, each named with its D-number
|
||||
// or ruling; everything else must be equal.
|
||||
export {diffInventory as diffTaggingInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
/** The open Tag image dialog (`dialog#modal`), its controls in order with their concept tokens. */
|
||||
export function taggingInventory(dialog) {
|
||||
const css=(node,...keys)=>{const style=getComputedStyle(node);return Object.fromEntries(keys.map(key=>[key,style[key]]));};
|
||||
const px=value=>Math.round(value*2)/2;
|
||||
const box=node=>{const rect=node.getBoundingClientRect();return {width:px(rect.width),height:px(rect.height)};};
|
||||
const font=node=>css(node,'fontSize','fontWeight','lineHeight','color','marginTop','marginBottom');
|
||||
const edges=node=>css(node,'paddingTop','paddingRight','paddingBottom','paddingLeft','borderTopWidth','borderTopStyle','borderTopColor',
|
||||
'borderTopLeftRadius','backgroundColor');
|
||||
const buttonOf=node=>({text:node.textContent.trim(),label:node.getAttribute('aria-label'),primary:node.classList.contains('ca-primary'),
|
||||
icon:node.querySelector('svg')!==null,disabled:node.getAttribute('aria-disabled')==='true'||node.disabled,
|
||||
...css(node,'minHeight','fontSize','color','backgroundColor','borderTopColor','borderTopLeftRadius')});
|
||||
const controlOf=node=>({tag:node.tagName.toLowerCase(),id:node.id,type:node.type??null,value:node.value,
|
||||
min:node.getAttribute('min'),max:node.getAttribute('max'),step:node.getAttribute('step'),
|
||||
describedBy:node.getAttribute('aria-describedby'),
|
||||
options:node.tagName==='SELECT'?[...node.options].map(option=>[option.value,option.textContent]):null,
|
||||
...css(node,'fontSize','minHeight','backgroundColor','color','borderTopColor','borderTopLeftRadius','paddingLeft')});
|
||||
const fieldOf=field=>{const label=field.querySelector('label'),control=field.querySelector('input,textarea,select');
|
||||
return {label:label.textContent,labelled:label.htmlFor===control.id,...css(field,'display','rowGap','marginTop','marginBottom'),control:controlOf(control)};};
|
||||
const header=dialog.querySelector('.dialog-header'),title=header.querySelector('h2'),close=header.querySelector('button');
|
||||
const body=dialog.querySelector('#modal-body');
|
||||
const heading=body.querySelector('.tag-image-heading');
|
||||
const preview=heading.querySelector('.tag-image-preview'),copy=heading.querySelector(':scope > div');
|
||||
const sizeChip=copy.querySelector('.chip');
|
||||
const note=body.querySelector('.tag-scope-note');
|
||||
const manual=body.querySelector('#manual-image-tags'),manualField=manual.closest('.field'),help=body.querySelector('#manual-tags-help');
|
||||
const panel=body.querySelector('.tag-model-panel'),options=panel.querySelector('#image-tag-model-options');
|
||||
const generate=[...panel.querySelectorAll('button')].at(-1);
|
||||
const suggestions=body.querySelector('section.tag-suggestion-panel');
|
||||
const list=suggestions.querySelector('#tag-suggestions'),status=suggestions.querySelector('#tag-generation-status');
|
||||
const count=suggestions.querySelector('#suggestion-count');
|
||||
const actions=body.querySelector('.dialog-actions');
|
||||
// A suggestion is a concept checkbox row (`label.tag-suggestion`) or a production chip (`span.chip.tag-suggestion`).
|
||||
const rows=[...list.querySelectorAll('.tag-suggestion')].map(row=>{
|
||||
const name=row.querySelector('span:not(.tag-confidence)'),confidence=row.querySelector('.tag-confidence');
|
||||
const checkbox=row.querySelector('input[type=checkbox]'),remove=row.querySelector('button');
|
||||
return {name:name.textContent,confidence:confidence.textContent,
|
||||
control:checkbox?{kind:'checkbox',checked:checkbox.checked}:remove?{kind:'remove',label:remove.getAttribute('aria-label')}:null,
|
||||
...css(row,'display','alignItems','borderTopColor','backgroundColor','borderTopWidth','borderTopStyle','borderTopLeftRadius','fontSize','paddingTop','paddingLeft'),
|
||||
confidenceToken:css(confidence,'color','fontSize')};
|
||||
});
|
||||
return {
|
||||
dialog:{...css(dialog,'paddingTop','paddingRight','paddingBottom','paddingLeft','borderTopLeftRadius','backgroundColor','color','overflowY'),
|
||||
width:px(dialog.getBoundingClientRect().width),modal:dialog.matches(':modal'),tagging:dialog.classList.contains('tagging-modal'),
|
||||
labelledBy:document.getElementById(dialog.getAttribute('aria-labelledby'))===title,
|
||||
noHorizontalOverflow:dialog.scrollWidth<=dialog.clientWidth+1},
|
||||
header:{title:{text:title.textContent,...font(title)},close:{label:close.getAttribute('aria-label'),text:close.textContent.trim()}},
|
||||
demo:body.querySelector('.demo-note')?.textContent??null,
|
||||
order:[...body.children].map(node=>node.className),
|
||||
heading:{...css(heading,'display','columnGap','alignItems','marginBottom'),
|
||||
preview:{tag:preview.tagName.toLowerCase(),alt:preview.getAttribute('alt'),...box(preview),...css(preview,'objectFit','borderTopLeftRadius','borderTopWidth','borderTopStyle','borderTopColor','flexShrink')},
|
||||
h3:{text:copy.querySelector('h3').textContent,...font(copy.querySelector('h3'))},
|
||||
filename:{text:copy.querySelector('p.small').textContent,...font(copy.querySelector('p.small'))},
|
||||
size:sizeChip?{text:sizeChip.textContent,...css(sizeChip,'marginTop','fontSize','borderTopColor','borderTopLeftRadius','paddingLeft')}:null},
|
||||
scopeNote:note?{text:note.textContent,...edges(note),...css(note,'borderLeftWidth','borderLeftStyle','borderLeftColor','fontSize','color')}:null,
|
||||
manual:{...fieldOf(manualField),help:{text:help.textContent,...font(help)},described:help.id===manual.getAttribute('aria-describedby')},
|
||||
modelPanel:{...edges(panel),...css(panel,'marginTop'),h3:{text:panel.querySelector('h3').textContent,...font(panel.querySelector('h3'))},
|
||||
fields:[...panel.querySelectorAll('.field')].map(fieldOf),
|
||||
twoCol:options.querySelector('.two-col')?{columns:getComputedStyle(options.querySelector('.two-col')).gridTemplateColumns.split(' ').length,
|
||||
...css(options.querySelector('.two-col'),'display','columnGap')}:null,
|
||||
generate:buttonOf(generate)},
|
||||
suggestions:{labelled:suggestions.getAttribute('aria-labelledby')===suggestions.querySelector('h3').id,...css(suggestions,'marginTop'),
|
||||
title:{text:suggestions.querySelector('h3').textContent,...font(suggestions.querySelector('h3'))},
|
||||
count:{text:count.textContent,...font(count)},
|
||||
copy:list.querySelector(':scope > p.small')?.textContent??null,
|
||||
rows,
|
||||
status:{text:status.textContent,role:status.getAttribute('role'),...css(status,'marginTop','fontSize','color')}},
|
||||
actions:{...css(actions,'position','bottom','marginLeft','marginRight','marginBottom','paddingTop','paddingLeft','backgroundColor',
|
||||
'borderTopWidth','borderTopStyle','borderTopColor','zIndex','display','justifyContent','columnGap'),
|
||||
buttons:[...actions.querySelectorAll('button')].map(buttonOf)},
|
||||
noPageScroll:document.documentElement.scrollWidth<=innerWidth+1,
|
||||
};
|
||||
}
|
||||
|
||||
/** The first image card's Info and Tag group (M1), with its card's filename. */
|
||||
export function cardActionsInventory(card) {
|
||||
const css=(node,...keys)=>{const style=getComputedStyle(node);return Object.fromEntries(keys.map(key=>[key,style[key]]));};
|
||||
const info=card.querySelector('.media-info'),actions=info.querySelector(':scope > .image-card-actions');
|
||||
return {
|
||||
info:css(info,'display','justifyContent','alignItems','columnGap'),
|
||||
actions:{...css(actions,'display','alignItems','columnGap','flexShrink','minWidth'),lastInInfo:info.lastElementChild===actions},
|
||||
buttons:[...actions.querySelectorAll(':scope > button')].map(node=>{const rect=node.getBoundingClientRect();return {
|
||||
className:node.className,label:node.getAttribute('aria-label'),icon:node.querySelector('svg')?.innerHTML.replace(/\s+/g,' ').trim()??null,
|
||||
width:Math.round(rect.width),height:Math.round(rect.height),
|
||||
...css(node,'minWidth','minHeight','paddingLeft','paddingTop','borderTopLeftRadius','borderTopColor','backgroundColor','color')};}),
|
||||
};
|
||||
}
|
||||
|
||||
// D84 (plan D61): the legacy engines' values, labels and sizes, GGUF JoyCaption in the concept's Ollama slot.
|
||||
const MODEL_LABELS=Object.freeze({'wd14-swinv2':['wd14-swinv2','WD14 SwinV2 · ~400 MB'],'wd14-vit':['wd14-vit','WD14 ViT · ~400 MB'],
|
||||
ollama:['gguf','GGUF JoyCaption (Recommended) · ~4.5 GB'],huggingface:['hf','HuggingFace JoyCaption · ~16 GB']});
|
||||
const CONCEPT_LABELS=Object.freeze({'WD14 SwinV2 · fast classifier':'WD14 SwinV2 · ~400 MB','WD14 ViT · visual tags':'WD14 ViT · ~400 MB',
|
||||
'Ollama · local language model':'GGUF JoyCaption (Recommended) · ~4.5 GB','HuggingFace · language model':'HuggingFace JoyCaption · ~16 GB'});
|
||||
|
||||
/**
|
||||
* The concept inventory with the recorded deviations applied:
|
||||
* - no prototype note (Global Constraints: `Prototype · suggestions are simulated…` is not shipped);
|
||||
* - D83 (ruling T4): while suggestions exist, "Remove anything you do not want to add." leads them, and every
|
||||
* suggestion is a chip with a `Remove <name>` × instead of a checked checkbox row: the concept's own `.chip` corner
|
||||
* (`chip`: a concept `.chip` inventory, the heading's size chip) around the row's own text (12px) in the checked
|
||||
* row's colours (accent border, `--soft-blue`);
|
||||
* - D84 (plan D61): the legacy engine labels and values, and the server's default instructions (`instructions`);
|
||||
* - D91 (plan D68): a Suggestions count field after the model's own fields, on the concept's number field tokens
|
||||
* (`numberField`: the concept's General confidence field) with the configured count (`limit`);
|
||||
* - D29: the prototype's "Sample suggestions from …" status reads "Suggestions from …".
|
||||
* The opened state is compared unmapped: both show the concept's on-screen "Generate suggestions with the selected
|
||||
* model before approving." (its renderTagModelOptions, a1062–1076, replaces the markup's sentence at once).
|
||||
*/
|
||||
export function mapConceptTagging(reference,{instructions,limit=10,chip,numberField}) {
|
||||
const [select,...own]=reference.modelPanel.fields;
|
||||
const suggestionsField={...numberField,label:'Suggestions',control:{...numberField.control,id:'image-suggestion-limit',
|
||||
value:String(limit),min:'1',max:'100',step:'1'}};
|
||||
const mappedOwn=own.map(field=>field.control.id==='image-tag-instructions'?{...field,control:{...field.control,value:instructions}}:field);
|
||||
const rows=reference.suggestions.rows.map(row=>({...row,
|
||||
control:row.control?.kind==='checkbox'&&row.control.checked?{kind:'remove',label:`Remove ${row.name}`}:row.control,
|
||||
borderTopLeftRadius:chip.borderTopLeftRadius}));
|
||||
const copy=rows.length&&reference.suggestions.copy===null?'Remove anything you do not want to add.':reference.suggestions.copy;
|
||||
const status=reference.suggestions.status.text.replace(/^Sample suggestions from (.+)\. Approve tags to save\.$/,
|
||||
(_,label)=>`Suggestions from ${CONCEPT_LABELS[label]??label}. Approve tags to save.`);
|
||||
return {...reference,
|
||||
demo:null,
|
||||
order:reference.order.filter(name=>name!=='demo-note'),
|
||||
modelPanel:{...reference.modelPanel,
|
||||
fields:[{...select,control:{...select.control,value:MODEL_LABELS[select.control.value]?.[0]??select.control.value,
|
||||
options:select.control.options.map(([value])=>MODEL_LABELS[value])}},...mappedOwn,suggestionsField]},
|
||||
suggestions:{...reference.suggestions,copy,rows,status:{...reference.suggestions.status,text:status}},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Production's own chip geometry beyond the concept's `.chip` (D83): the chip's padding hugs its × button. Only the
|
||||
* leaves D83 replaces are dropped from both sides; names, confidences, colours, radius and font stay compared.
|
||||
*/
|
||||
export function withoutChipGeometry(inventory) {
|
||||
return {...inventory,suggestions:{...inventory.suggestions,rows:inventory.suggestions.rows.map(({paddingTop,paddingLeft,alignItems,...row})=>row)}};
|
||||
}
|
||||
|
||||
/**
|
||||
* Needs review (plan Task F3, Mock inventory M4: app.js renderReview a749–760, style.css s407–415, s796–814 and their
|
||||
* breakpoints): the page's main element as a tree, runs in the page. Every shown box below `main` to `depth` levels is
|
||||
* recorded with its tag and classes, its box relative to its parent, its fonts, edges and layout, a leaf's text, a
|
||||
* button's label and icon, a select's options; a `display: contents` node is replaced by its children (D20). A flex
|
||||
* row of actions also records the gaps between its children and its right inset, so a row whose button copy differs
|
||||
* (D29) still compares by order, spacing and alignment.
|
||||
*/
|
||||
export function reviewInventory(selector) {
|
||||
const main=document.querySelector(selector);
|
||||
// Half pixels: the concept's 13.5px line heights add up exactly, so a box that loses one line moves by 13.5px.
|
||||
const px=value=>Math.round(value*2)/2;
|
||||
const css=(node,...keys)=>{const style=getComputedStyle(node);return Object.fromEntries(keys.map(key=>[key,style[key]]));};
|
||||
const rect=node=>node.getBoundingClientRect();
|
||||
const shown=node=>node.getClientRects().length>0&&getComputedStyle(node).visibility!=='hidden';
|
||||
const relative=(node,origin)=>{const a=rect(node),b=rect(origin);return {left:px(a.left-b.left),top:px(a.top-b.top),width:px(a.width),height:px(a.height)};};
|
||||
const font=node=>css(node,'fontFamily','fontSize','fontWeight','lineHeight','letterSpacing','color','textTransform','textAlign');
|
||||
const edges=node=>css(node,'paddingTop','paddingRight','paddingBottom','paddingLeft','marginTop','marginRight','marginBottom','marginLeft',
|
||||
'borderTopWidth','borderTopStyle','borderTopColor','borderRightWidth','borderRightStyle','borderRightColor','borderBottomWidth',
|
||||
'borderBottomStyle','borderBottomColor','borderLeftWidth','borderLeftStyle','borderLeftColor','borderTopLeftRadius','backgroundColor',
|
||||
'boxShadow','outlineStyle');
|
||||
const layout=node=>css(node,'display','position','flexDirection','flexWrap','justifyContent','alignItems','columnGap','rowGap',
|
||||
'gridTemplateColumns','overflowX','overflowY','objectFit','minHeight','minWidth');
|
||||
const tagOf=node=>`${node.localName}${[...node.classList].sort().map(name=>`.${name}`).join('')}`;
|
||||
const textOf=node=>node.textContent.replace(/\s+/g,' ').trim();
|
||||
const boxed=node=>[...node.children].flatMap(child=>getComputedStyle(child).display==='contents'?boxed(child):[child]);
|
||||
const LEAVES=new Set(['button','select','input','img','svg','textarea','progress']);
|
||||
function nodeOf(node,origin,depth) {
|
||||
const base={tag:tagOf(node),...relative(node,origin),...font(node),...edges(node),...layout(node)};
|
||||
const name=node.localName;
|
||||
if(name==='svg')return {tag:base.tag,width:base.width,height:base.height,d:node.querySelector('path')?.getAttribute('d')??null,
|
||||
...css(node,'strokeWidth','color','verticalAlign')};
|
||||
if(name==='button')return {...base,text:textOf(node),label:node.getAttribute('aria-label'),pressed:node.getAttribute('aria-pressed'),
|
||||
disabled:node.getAttribute('aria-disabled')==='true'||node.disabled,primary:node.classList.contains('ca-primary'),
|
||||
children:boxed(node).filter(shown).map(child=>nodeOf(child,node,0))};
|
||||
if(name==='select')return {...base,id:node.id,options:[...node.options].map(option=>[option.textContent,option.disabled])};
|
||||
if(name==='input')return {...base,id:node.id,type:node.type,label:node.getAttribute('aria-label')};
|
||||
if(name==='img')return {...base,alt:node.getAttribute('alt')};
|
||||
const children=boxed(node).filter(shown);
|
||||
const item={...base,id:node.id||null,role:node.getAttribute('role'),label:node.getAttribute('aria-label'),
|
||||
text:children.length?null:textOf(node),htmlFor:node.htmlFor||null};
|
||||
if(/(^|\s)(dialog-actions|row-actions|image-card-actions)(\s|$)/.test(node.className)&&children.length) {
|
||||
const boxes=children.map(rect);
|
||||
item.gaps=boxes.slice(1).map((box,index)=>px(box.left-boxes[index].right));
|
||||
item.rightInset=px(rect(node).right-boxes.at(-1).right);
|
||||
}
|
||||
if(depth>0&&!LEAVES.has(name))item.children=children.map(child=>nodeOf(child,node,depth-1));
|
||||
return item;
|
||||
}
|
||||
return {main:nodeOf(main,main,8),noPageScroll:document.documentElement.scrollWidth<=innerWidth+1};
|
||||
}
|
||||
|
||||
/**
|
||||
* `tree` with `edit(node)` applied to every node `match` picks (a tag prefix or a predicate), depth first, new objects
|
||||
* only.
|
||||
*/
|
||||
function editNodes(tree,match,edit) {
|
||||
const children=tree.children?.map(child=>editNodes(child,match,edit));
|
||||
const node=children?{...tree,children}:tree;
|
||||
const picked=typeof match==='function'?match(node):node.tag?.startsWith(match);
|
||||
return picked?edit(node):node;
|
||||
}
|
||||
const findNode=(tree,prefix)=>tree.tag?.startsWith(prefix)?tree:tree.children?.map(child=>findNode(child,prefix)).find(Boolean)??null;
|
||||
|
||||
/**
|
||||
* The concept's Metadata review inventory (reviewInventory) with the recorded deviations applied, the remaining sample
|
||||
* data taken from production (`actual`) only where the concept's own copy cannot say it:
|
||||
* - D29: no "Sample review queue" chip in the notice; "Resolve the association" and "Apply association" without the
|
||||
* prototype's "sample";
|
||||
* - D119: queue cards carry no selection checkbox and no star (the overlay row stays, empty); a card's title is its
|
||||
* file name (an undecided item names no prompt) and its line is the extension only (D11 omits the unknown size and
|
||||
* seed, which a queue item does not carry); the image's alt is the file name; the prompt blocks' Copy and Send names
|
||||
* carry no prompt title;
|
||||
* - D3: the grid keeps the Images grid's `position: relative` (its cards stay in its flow);
|
||||
* - production's hooks and names: the `review-queue` / `review-panel` classes, the queue's list roles and name, the
|
||||
* panel heading's id (the region's name) and the prompt sections' region names;
|
||||
* - sample data: the Sampler context's candidate options (the file's own evidence; the last option, Leave unresolved,
|
||||
* stays compared) and the widths that follow from copy the deviations above change (a card's text column, the
|
||||
* panel's action buttons; their gaps and right inset stay compared).
|
||||
* Where that copy changes the height of a box, the boxes after it move by exactly that much: the stacked notice (390)
|
||||
* loses the chip and its row gap; a card whose concept line wraps (390: "1536 × 1152 · PNG · Seed 42001") takes
|
||||
* production's one-line height, its row of the grid shrinks by the most any of its cards lost, and in one column the
|
||||
* panel below the grid moves up by the grid's loss. The actions keep production's centred place in the card's info.
|
||||
*/
|
||||
export function mapConceptReview(reference,actual) {
|
||||
const named=(node,id)=>({...node,id});
|
||||
const actualCards=findNode(actual.main,'div.image-grid')?.children??[];
|
||||
const actualActions=findNode(actual.main,'div.dialog-actions');
|
||||
let cardIndex=0;
|
||||
let main={...reference.main,id:actual.main.id};
|
||||
let noticeLoss=0;
|
||||
main=editNodes(main,'div.notice',notice=>{
|
||||
const chip=notice.children.find(child=>child.tag==='span.chip');
|
||||
if(chip&¬ice.flexDirection==='column')noticeLoss=chip.height+parseFloat(notice.rowGap);
|
||||
return {...notice,height:notice.height-noticeLoss,children:notice.children.filter(child=>child!==chip)};
|
||||
});
|
||||
const noticeIndex=main.children.findIndex(child=>child.tag==='div.notice');
|
||||
main={...main,height:main.height-noticeLoss,children:main.children.map((child,index)=>index>noticeIndex?{...child,top:child.top-noticeLoss}:child)};
|
||||
const losses=[];
|
||||
main=editNodes(main,'div.image-grid',grid=>({...grid,tag:'div.image-grid.review-queue',position:'relative',role:'list',label:'Review queue'}));
|
||||
main=editNodes(main,'article.media-card',card=>{
|
||||
const twin=actualCards[cardIndex++];
|
||||
const filename=findNode(card,'button.image-open').label.replace(/^Open /,'');
|
||||
const art=editNodes(card.children[0],'div.image-overlays',overlays=>({...overlays,height:0,text:'',children:[]}));
|
||||
const withAlt=editNodes(art,'img',img=>({...img,alt:filename}));
|
||||
const info=card.children[1];
|
||||
const [copy,...rest]=info.children;
|
||||
const twinCopy=twin?.children[1]?.children[0];
|
||||
const [h3,p]=copy.children;
|
||||
const twinLine=twinCopy?.children[1];
|
||||
const loss=twinLine?p.height-twinLine.height:0;
|
||||
losses.push({top:card.top,loss});
|
||||
const mappedCopy={...copy,width:twinCopy?.width,height:copy.height-loss,children:[{...h3,text:filename,width:twinCopy?.children[0]?.width},
|
||||
{...p,text:'PNG',width:twinLine?.width,height:p.height-loss}]};
|
||||
const [actions,...after]=rest;
|
||||
const twinActions=twin?.children[1]?.children[1];
|
||||
const mappedActions=actions&&loss?{...actions,top:twinActions?.top}:actions;
|
||||
return {...card,role:'listitem',height:card.height-loss,
|
||||
children:[withAlt,{...info,height:info.height-loss,children:[mappedCopy,...(actions?[mappedActions]:[]),...after]},...card.children.slice(2)]};
|
||||
});
|
||||
// Each grid row loses what its most shortened card lost; later rows move up by the rows before them.
|
||||
const rowLoss=new Map();
|
||||
for(const {top,loss} of losses)rowLoss.set(top,Math.max(rowLoss.get(top)??0,loss));
|
||||
const lossBefore=top=>[...rowLoss].filter(([rowTop])=>rowTop<top).reduce((sum,[,loss])=>sum+loss,0);
|
||||
const gridLoss=[...rowLoss.values()].reduce((sum,loss)=>sum+loss,0);
|
||||
main=editNodes(main,'div.image-grid',grid=>({...grid,height:grid.height-gridLoss,
|
||||
children:grid.children.map(card=>({...card,top:card.top-lossBefore(card.top)}))}));
|
||||
main=editNodes(main,'div.gallery-layout',layout=>{
|
||||
if(!gridLoss||layout.gridTemplateColumns.trim().split(/\s+/).length>1)return layout;
|
||||
return {...layout,height:layout.height-gridLoss,children:layout.children.map((child,index)=>index?{...child,top:child.top-gridLoss}:child)};
|
||||
});
|
||||
if(gridLoss&&findNode(main,'div.gallery-layout').gridTemplateColumns.trim().split(/\s+/).length===1)main={...main,height:main.height-gridLoss};
|
||||
main=editNodes(main,'section.surface',surface=>{
|
||||
const [h2,...rest]=surface.children;
|
||||
return {...surface,tag:'section.review-panel.surface',children:[named({...h2,text:h2.text.replace('Resolve the sample association','Resolve the association')},'review-panel-title'),...rest]};
|
||||
});
|
||||
main=editNodes(main,'select',select=>{
|
||||
const twin=findNode(actual.main,'select');
|
||||
return select.id==='review-context'?{...select,options:[...twin.options.slice(0,-1),select.options.at(-1)]}:select;
|
||||
});
|
||||
main=editNodes(main,node=>/^section\..*\bprompt-section\b/.test(node.tag),section=>({...section,label:section.tag.includes('positive')?'Positive prompt':'Negative prompt'}));
|
||||
main=editNodes(main,node=>node.tag?.startsWith('button')&&/^(Copy|Send) (positive|negative) prompt/.test(node.label??''),
|
||||
button=>({...button,label:button.label.replace(/: .+$/,'')}));
|
||||
main=editNodes(main,'div.dialog-actions',actions=>({...actions,children:actions.children.map((button,index)=>({...button,
|
||||
text:button.text.replace('Apply sample association','Apply association'),
|
||||
left:actualActions?.children[index]?.left,width:actualActions?.children[index]?.width}))}));
|
||||
return {...reference,main};
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
// The Tag image dialog and the card's Info and Tag group against the approved concept (#249 PR C, plan Task C4, Mock
|
||||
// inventory M1 and M2). Production runs on the concept-shaped fixture with the concept image's own tags saved as its
|
||||
// image tags and the injected fake model answering the concept's own sample (alpine: mountain 98%, observatory 95%,
|
||||
// snow 88%, blue_hour 81%), so every remaining difference is a recorded deviation applied by mapConceptTagging
|
||||
// (D29, D83, D84, D91) or a test failure. No model is ever downloaded: every test checks no download request was sent.
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction, saveImageTags, mutate, SIZES, THEMES } from './concept-inventory.mjs';
|
||||
import { taggingInventory, cardActionsInventory, mapConceptTagging, diffTaggingInventory, withoutChipGeometry } from './tagging-canonical-comparison.mjs';
|
||||
|
||||
const MODULE = '/prompt_manager/js/workspace/image-tagging.mjs';
|
||||
// The concept's first image (app.js images[0]) and its sample suggestions (generateTags, scene "alpine").
|
||||
const CONCEPT_FILE = 'alpine_00001.png';
|
||||
const CONCEPT_TAGS = ['cinematic', 'blue_hour', 'architecture'];
|
||||
const ALPINE = [['mountain', 0.98], ['observatory', 0.95], ['snow', 0.88], ['blue_hour', 0.81]];
|
||||
// The fake model's server default instructions (tests/browser/server.py TestModel.default_prompt), D84.
|
||||
const INSTRUCTIONS = 'Describe the image';
|
||||
const CONCEPT_MODEL = { gguf: 'ollama' };
|
||||
|
||||
const tagDialog = page => page.getByRole('dialog', { name: 'Tag image', exact: true });
|
||||
const firstCard = (page, side) => page.locator(side === 'concept' ? '#image-grid > article.media-card' : '#workspace-main article[data-image-key]').first();
|
||||
|
||||
test.beforeEach(async ({ page, request }) => {
|
||||
const manifest = await seedConcept(request);
|
||||
expect(manifest).toBeTruthy();
|
||||
await expect(await request.post('/__test__/tagging/suggestions', { data: { scored: ALPINE } })).toBeOK();
|
||||
page.downloads = [];
|
||||
page.on('request', item => { if (/\/autotag\/download\//.test(item.url())) page.downloads.push(item.url()); });
|
||||
page.errors = [];
|
||||
page.on('pageerror', error => page.errors.push(error.message));
|
||||
});
|
||||
test.afterEach(async ({ page, request }) => {
|
||||
await leave(page);
|
||||
expect(page.downloads).toEqual([]);
|
||||
expect(page.errors).toEqual([]);
|
||||
await request.post('/__test__/tagging/suggestions', { data: { scored: null } });
|
||||
});
|
||||
|
||||
async function conceptPath(request) {
|
||||
const listing = await (await request.get('/prompt_manager/images/output?mode=workspace&limit=40')).json();
|
||||
const image = listing.images.find(item => item.filename === CONCEPT_FILE);
|
||||
expect(image).toBeTruthy();
|
||||
return image.path;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close an open Tag dialog before opening the next page, so each comparison starts from a settled page. Leaving with
|
||||
* the dialog open is covered on its own (workspace-tagging-lifecycle.spec.mjs: the keepalive release on pagehide).
|
||||
*/
|
||||
async function leave(page) {
|
||||
if (await tagDialog(page).count()) {
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(tagDialog(page)).toHaveCount(0);
|
||||
}
|
||||
}
|
||||
/** Open `side`'s Tag image dialog from its first card, settled (production: the capture preview and saved tags). */
|
||||
async function openTagDialog(page, side, theme) {
|
||||
await leave(page);
|
||||
if (side === 'concept') await openConcept(page, 'images', theme);
|
||||
else await openProduction(page, 'images', theme);
|
||||
await firstCard(page, side).getByRole('button', { name: `Tag or auto-tag ${CONCEPT_FILE}`, exact: true }).click();
|
||||
await expect(tagDialog(page)).toBeVisible();
|
||||
await settled(page, side);
|
||||
}
|
||||
async function settled(page, side) {
|
||||
if (side === 'production') {
|
||||
await expect(tagDialog(page).locator('img.tag-image-preview')).toHaveAttribute('src', /\/image-annotations\/capture\//);
|
||||
await expect(tagDialog(page).locator('#manual-image-tags')).toHaveValue(CONCEPT_TAGS.join(', '));
|
||||
}
|
||||
await page.evaluate(async () => {
|
||||
await document.fonts.ready;
|
||||
await Promise.all([...document.querySelectorAll('#modal img')].map(image => image.decode().catch(() => {})));
|
||||
});
|
||||
}
|
||||
async function generateWith(page, side, model) {
|
||||
const select = tagDialog(page).getByRole('combobox', { name: 'Tagging model', exact: true });
|
||||
if (model) await select.selectOption(side === 'concept' ? CONCEPT_MODEL[model] ?? model : model);
|
||||
await tagDialog(page).getByRole('button', { name: 'Generate suggestions', exact: true }).click();
|
||||
await expect(tagDialog(page).locator('#suggestion-count')).not.toHaveText('Not generated');
|
||||
}
|
||||
// The pointer rests outside the dialog, so no control is measured in its hover state.
|
||||
const inventory = async page => {
|
||||
await page.mouse.move(0, 0);
|
||||
return page.locator('dialog#modal').evaluate(taggingInventory);
|
||||
};
|
||||
|
||||
/** The concept and production inventories of one dialog state, the concept mapped (D29, D83, D84, D91). */
|
||||
async function compareState(page, { theme, act = async () => {} }) {
|
||||
await openTagDialog(page, 'concept', theme);
|
||||
const conceptOpened = await inventory(page);
|
||||
await act(page, 'concept');
|
||||
const concept = await inventory(page);
|
||||
await openTagDialog(page, 'production', theme);
|
||||
await act(page, 'production');
|
||||
const actual = await inventory(page);
|
||||
const numberField = conceptOpened.modelPanel.fields.find(field => field.control.id === 'image-general-threshold');
|
||||
const reference = mapConceptTagging(concept, { instructions: INSTRUCTIONS, chip: concept.heading.size, numberField });
|
||||
return { reference: withoutChipGeometry(reference), actual: withoutChipGeometry(actual) };
|
||||
}
|
||||
|
||||
for (const [width, height] of SIZES) for (const theme of THEMES) {
|
||||
test(`concept and production Tag dialog match at ${width}x${height} ${theme}`, async ({ page, request }) => {
|
||||
test.setTimeout(90_000);
|
||||
await saveImageTags(request, await conceptPath(request), CONCEPT_TAGS);
|
||||
await page.setViewportSize({ width, height });
|
||||
// Opened, generated with WD14 (the concept's default), then generated with a language model.
|
||||
for (const [state, act] of [
|
||||
['opened', async () => {}],
|
||||
['generated WD14', (page, side) => generateWith(page, side)],
|
||||
['generated GGUF', (page, side) => generateWith(page, side, 'gguf')],
|
||||
]) {
|
||||
const { reference, actual } = await compareState(page, { theme, act });
|
||||
expect(diffTaggingInventory(reference, actual), `${state} at ${width}x${height} ${theme}`).toEqual([]);
|
||||
}
|
||||
// Pinned concept geometry (M2), so a mapping mistake on both sides cannot pass unseen.
|
||||
const { actual } = await compareState(page, { theme, act: (page, side) => generateWith(page, side) });
|
||||
const small = width <= 700;
|
||||
expect([actual.heading.preview.width, actual.heading.preview.height]).toEqual(small ? [86, 86] : [112, 100]);
|
||||
expect(actual.heading.columnGap).toBe('16px');
|
||||
expect([actual.modelPanel.paddingTop, actual.modelPanel.borderTopLeftRadius]).toEqual(['16px', '6px']);
|
||||
expect(actual.actions.position).toBe('sticky');
|
||||
expect(actual.suggestions.rows.map(row => [row.name, row.confidence, row.control.label])).toEqual(
|
||||
ALPINE.map(([name, confidence]) => [name, `${Math.round(confidence * 100)}%`, `Remove ${name}`]));
|
||||
const accent = await page.evaluate(() => {
|
||||
const probe = document.createElement('i');
|
||||
probe.style.cssText = 'border-top:1px solid var(--ca-accent);background:var(--soft-blue)';
|
||||
document.body.append(probe);
|
||||
const style = getComputedStyle(probe), tokens = [style.borderTopColor, style.backgroundColor];
|
||||
probe.remove();
|
||||
return tokens;
|
||||
});
|
||||
for (const row of actual.suggestions.rows) expect([row.borderTopColor, row.backgroundColor]).toEqual(accent);
|
||||
expect(actual.dialog.noHorizontalOverflow).toBe(true);
|
||||
expect(actual.noPageScroll).toBe(true);
|
||||
});
|
||||
}
|
||||
|
||||
test('an empty result matches the concept below its threshold', async ({ page, request }) => {
|
||||
await saveImageTags(request, await conceptPath(request), CONCEPT_TAGS);
|
||||
await expect(await request.post('/__test__/tagging/suggestions', { data: { scored: [] } })).toBeOK();
|
||||
const { reference, actual } = await compareState(page, { theme: 'dark', act: async (page, side) => {
|
||||
// The concept filters its sample by the General confidence; above every sample nothing is shown.
|
||||
if (side === 'concept') await tagDialog(page).getByRole('spinbutton', { name: 'General confidence', exact: true }).fill('1');
|
||||
await generateWith(page, side);
|
||||
if (side === 'concept') await tagDialog(page).getByRole('spinbutton', { name: 'General confidence', exact: true }).fill('0.35');
|
||||
} });
|
||||
expect(actual.suggestions.copy).toBe('No suggestions met this threshold. Adjust it or enter tags manually.');
|
||||
expect(diffTaggingInventory(reference, actual)).toEqual([]);
|
||||
});
|
||||
|
||||
test('review-scoped dialog shows the scope note', async ({ page, request }) => {
|
||||
const path = await conceptPath(request);
|
||||
await saveImageTags(request, path, CONCEPT_TAGS);
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await openConcept(page, 'review', 'dark');
|
||||
await page.locator('#main section.surface').getByRole('button', { name: 'Tag image', exact: true }).click();
|
||||
await expect(tagDialog(page)).toBeVisible();
|
||||
await settled(page, 'concept');
|
||||
const opened = await inventory(page);
|
||||
// Needs review ships with PR F; until then the review source opens the same dialog from the module.
|
||||
await leave(page);
|
||||
await openProduction(page, 'images', 'dark');
|
||||
await page.evaluate(async ({ module, path }) => {
|
||||
const { openImageTagging } = await import(module);
|
||||
const { createScope } = await import('/prompt_manager/js/shared/toolkit/core.js');
|
||||
const api = window.ApiClient.createApiClient();
|
||||
const listing = await api.get('/prompt_manager/images/output?mode=workspace&limit=40');
|
||||
openImageTagging({ api, scope: createScope(), image: listing.images.find(item => item.path === path), source: 'review' });
|
||||
}, { module: MODULE, path });
|
||||
await expect(tagDialog(page)).toBeVisible();
|
||||
await settled(page, 'production');
|
||||
const actual = await inventory(page);
|
||||
expect(actual.scopeNote.text).toBe('Tags belong to this image. Resolve its prompt association separately in Needs review.');
|
||||
expect(actual.order).toEqual(['tag-image-heading', 'tag-scope-note small', 'field', 'tag-model-panel', 'tag-suggestion-panel', 'dialog-actions']);
|
||||
const numberField = opened.modelPanel.fields.find(field => field.control.id === 'image-general-threshold');
|
||||
const reference = mapConceptTagging(opened, { instructions: INSTRUCTIONS, chip: opened.heading.size, numberField });
|
||||
expect(diffTaggingInventory(reference, actual)).toEqual([]);
|
||||
});
|
||||
|
||||
for (const [width, height] of SIZES) test(`card action group matches M1 at ${width}x${height}`, async ({ page }) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
await openConcept(page, 'images', 'dark');
|
||||
const reference = await firstCard(page, 'concept').evaluate(cardActionsInventory);
|
||||
await openProduction(page, 'images', 'dark');
|
||||
const actual = await firstCard(page, 'production').evaluate(cardActionsInventory);
|
||||
expect(actual.buttons.map(button => button.label)).toEqual([`Inspect ${CONCEPT_FILE}`, `Tag or auto-tag ${CONCEPT_FILE}`]);
|
||||
expect(diffTaggingInventory(reference, actual)).toEqual([]);
|
||||
});
|
||||
|
||||
test('the comparison notices a changed panel border or primary label', async ({ page, request }) => {
|
||||
await saveImageTags(request, await conceptPath(request), CONCEPT_TAGS);
|
||||
const { reference, actual } = await compareState(page, { theme: 'dark' });
|
||||
expect(diffTaggingInventory(reference, actual)).toEqual([]);
|
||||
const undo = await mutate(page, { css: '.tag-model-panel { border: 0 !important; }' });
|
||||
const borderless = withoutChipGeometry(await inventory(page));
|
||||
expect(diffTaggingInventory(reference, borderless).map(item => item.path)).toEqual(
|
||||
expect.arrayContaining(['modelPanel.borderTopWidth', 'modelPanel.borderTopStyle']));
|
||||
await undo();
|
||||
await tagDialog(page).locator('.dialog-actions .ca-primary').evaluate(node => { node.textContent = 'Save'; });
|
||||
const relabelled = withoutChipGeometry(await inventory(page));
|
||||
expect(diffTaggingInventory(reference, relabelled).map(item => item.path)).toEqual(['actions.buttons.1.text']);
|
||||
});
|
||||
|
||||
// Paired captures for the PR (no screenshot baselines): every production state beside the concept's nearest one. The
|
||||
// states the concept lacks (unavailable model, saving, conflict, file changed) pair with the concept's opened dialog.
|
||||
test('paired Tag dialog captures', async ({ page, request }, testInfo) => {
|
||||
test.setTimeout(120_000);
|
||||
const path = await conceptPath(request);
|
||||
await saveImageTags(request, path, CONCEPT_TAGS);
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const shot = async name => {
|
||||
await page.evaluate(async () => { await document.fonts.ready; await Promise.all([...document.images].map(image => image.decode().catch(() => {}))); });
|
||||
const file = testInfo.outputPath(`${name}.png`);
|
||||
await page.screenshot({ path: file });
|
||||
await testInfo.attach(name, { path: file, contentType: 'image/png' });
|
||||
};
|
||||
const both = async (name, act) => {
|
||||
for (const side of ['concept', 'production']) {
|
||||
await openTagDialog(page, side, 'dark');
|
||||
await act(page, side);
|
||||
await shot(`${side}-${name}`);
|
||||
}
|
||||
};
|
||||
await both('opened', async () => {});
|
||||
await both('generated-wd14', (page, side) => generateWith(page, side));
|
||||
await both('generated-gguf', (page, side) => generateWith(page, side, 'gguf'));
|
||||
await expect(await request.post('/__test__/tagging/suggestions', { data: { scored: [] } })).toBeOK();
|
||||
await both('empty', async (page, side) => {
|
||||
if (side === 'concept') await tagDialog(page).getByRole('spinbutton', { name: 'General confidence', exact: true }).fill('1');
|
||||
await generateWith(page, side);
|
||||
});
|
||||
await expect(await request.post('/__test__/tagging/suggestions', { data: { scored: ALPINE } })).toBeOK();
|
||||
const save = () => tagDialog(page).locator('.dialog-actions .ca-primary').click();
|
||||
const status = () => tagDialog(page).locator('#tag-generation-status');
|
||||
|
||||
// Unavailable model.
|
||||
await expect(await request.post('/__test__/tagging/models', { data: { unavailable: ['wd14-swinv2'] } })).toBeOK();
|
||||
await openTagDialog(page, 'production', 'dark');
|
||||
await generateWith(page, 'production').catch(() => {});
|
||||
await expect(status()).toHaveText('WD14 SwinV2 is not downloaded. Download it from the legacy Auto-tag page.');
|
||||
await shot('production-unavailable');
|
||||
await expect(await request.post('/__test__/tagging/models', { data: { unavailable: [] } })).toBeOK();
|
||||
|
||||
// Saving: the write is held until the capture is taken.
|
||||
let release;
|
||||
const held = new Promise(resolve => { release = resolve; });
|
||||
const writes = url => url.pathname === '/prompt_manager/image-annotations/tags';
|
||||
await page.route(writes, async route => { await held; await route.continue(); });
|
||||
await openTagDialog(page, 'production', 'dark');
|
||||
const sent = page.waitForRequest(item => writes(new URL(item.url())));
|
||||
await save();
|
||||
await sent;
|
||||
await expect(tagDialog(page).locator('.dialog-actions .ca-primary')).toHaveAttribute('aria-disabled', 'true');
|
||||
await shot('production-saving');
|
||||
release();
|
||||
await expect(tagDialog(page)).toHaveCount(0);
|
||||
await page.unroute(writes);
|
||||
|
||||
// Conflict: another writer saved this file's set after the dialog observed it.
|
||||
await openTagDialog(page, 'production', 'dark');
|
||||
await saveImageTags(request, path, ['changed_elsewhere']);
|
||||
await save();
|
||||
await expect(status()).not.toHaveText('');
|
||||
await shot('production-conflict');
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
// File changed: the server's own envelope for bytes replaced after the preview (B1 `file-changed`).
|
||||
await page.route(writes, route => route.fulfill({ status: 409, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, code: 'file-changed', error: 'The file changed since it was previewed. Reopen the Tag dialog.' }) }));
|
||||
await saveImageTags(request, path, CONCEPT_TAGS);
|
||||
await openTagDialog(page, 'production', 'dark');
|
||||
await save();
|
||||
await expect(status()).not.toHaveText('');
|
||||
await shot('production-file-changed');
|
||||
await page.unroute(writes);
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
// Unconfirmed save (a 504): D116's Retry leads the actions, before Cancel and the disabled primary.
|
||||
await page.route(writes, route => route.fulfill({ status: 504, body: 'Gateway Timeout' }));
|
||||
await openTagDialog(page, 'production', 'dark');
|
||||
await save();
|
||||
await expect(status()).toHaveText('The save could not be confirmed. Retry sends the same tags again.');
|
||||
expect((await inventory(page)).actions.buttons.map(button => [button.text, button.primary, button.disabled])).toEqual(
|
||||
[['Retry', false, false], ['Cancel', false, false], ['Save manual tags', true, true]]);
|
||||
await shot('production-unknown-outcome');
|
||||
await page.unroute(writes);
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
// Review scope: the concept's Needs review dialog and production's review-sourced dialog.
|
||||
await leave(page);
|
||||
await openConcept(page, 'review', 'dark');
|
||||
await page.locator('#main section.surface').getByRole('button', { name: 'Tag image', exact: true }).click();
|
||||
await expect(tagDialog(page)).toBeVisible();
|
||||
await shot('concept-review');
|
||||
await leave(page);
|
||||
await openProduction(page, 'images', 'dark');
|
||||
await page.evaluate(async ({ module, path }) => {
|
||||
const { openImageTagging } = await import(module);
|
||||
const { createScope } = await import('/prompt_manager/js/shared/toolkit/core.js');
|
||||
const api = window.ApiClient.createApiClient();
|
||||
const listing = await api.get('/prompt_manager/images/output?mode=workspace&limit=40');
|
||||
openImageTagging({ api, scope: createScope(), image: listing.images.find(item => item.path === path), source: 'review' });
|
||||
}, { module: MODULE, path });
|
||||
await expect(tagDialog(page).locator('.tag-scope-note')).toBeVisible();
|
||||
await settled(page, 'production');
|
||||
await shot('production-review');
|
||||
await leave(page);
|
||||
});
|
||||
@@ -64,12 +64,10 @@ export function pageInventoryOf(main) {
|
||||
// Recorded deviations only (plan "Mock inventory" + decisions D1-D15 + ledger rulings). Everything else must be equal.
|
||||
export function mapVocabularyConcept(reference) {
|
||||
return {...reference,
|
||||
heading:{...reference.heading,actions:{...reference.heading.actions,
|
||||
// D2: Auto-tag images is omitted until the image-annotation slice owns image tags.
|
||||
buttons:reference.heading.actions.buttons.filter(item=>item.text!=='Auto-tag images')}},
|
||||
table:{...reference.table,rows:reference.table.rows.map(row=>({...row,
|
||||
// D3: no image-tag owner yet, so the Images cell shows "—" with a visually hidden "Not available yet".
|
||||
images:'—',imagesHidden:'Not available yet'}))},
|
||||
// Data, not a deviation: the concept tags each sample image with its prompt's tags; the vocabulary fixture
|
||||
// tags no image, so every Images count is 0 (#249 C2 retired #245 D3).
|
||||
images:'0'}))},
|
||||
operations:reference.operations.map(row=>({...row,p:{...row.p,
|
||||
// Ledger ruling: "1 positive prompts" -> singular, consistent with the shared plural helper (#238/#243).
|
||||
text:row.p.text.replace(/^1 positive prompts$/,'1 positive prompt')}})),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
|
||||
// Artifact-only captures of Tags & categories states that have no concept counterpart: the Manage categories
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
// Each Playwright worker owns its server: `request`, `page` and manual
|
||||
// `browser.newContext()` all reach it through `baseURL`, and another server
|
||||
// process never sees its database.
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { startBrowserServer } from '../../tools/vendor/start-browser-server.mjs';
|
||||
|
||||
const search = text => `/prompt_manager/search?text=${encodeURIComponent(text)}`;
|
||||
|
||||
test('every fixture reaches this worker\'s own server and another server keeps its own database', async ({ request, page, browser, baseURL, workerServer }, testInfo) => {
|
||||
expect(baseURL).toBe(workerServer.url);
|
||||
await expect(await request.post('/__test__/reset')).toBeOK();
|
||||
const marker = `worker ${testInfo.workerIndex} isolation ${testInfo.project.name} ${testInfo.repeatEachIndex}`;
|
||||
await expect(await request.post('/prompt_manager/save', { data: { text: marker } })).toBeOK();
|
||||
expect((await (await request.get(search(marker))).json()).count).toBe(1);
|
||||
|
||||
const viaPage = await page.request.get(search(marker));
|
||||
expect(viaPage.url().startsWith(`${baseURL}/`)).toBe(true);
|
||||
expect((await viaPage.json()).count).toBe(1);
|
||||
const context = await browser.newContext();
|
||||
try {
|
||||
const response = await (await context.newPage()).goto(search(marker));
|
||||
expect(response.url().startsWith(`${baseURL}/`)).toBe(true);
|
||||
expect((await response.json()).count).toBe(1);
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
|
||||
const other = await startBrowserServer();
|
||||
try {
|
||||
expect(other.url).not.toBe(baseURL);
|
||||
expect((await (await fetch(`${other.url}${search(marker)}`)).json()).count).toBe(0);
|
||||
} finally {
|
||||
await other.stop();
|
||||
}
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
test.beforeEach(async ({ request, page, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
@@ -19,6 +19,13 @@ test.afterEach(async ({ page, browserName }) => {
|
||||
expect(page.workflowErrors).toEqual([]);
|
||||
});
|
||||
|
||||
// #249 S6b: the legacy admin page says where the library lives (the fixture's database is configured explicitly).
|
||||
test('the admin page names the library location', async ({ page }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await expect(page.locator('#libraryLocation')).toHaveText('Library: browser.db (set in config.json)');
|
||||
await expect(page.locator('#totalPrompts')).not.toHaveText('-');
|
||||
});
|
||||
|
||||
test('actual gallery form saves a prompt that the admin page can edit and reload', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await page.locator('#addPromptBtn').click();
|
||||
@@ -43,6 +50,55 @@ test('actual gallery form saves a prompt that the admin page can edit and reload
|
||||
expect((await saved.json()).prompt_id).not.toBe(Number(id));
|
||||
});
|
||||
|
||||
// WebKit fails a fetch that a leaving page starts ("Fetch API cannot load … due to access control checks."), and
|
||||
// Playwright reports that console error as a page error. A save's follow-up reads must not start once the page hides:
|
||||
// here the statistics read after an in-place edit is held at the server while the page reloads.
|
||||
test('an in-place edit whose follow-up reads outlive the page starts no read while it leaves', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
const text = page.locator('.prompt-text').filter({ hasText: 'Cat portrait' });
|
||||
const id = await text.getAttribute('data-id');
|
||||
await expect(await request.post('/__test__/hold', { data: { path: '/prompt_manager/stats' } })).toBeOK();
|
||||
await page.locator(`[data-action="edit"][data-prompt-id="${id}"]`).click();
|
||||
const editor = page.locator(`.prompt-text[data-id="${id}"]`);
|
||||
await editor.fill('Cat portrait at dusk');
|
||||
await editor.press('Enter');
|
||||
const held = async () => (await (await request.get('/__test__/hold')).json()).held['/prompt_manager/stats'];
|
||||
await expect.poll(held).toBe(1);
|
||||
await page.reload({ waitUntil: 'commit' });
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await expect(page.locator('.prompt-text').filter({ hasText: 'Cat portrait at dusk' })).toBeVisible();
|
||||
});
|
||||
|
||||
// A leave that never happens (a stopped navigation, or one answered with a download) must not leave the list dead.
|
||||
test('a page whose leave was called off reads its list again on the next input', async ({ page }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await expect(page.locator('.prompt-text').first()).toBeVisible();
|
||||
await page.evaluate(() => window.dispatchEvent(new Event('beforeunload')));
|
||||
await page.mouse.click(1, 1);
|
||||
const reread = page.waitForResponse(response => new URL(response.url()).pathname === '/prompt_manager/recent');
|
||||
await page.locator('#sortBy').selectOption('created_asc');
|
||||
expect(new URL((await reread).url()).searchParams.get('sort')).toBe('created_asc');
|
||||
await expect(page.locator('#resultsTitle')).toHaveText('Recent Prompts');
|
||||
});
|
||||
|
||||
// The gallery's startup thumbnail check runs a second after load. When that second ends while the page is leaving,
|
||||
// the check must start no read: WebKit fails it with "Fetch API cannot load … due to access control checks." and
|
||||
// Playwright reports a page error (seen in CI on PR #289). The page clock makes the second end after the leave began.
|
||||
for (const leave of ['beforeunload', 'pagehide']) {
|
||||
test(`the gallery startup check starts no read after ${leave}`, async ({ page }) => {
|
||||
await page.clock.install();
|
||||
const statusReads = [];
|
||||
page.on('request', req => { if (req.url().includes('/images/thumbnails/status')) statusReads.push(req.url()); });
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2);
|
||||
await page.evaluate(type => window.dispatchEvent(new Event(type)), leave);
|
||||
await page.clock.runFor(1500);
|
||||
// A read made after the clock ran is listed, so the listener saw every status read.
|
||||
await page.evaluate(() => fetch('/prompt_manager/images/thumbnails/status?probe=1'));
|
||||
expect(statusReads.map(url => new URL(url).search)).toEqual(['?probe=1']);
|
||||
});
|
||||
}
|
||||
|
||||
test('actual gallery review generates and persists tags for the selected root', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2);
|
||||
@@ -64,11 +120,16 @@ test('actual Tags page reloads the selected filter and opens its image viewer',
|
||||
await page.goto('/prompt_manager/admin#/tags');
|
||||
await page.locator('[data-tag-name="cats"]').click();
|
||||
await expect(page.locator('#tagPromptsGrid')).toContainText('Cat portrait');
|
||||
const dogs = page.waitForResponse(response => response.url().includes('/tags/dogs/prompts?'));
|
||||
await page.locator('[data-tag-name="dogs"]').click();
|
||||
const [shown] = (await (await dogs).json()).prompts[0].images;
|
||||
await expect(page.locator('#tagPromptsGrid')).toContainText('Dog landscape');
|
||||
await expect(page.locator('#tagPromptsGrid')).not.toContainText('Cat portrait');
|
||||
await page.locator('#tagPromptsGrid img').first().click();
|
||||
await expect(page.locator('.viewer-canvas img')).toBeVisible();
|
||||
// The viewer's title names the shown image through the page's title callback. ViewerJS renders it only once the
|
||||
// image is viewed, so closing at "visible" alone left the callback unrun on slower hosts (CI coverage missed it).
|
||||
await expect(page.locator('.viewer-title')).toHaveText(shown.filename);
|
||||
await page.locator('.viewer-container').getByRole('button', { name: 'Close', exact: true }).click();
|
||||
await expect(page.locator('.viewer-container')).toHaveCount(0);
|
||||
});
|
||||
@@ -300,39 +361,48 @@ test('prompt deletion requires confirmation and can retry after a network failur
|
||||
}
|
||||
});
|
||||
|
||||
test('duplicate cleanup cancels safely and deletes only the selected file after a retry', async ({ page, request }) => {
|
||||
// The duplicate scan reads the output root only, but the fixture's input root holds a different
|
||||
// shared/same.png. A bare /images/serve/<path> URL tries every gallery root in turn, so once the
|
||||
// output copy is deleted it answers 200 with the input copy: check each file in its own root.
|
||||
const serveIn = (root, path) => `/prompt_manager/images/serve/${path.split(/[\\/]/).map(encodeURIComponent).join('/')}?root=${root}`;
|
||||
const rootOf = url => new URL(url, 'http://gallery.test').searchParams.get('root');
|
||||
|
||||
// Which copy the scan keeps depends on file times: the fixture stamps shared/same.png with the
|
||||
// wall clock, and the kernel can stamp its bulk copies a fraction of a millisecond earlier.
|
||||
// listOrder rewrites the finished scan's groups so a test can pick the order it needs.
|
||||
async function openDuplicateScan(page, request, listOrder = null) {
|
||||
await expect(await request.post('/__test__/reset', { data: { extra_images: 2 } })).toBeOK();
|
||||
if (listOrder) {
|
||||
await page.route('**/prompt_manager/scan_duplicates/stream', async route => {
|
||||
const response = await route.fetch();
|
||||
const body = (await response.text()).replace(/^data: (.*)$/gm, (line, json) => {
|
||||
const event = JSON.parse(json);
|
||||
if (event.type !== 'complete') return line;
|
||||
for (const group of event.duplicates) group.images = listOrder(group.images);
|
||||
return `data: ${JSON.stringify(event)}`;
|
||||
});
|
||||
await route.fulfill({ response, body });
|
||||
});
|
||||
}
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(4);
|
||||
const originals = await page.locator('#galleryGrid img[data-original]').evaluateAll(images => images.map(image => image.dataset.original));
|
||||
const output = rootOf(originals.find(url => url.includes('/bulk/')));
|
||||
const input = rootOf(originals.find(url => url.includes('/shared/same.png') && rootOf(url) !== output));
|
||||
expect(output).not.toBeNull();
|
||||
expect(input).not.toBeNull();
|
||||
await page.locator('#settingsBtn').click();
|
||||
await page.locator('#scanDuplicatesBtn').click();
|
||||
await expect(page.locator('#duplicatesCount')).toHaveText('1');
|
||||
const candidates = page.locator('.duplicate-checkbox');
|
||||
await expect(candidates).toHaveCount(2);
|
||||
const selected = await candidates.first().getAttribute('data-image-path');
|
||||
const retained = await candidates.last().getAttribute('data-image-path');
|
||||
const serve = path => `/prompt_manager/images/serve/${path.split(/[\\/]/).map(encodeURIComponent).join('/')}`;
|
||||
await candidates.first().check();
|
||||
let deletes = 0;
|
||||
page.on('request', request => { if (request.url().endsWith('/delete_duplicate_images')) deletes++; });
|
||||
confirmNextDialog(page, false, 'permanently delete 1 duplicate image files');
|
||||
await page.locator('#removeDuplicatesBtn').click();
|
||||
expect(deletes).toBe(0);
|
||||
await expect(await request.get(serve(selected))).toBeOK();
|
||||
await page.route('**/prompt_manager/delete_duplicate_images', route => route.abort('failed'), { times: 1 });
|
||||
confirmNextDialog(page, true, 'permanently delete 1 duplicate image files');
|
||||
await page.locator('#removeDuplicatesBtn').click();
|
||||
await expect(page.getByText('Failed to delete duplicate images', { exact: true })).toBeVisible();
|
||||
await expect(page.locator('#removeDuplicatesBtn')).toBeEnabled();
|
||||
await expect(await request.get(serve(selected))).toBeOK();
|
||||
confirmNextDialog(page, true, 'permanently delete 1 duplicate image files');
|
||||
const deleted = page.waitForResponse(response => response.url().endsWith('/delete_duplicate_images'));
|
||||
await page.locator('#removeDuplicatesBtn').click();
|
||||
expect((await (await deleted).json())).toMatchObject({ success: true, deleted_count: 1, failed_count: 0 });
|
||||
await expect(page.locator('#duplicatesModal')).toBeHidden();
|
||||
expect((await request.get(serve(selected))).status()).toBe(404);
|
||||
await expect(await request.get(serve(retained))).toBeOK();
|
||||
return { originals, output, input, candidates };
|
||||
}
|
||||
|
||||
async function expectOnlySelectedGone(page, request, { originals, output, input }, selected, retained) {
|
||||
expect((await request.get(serveIn(output, selected))).status()).toBe(404);
|
||||
await expect(await request.get(serveIn(output, retained))).toBeOK();
|
||||
await expect(await request.get(serveIn(input, 'shared/same.png'))).toBeOK();
|
||||
await page.reload();
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(3);
|
||||
const remaining = await page.locator('#galleryGrid img[data-original]').evaluateAll(images => images.map(image => image.dataset.original));
|
||||
@@ -341,4 +411,47 @@ test('duplicate cleanup cancels safely and deletes only the selected file after
|
||||
expect(originals).toContain(url);
|
||||
await expect(await request.get(url)).toBeOK();
|
||||
}
|
||||
}
|
||||
|
||||
test('duplicate cleanup cancels safely and deletes only the selected file after a retry', async ({ page, request }) => {
|
||||
const scan = await openDuplicateScan(page, request);
|
||||
const { output, candidates } = scan;
|
||||
const selected = await candidates.first().getAttribute('data-image-path');
|
||||
const retained = await candidates.last().getAttribute('data-image-path');
|
||||
await candidates.first().check();
|
||||
let deletes = 0;
|
||||
page.on('request', request => { if (request.url().endsWith('/delete_duplicate_images')) deletes++; });
|
||||
confirmNextDialog(page, false, 'permanently delete 1 duplicate image files');
|
||||
await page.locator('#removeDuplicatesBtn').click();
|
||||
expect(deletes).toBe(0);
|
||||
await expect(await request.get(serveIn(output, selected))).toBeOK();
|
||||
await page.route('**/prompt_manager/delete_duplicate_images', route => route.abort('failed'), { times: 1 });
|
||||
confirmNextDialog(page, true, 'permanently delete 1 duplicate image files');
|
||||
await page.locator('#removeDuplicatesBtn').click();
|
||||
await expect(page.getByText('Failed to delete duplicate images', { exact: true })).toBeVisible();
|
||||
await expect(page.locator('#removeDuplicatesBtn')).toBeEnabled();
|
||||
await expect(await request.get(serveIn(output, selected))).toBeOK();
|
||||
confirmNextDialog(page, true, 'permanently delete 1 duplicate image files');
|
||||
const deleted = page.waitForResponse(response => response.url().endsWith('/delete_duplicate_images'));
|
||||
await page.locator('#removeDuplicatesBtn').click();
|
||||
expect((await (await deleted).json())).toMatchObject({ success: true, deleted_count: 1, failed_count: 0 });
|
||||
await expect(page.locator('#duplicatesModal')).toBeHidden();
|
||||
await expectOnlySelectedGone(page, request, scan, selected, retained);
|
||||
});
|
||||
|
||||
test('deleting a duplicate whose path another gallery root also holds removes only the scanned copy', async ({ page, request }) => {
|
||||
// The order CI saw: a bulk copy stamped first, so shared/same.png is the first Delete candidate.
|
||||
const ciOrder = ['bulk/0000.png', 'shared/same.png', 'bulk/0001.png'];
|
||||
const rank = image => ciOrder.indexOf(image.path.replace(/\\/g, '/'));
|
||||
const scan = await openDuplicateScan(page, request, images => [...images].sort((a, b) => rank(a) - rank(b)));
|
||||
const selected = await scan.candidates.first().getAttribute('data-image-path');
|
||||
const retained = await scan.candidates.last().getAttribute('data-image-path');
|
||||
expect(selected.replace(/\\/g, '/')).toBe('shared/same.png');
|
||||
await scan.candidates.first().check();
|
||||
confirmNextDialog(page, true, 'permanently delete 1 duplicate image files');
|
||||
const deleted = page.waitForResponse(response => response.url().endsWith('/delete_duplicate_images'));
|
||||
await page.locator('#removeDuplicatesBtn').click();
|
||||
expect((await (await deleted).json())).toMatchObject({ success: true, deleted_count: 1, failed_count: 0 });
|
||||
await expect(page.locator('#duplicatesModal')).toBeHidden();
|
||||
await expectOnlySelectedGone(page, request, scan, selected, retained);
|
||||
});
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs';
|
||||
|
||||
// The Activity store's periodic list read (activity-store.mjs POLL_MS) and the detail view's read throttle
|
||||
// (activity-job-view.mjs DETAIL_INTERVAL_MS).
|
||||
const POLL_MS = 15_000;
|
||||
const DETAIL_INTERVAL_MS = 1_000;
|
||||
// A request is reported to the test a little after the browser sends it; that delay is not a shorter wait.
|
||||
const REPORT_SLACK_MS = 100;
|
||||
|
||||
// Activity (#249 Activity-1): the concept's Activity page (app.js renderActivity 732–738), its top-bar button
|
||||
// (index.html 51–52) and sidebar row (app.js ~341), with the recorded D52 (counts of running jobs), D53 (the progress
|
||||
// pill) and D54 (real job items). Fake jobs come from tests/browser/workspace_activity_fixture.py.
|
||||
@@ -47,8 +54,8 @@ async function step(request, id, data) {
|
||||
await expect(await request.post(`/__test__/jobs/${id}/step`, { data })).toBeOK();
|
||||
}
|
||||
|
||||
async function finish(request, id, state = 'finished') {
|
||||
const response = await request.post(`/__test__/jobs/${id}/finish`, { data: { state } });
|
||||
async function finish(request, id, state = 'finished', extra = {}) {
|
||||
const response = await request.post(`/__test__/jobs/${id}/finish`, { data: { state, ...extra } });
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).job;
|
||||
}
|
||||
@@ -277,7 +284,22 @@ test('reload during a scan reattaches to the running job', async ({ page, reques
|
||||
await expect(pill(page)).toBeHidden();
|
||||
});
|
||||
|
||||
test('an empty Activity says no job has run, and a list that cannot be read says so', async ({ page }) => {
|
||||
// The empty and failed states were only reached in passing by other tests, so their coverage depended on timing.
|
||||
const empty = main(page).locator('section.surface > p.empty-state');
|
||||
await page.goto(`${workspace}#/activity`);
|
||||
await expect(empty).toHaveText('No jobs have run since ComfyUI started. Scans and other long operations appear here.');
|
||||
await expect(empty).not.toHaveAttribute('role');
|
||||
await expect(items(page)).toHaveCount(0);
|
||||
await page.route(url => url.pathname === '/prompt_manager/jobs', route => route.abort());
|
||||
await page.reload();
|
||||
await expect(empty).toHaveText('Activity could not be read. It is tried again shortly.');
|
||||
await expect(empty).toHaveAttribute('role', 'alert');
|
||||
});
|
||||
|
||||
test('lost server clears the pill and marks jobs ended', async ({ page, request }) => {
|
||||
// A poll interval after the restart, then failed re-reads backing off for 1 + 2 + 4 s.
|
||||
test.setTimeout(90_000);
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 4000 });
|
||||
@@ -294,9 +316,12 @@ test('lost server clears the pill and marks jobs ended', async ({ page, request
|
||||
await expect(items(page).first().locator('p')).toHaveText(LOST);
|
||||
await expect(items(page).first().locator('progress, .row-actions')).toHaveCount(0);
|
||||
await expect(navRow(page).locator('.nav-count')).toHaveCount(0);
|
||||
// One re-read found the restart; there is no retry storm and no finish toast for a job that only vanished.
|
||||
await page.waitForTimeout(2500);
|
||||
expect(reads.length).toBeLessThanOrEqual(2);
|
||||
// One re-read found the restart; there is no retry storm and no finish toast for a job that only vanished. The
|
||||
// read after it is the periodic one, a whole poll interval later: any extra read lands in that gap and fails it.
|
||||
const found = reads.length;
|
||||
expect(found, 'reads that found the restart').toBeGreaterThanOrEqual(1);
|
||||
await expect.poll(() => reads.length, { timeout: POLL_MS + 10_000 }).toBeGreaterThan(found);
|
||||
expect(reads[found] - reads[found - 1], 'the next read waits for the poll').toBeGreaterThanOrEqual(POLL_MS - REPORT_SLACK_MS);
|
||||
await expect(toast(page)).toHaveText('');
|
||||
|
||||
// A server that cannot be read at all also ends the running job, then is read again with a growing backoff.
|
||||
@@ -308,9 +333,12 @@ test('lost server clears the pill and marks jobs ended', async ({ page, request
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(pill(page)).toBeHidden();
|
||||
await expect(items(page).filter({ hasText: LOST })).toHaveCount(1);
|
||||
await page.waitForTimeout(3500);
|
||||
// 1 s, then 2 s: at most two re-reads in 3.5 s.
|
||||
expect(failed.length).toBeLessThanOrEqual(2);
|
||||
// 1 s, then 2 s, then 4 s: each failed re-read waits twice as long as the one before. The gaps between the
|
||||
// reads themselves are the evidence, so a slow runner noticing the ended job late cannot add a read to a window.
|
||||
await expect.poll(() => failed.length, { timeout: 20_000 }).toBeGreaterThanOrEqual(3);
|
||||
const gaps = failed.slice(1, 3).map((at, index) => at - failed[index]);
|
||||
expect(gaps[0]).toBeGreaterThanOrEqual(2000 - REPORT_SLACK_MS);
|
||||
expect(gaps[1]).toBeGreaterThanOrEqual(4000 - REPORT_SLACK_MS);
|
||||
await page.unroute('**/prompt_manager/jobs**');
|
||||
expect(second.id).not.toBe(job.id);
|
||||
});
|
||||
@@ -392,16 +420,69 @@ test('start scan creates a job, toasts, and opens its view', async ({ page, requ
|
||||
await expect(scanLog(page).locator('pre')).toContainText('Scan finished');
|
||||
});
|
||||
|
||||
// ── Task 4 (Activity-2): scan scope, new and unresolved files by default ─────────────────────
|
||||
|
||||
test('all files checkbox changes the scope line and the request', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const bodies = [];
|
||||
page.on('request', item => {
|
||||
if (item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/jobs/scan') bodies.push(item.postDataJSON());
|
||||
});
|
||||
const review = page.getByRole('dialog', { name: 'Review metadata scan', exact: true });
|
||||
const scopeValue = review.locator('.meta-grid > div').filter({ has: page.locator('dt', { hasText: 'Scope' }) }).locator('dd');
|
||||
// The concept's switch: a checkbox with role=switch.
|
||||
const allFiles = review.getByRole('switch', { name: 'All files', exact: true });
|
||||
const openReview = async () => {
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await page.getByRole('button', { name: 'Import', exact: true }).click();
|
||||
await page.getByRole('dialog', { name: 'Import & inspect', exact: true }).locator('.operation')
|
||||
.filter({ hasText: 'Scan for metadata' }).getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(review).toBeVisible();
|
||||
};
|
||||
|
||||
// The default: the concept's "New and unresolved files", the box unchecked; the request asks for "new".
|
||||
await openReview();
|
||||
await expect(scopeValue).toHaveText('New and unresolved files');
|
||||
await expect(allFiles).not.toBeChecked();
|
||||
await expect(review.locator('.setting-row p')).toHaveText('Also read the files already linked to their prompt again.');
|
||||
await expect(allFiles).toHaveAccessibleDescription('Also read the files already linked to their prompt again.');
|
||||
// Checking and unchecking switches the Scope value both ways.
|
||||
await allFiles.check();
|
||||
await expect(scopeValue).toHaveText('All files');
|
||||
await allFiles.uncheck();
|
||||
await expect(scopeValue).toHaveText('New and unresolved files');
|
||||
await review.getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
await expect.poll(() => bodies.length).toBe(1);
|
||||
expect(bodies[0]).toEqual({ scope: 'new' });
|
||||
await expect(heading(page).locator('h1')).toHaveText('Scan for metadata');
|
||||
// Wait for the first scan to end so the second is not refused as busy.
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toContainText('Scan finished', { timeout: 15000 });
|
||||
await expect(scanLog(page).locator('pre')).toContainText('new and unresolved files');
|
||||
|
||||
// Checked: "All files", and the request asks for "all".
|
||||
await openReview();
|
||||
await allFiles.check();
|
||||
await expect(scopeValue).toHaveText('All files');
|
||||
await review.getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
await expect.poll(() => bodies.length).toBe(2);
|
||||
expect(bodies[1]).toEqual({ scope: 'all' });
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toContainText('Scan started · ', { timeout: 15000 });
|
||||
await expect(scanLog(page).locator('pre')).toContainText('all files');
|
||||
});
|
||||
|
||||
test('job view counters follow the mapping and update live', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 8240, counters: { added: 1420, known: 6511, review: 37, no_metadata: 254, errors: 18 } });
|
||||
await openJob(page, job.id);
|
||||
// The header: the title, the detail line, then Download report (Task 5) and Cancel scan (danger).
|
||||
// The header: the title, the detail line, then Download report (enabled in Task 5) and Cancel scan (danger).
|
||||
await expect(heading(page).locator('h1')).toHaveText('Scan for metadata');
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Started \d\d:\d\d · 2 output folders · closing this view does not cancel it$/);
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report', 'Cancel scan']);
|
||||
await expect(heading(page).getByRole('button', { name: 'Download report', exact: true })).toBeDisabled();
|
||||
await expect(heading(page).getByRole('button', { name: 'Download report', exact: true })).toBeEnabled();
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).toHaveClass(/danger/);
|
||||
// Progress: the bar, the server's line and the five tiles in order and colour.
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('Processing file 8,240 of 19,622');
|
||||
@@ -554,10 +635,25 @@ test('filter narrows the selected outcome', async ({ page, request }) => {
|
||||
|
||||
test('job view renders 200-row cap note', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
// The page's timers are Playwright's, running at real speed, so the M12 check below can run a read throttle out.
|
||||
await page.clock.install();
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 260, bulk: { group: 'added', count: 250 }, records: [
|
||||
...Array.from({ length: 10 }, (_, index) => ({ group: 'no_metadata', path: `misc/plain_${index}.png`, detail: 'No prompt metadata in the file' })),
|
||||
] });
|
||||
// Every detail read of this job: in flight, and whether one has returned the full listing.
|
||||
const detailUrl = `/prompt_manager/jobs/${job.id}`;
|
||||
const isDetail = item => new URL(item.url()).pathname === detailUrl;
|
||||
const detailReads = { inFlight: 0, full: false };
|
||||
page.on('request', item => { if (isDetail(item)) detailReads.inFlight += 1; });
|
||||
page.on('requestfailed', item => { if (isDetail(item)) detailReads.inFlight -= 1; });
|
||||
page.on('response', async response => {
|
||||
if (!isDetail(response)) return;
|
||||
const body = await response.json().catch(() => null);
|
||||
const outcomes = body?.job?.outcomes ?? {};
|
||||
if (outcomes.added?.length === 200 && outcomes.no_metadata?.length === 10) detailReads.full = true;
|
||||
detailReads.inFlight -= 1;
|
||||
});
|
||||
await openJob(page, job.id);
|
||||
await expect(tiles(page).first().locator('dd')).toHaveText('250');
|
||||
await expect(fileRows(page)).toHaveCount(200);
|
||||
@@ -573,15 +669,22 @@ test('job view renders 200-row cap note', async ({ page, request }) => {
|
||||
|
||||
// Review M12: with the list capped and the log closed, more files only update the counters; the detail is read
|
||||
// again only when the log opens with new lines.
|
||||
// The reads from opening the view have settled: one returned the full listing and none is in flight. The view
|
||||
// re-reads only while what the summary lists differs from the last read, so no further read is due.
|
||||
await expect.poll(() => detailReads.full && detailReads.inFlight === 0).toBe(true);
|
||||
const reads = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === `/prompt_manager/jobs/${job.id}`) reads.push(Date.now()); });
|
||||
await step(request, job.id, { processed: 300, bulk: { group: 'added', count: 40 } });
|
||||
await expect(tiles(page).first().locator('dd')).toHaveText('290');
|
||||
await page.waitForTimeout(1500);
|
||||
// Every read the counter update could have scheduled is due within one throttle interval: run it out, so any such
|
||||
// read is sent before the log opens. Requests reach the test in the order they were sent, so once the log's read
|
||||
// has answered, a read sent before it would already be listed.
|
||||
await page.clock.runFor(DETAIL_INTERVAL_MS);
|
||||
expect(reads).toEqual([]);
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect.poll(() => reads.length).toBe(1);
|
||||
await expect(scanLog(page).locator('pre')).toContainText('bulk/file_00039.png');
|
||||
expect(reads, 'only the log\'s own read').toHaveLength(1);
|
||||
});
|
||||
|
||||
test('cancel scan ends cancelled with partial counts', async ({ page, request }) => {
|
||||
@@ -609,16 +712,80 @@ test('cancel scan ends cancelled with partial counts', async ({ page, request })
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).not.toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
|
||||
// Truth rule: a held legacy-page job (a restore from the admin page) cannot be cancelled, so neither the Activity
|
||||
// list nor the job's view offers Cancel; the cancellable scan beside it keeps its own (review I3).
|
||||
test('a held legacy job shows no Cancel in the list or its view', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const response = await request.post('/__test__/jobs/hold', { data: { kind: 'restore', title: 'Database restore (admin page)' } });
|
||||
await expect(response).toBeOK();
|
||||
const held = (await response.json()).job;
|
||||
expect(held.cancellable).toBe(false);
|
||||
await page.goto(`${workspace}#/activity`);
|
||||
const item = items(page).filter({ has: page.getByRole('link', { name: /Database restore \(admin page\)$/ }) });
|
||||
await expect(item.locator('.chip')).toHaveText('Running');
|
||||
await expect(item.locator('progress')).toHaveCount(1);
|
||||
await expect(main(page).getByRole('button', { name: 'Cancel job', exact: true })).toHaveCount(0);
|
||||
|
||||
await item.getByRole('link').click();
|
||||
await expect(heading(page).locator('h1')).toHaveText('Database restore (admin page)');
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report']);
|
||||
await expect(heading(page).getByRole('button', { name: /^Cancel/ })).toHaveCount(0);
|
||||
|
||||
// When the hold ends the view shows the end, still without a Cancel.
|
||||
await finish(request, held.id);
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Finished/);
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report']);
|
||||
});
|
||||
|
||||
// Since #289 an item dropped its Cancel job only when it was already in the page, so the first render of jobs the
|
||||
// store already knew (Activity opened from another page) kept a Cancel job on a job that had ended, and moved a
|
||||
// running job's progress after its actions. The read Activity starts on opening is held, so that first render is
|
||||
// what the page shows (the read's own render would otherwise repair it a moment later).
|
||||
test('jobs that ended before Activity opened show no Cancel job, a running one keeps its progress first', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const finished = await startJob(request, { title: 'Finished scan' });
|
||||
await finish(request, finished.id);
|
||||
const cancelled = await startJob(request, { title: 'Cancelled scan' });
|
||||
await finish(request, cancelled.id, 'cancelled');
|
||||
await startJob(request, { title: 'Running scan' });
|
||||
// The page's first list read has all three jobs once the running one shows in the pill.
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(pill(page)).toBeVisible();
|
||||
let hold;
|
||||
const held = new Promise(resolve => { hold = resolve; });
|
||||
// Only the first read is held; any later one goes through.
|
||||
await page.route(url => url.pathname === '/prompt_manager/jobs', route => {
|
||||
if (!hold) return route.continue();
|
||||
hold(route);
|
||||
hold = null;
|
||||
});
|
||||
await activityButton(page).click();
|
||||
const route = await held;
|
||||
await expect(items(page)).toHaveCount(3);
|
||||
const item = title => items(page).filter({ has: page.getByRole('link', { name: new RegExp(`${title}$`) }) });
|
||||
for (const ended of ['Finished scan', 'Cancelled scan']) {
|
||||
await expect(item(ended).locator('progress, .row-actions')).toHaveCount(0);
|
||||
await expect(item(ended).getByRole('button', { name: 'Cancel job', exact: true })).toHaveCount(0);
|
||||
}
|
||||
await expect(item('Running scan').getByRole('button', { name: 'Cancel job', exact: true })).toHaveCount(1);
|
||||
expect(await item('Running scan').evaluate(node => [...node.children].map(child => child.localName + (child.className ? `.${child.className}` : ''))))
|
||||
.toEqual(['div.panel-heading', 'p', 'progress', 'div.row-actions']);
|
||||
await route.continue();
|
||||
});
|
||||
|
||||
test('busy start names the running job', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
// The page's timers are Playwright's, so the toast's lifetime below is run out rather than waited for.
|
||||
await page.clock.install();
|
||||
const running = await startJob(request, { title: 'Database maintenance', kind: 'maintenance' });
|
||||
await startFromImport(page);
|
||||
await expect(toast(page)).toHaveText('Another job is running: Database maintenance.');
|
||||
await expect(toast(page).getByRole('link', { name: 'Database maintenance', exact: true })).toHaveAttribute('href', `#/activity/${running.id}`);
|
||||
await expect(page).toHaveURL(/#\/images$/);
|
||||
expect((await listed(request)).map(item => item.id)).toEqual([running.id]);
|
||||
// Review M14: a toast with a link outlasts the plain 6.5 s, so the link stays reachable.
|
||||
await page.waitForTimeout(7000);
|
||||
// Review M14: a toast with a link outlasts the plain 6.5 s, so the link stays reachable. Running the page's timers
|
||||
// 7 s on fires a 6.5 s clear for certain, however slowly this runner reached this line.
|
||||
await page.clock.runFor(7000);
|
||||
await expect(toast(page).getByRole('link', { name: 'Database maintenance', exact: true })).toBeVisible();
|
||||
await toast(page).getByRole('link', { name: 'Database maintenance', exact: true }).click();
|
||||
await expect(heading(page).locator('h1')).toHaveText('Database maintenance');
|
||||
@@ -639,6 +806,8 @@ test('an unknown or restarted job view says so', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(LOST);
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).toHaveCount(0);
|
||||
// A lost job has no report on the server (review M9): Download report is disabled rather than failing.
|
||||
await expect(heading(page).getByRole('button', { name: 'Download report', exact: true })).toBeDisabled();
|
||||
// What was read before the restart stays on the page.
|
||||
await expect(fileRows(page)).toHaveCount(1);
|
||||
await expect(pill(page)).toBeHidden();
|
||||
@@ -662,3 +831,273 @@ test('a finished job the server pruned leaves its open view honest', async ({ pa
|
||||
await expect(heading(page).locator('h1')).toBeFocused();
|
||||
await expect(page).toHaveTitle('Activity · Prompt Manager');
|
||||
});
|
||||
|
||||
// Activity-2 Task 5 (D59): Download report and the log's scroll position.
|
||||
test('download report saves the job JSON', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 2, records: [
|
||||
{ group: 'added', path: 'sub/a.png', root: 'output', detail: 'Added' },
|
||||
{ group: 'errors', path: 'broken/b.png', root: 'output', detail: 'Error' },
|
||||
] });
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
const button = heading(page).getByRole('button', { name: 'Download report', exact: true });
|
||||
await expect(button).toBeEnabled();
|
||||
const [download] = await Promise.all([page.waitForEvent('download'), button.click()]);
|
||||
expect(download.suggestedFilename()).toMatch(/^prompt-manager-scan-\d{8}-\d{4}\.json$/);
|
||||
const path = await download.path();
|
||||
const report = JSON.parse(await (await import('node:fs/promises')).readFile(path, 'utf8'));
|
||||
expect(report.job.id).toBe(job.id);
|
||||
expect(report.outcomes.added.rows.map(row => row.path)).toEqual(['sub/a.png']);
|
||||
expect(report.outcomes.errors.truncated).toBe(false);
|
||||
expect(report.log.length).toBeGreaterThan(0);
|
||||
await expect(toast(page)).toHaveText('Job report downloaded.');
|
||||
});
|
||||
|
||||
test('log keeps the reader\'s scroll position', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
const more = (from, count) => Array.from({ length: count }, (_, index) => ({ group: 'known', path: `log/f_${from + index}.png`,
|
||||
root: 'output', detail: 'Known' }));
|
||||
await step(request, job.id, { processed: 60, records: more(0, 60) });
|
||||
await openJob(page, job.id);
|
||||
await scanLog(page).locator('summary').click();
|
||||
const pre = scanLog(page).locator('pre');
|
||||
await expect(pre).toContainText('log/f_59.png');
|
||||
const metrics = () => pre.evaluate(node => ({ top: node.scrollTop, max: node.scrollHeight - node.clientHeight }));
|
||||
// Opening starts at the newest line.
|
||||
await expect.poll(async () => { const m = await metrics(); return m.max > 0 && m.top >= m.max - 2; }).toBe(true);
|
||||
// Pinned to the bottom: new lines are followed.
|
||||
await step(request, job.id, { processed: 90, records: more(60, 30) });
|
||||
await expect(pre).toContainText('log/f_89.png');
|
||||
await expect.poll(async () => { const m = await metrics(); return m.top >= m.max - 2; }).toBe(true);
|
||||
// Scrolled up: the position stays while more lines arrive.
|
||||
await pre.evaluate(node => { node.scrollTop = 40; });
|
||||
await step(request, job.id, { processed: 120, records: more(90, 30) });
|
||||
await expect(pre).toContainText('log/f_119.png');
|
||||
expect((await metrics()).top).toBe(40);
|
||||
// Closed and opened again (review M6): every open starts at the newest line, also after lines arrived meanwhile.
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page)).not.toHaveAttribute('open', '');
|
||||
await step(request, job.id, { processed: 150, records: more(120, 30) });
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(pre).toContainText('log/f_149.png');
|
||||
await expect.poll(async () => { const m = await metrics(); return m.max > 0 && m.top >= m.max - 2; }).toBe(true);
|
||||
});
|
||||
|
||||
test('a failed report download says so and leaves the button usable', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
await page.route(`**/prompt_manager/jobs/${job.id}/report`, route => route.fulfill({ status: 500, body: 'no' }));
|
||||
const button = heading(page).getByRole('button', { name: 'Download report', exact: true });
|
||||
await button.click();
|
||||
await expect(toast(page)).toHaveText('The report could not be downloaded. Try again.');
|
||||
await expect(button).not.toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
|
||||
// ── #249 PR D Task D3 (D89, plan D66): image tagging jobs in Activity ─────────────────────────
|
||||
|
||||
const TAG_JOB = { kind: 'image-tags', title: 'Auto-tag images', mode: 'auto', images: 4, total: 4 };
|
||||
|
||||
test('an image tagging job shows Tagged Unchanged Skipped Errors and opens tagged rows', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request, TAG_JOB);
|
||||
await step(request, job.id, { processed: 3, records: [
|
||||
{ group: 'tagged', path: 'shared/same.png', root: 'output', detail: 'Added 2 tags' },
|
||||
{ group: 'unchanged', path: 'shared/same.png', root: 'input', detail: 'No new tags' },
|
||||
{ group: 'skipped', path: 'video/clip_0004.webm', root: 'output', detail: 'Video: manual tags only' },
|
||||
] });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
// The pill names the kind's verb.
|
||||
await expect(pill(page)).toHaveText('Tagging · 75% · 3 / 4');
|
||||
await openJob(page, job.id);
|
||||
await expect(heading(page).locator('h1')).toHaveText('Auto-tag images');
|
||||
await expect(page).toHaveTitle('Auto-tag images · Prompt Manager');
|
||||
// The header's scope is the number of images, and the job cancels as "Cancel tagging" (danger).
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Started \d\d:\d\d · 4 images · closing this view does not cancel it$/);
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report', 'Cancel tagging']);
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel tagging', exact: true })).toHaveClass(/danger/);
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('Processing file 3 of 4');
|
||||
await expect(progressSection(page).locator('progress')).toHaveAttribute('aria-label', 'Auto-tag images progress');
|
||||
// Four tiles in the scan's tones: Tagged green, Unchanged plain, Skipped warning, Errors red.
|
||||
expect(await tileValues(page)).toEqual([
|
||||
['stat added', 'Tagged', '1'], ['stat', 'Unchanged', '1'], ['stat warn', 'Skipped', '1'], ['stat bad', 'Errors', '0'],
|
||||
]);
|
||||
// What got tagged and what went wrong first, as the scan's list does.
|
||||
await expect(outcomeButtons(page)).toHaveText(['Tagged1', 'Errors0', 'Skipped1', 'Unchanged1']);
|
||||
await expect(outcomeButtons(page).first()).toHaveAttribute('aria-pressed', 'true');
|
||||
|
||||
// Tagged rows open the viewer on that file (D57's identity).
|
||||
const tagged = fileRows(page).first();
|
||||
expect(await tagged.evaluate(node => node.localName)).toBe('button');
|
||||
await expect(tagged.locator('.path')).toHaveText('shared/same.png');
|
||||
await expect(tagged.locator('.why')).toHaveText('Added 2 tags');
|
||||
await expect(resultsNote(page)).toHaveText('Each row opens the image in the viewer.');
|
||||
await tagged.click();
|
||||
const viewer = page.locator('dialog#viewer-dialog[open]');
|
||||
await expect(viewer.locator('#viewer-stage img').first()).toHaveAttribute('src', /[?&]root=0(&|$)/);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer).toHaveCount(0);
|
||||
await expect(tagged).toBeFocused();
|
||||
// Unchanged rows open too, here the input root's file.
|
||||
await outcomeButtons(page).filter({ hasText: 'Unchanged' }).click();
|
||||
await fileRows(page).first().click();
|
||||
await expect(page.locator('dialog#viewer-dialog[open] #viewer-stage img').first()).toHaveAttribute('src', /[?&]root=1(&|$)/);
|
||||
await page.keyboard.press('Escape');
|
||||
// Skipped rows are plain text with their reason.
|
||||
await outcomeButtons(page).filter({ hasText: 'Skipped' }).click();
|
||||
expect(await fileRows(page).first().evaluate(node => node.localName)).toBe('div');
|
||||
await expect(fileRows(page).first().locator('.why')).toHaveText('Video: manual tags only');
|
||||
await expect(resultsNote(page)).toHaveCount(0);
|
||||
// The log is the job's own: "Tagging log", with the runner's prefixes.
|
||||
await expect(scanLog(page).locator('summary')).toHaveText('● Tagging log — raw detail, click to show');
|
||||
await scanLog(page).locator('summary').click();
|
||||
await expect(scanLog(page).locator('pre')).toContainText('+ shared/same.png · Added 2 tags');
|
||||
|
||||
// An error, then the end: the header and the Activity item read "N tagged · M errors".
|
||||
await step(request, job.id, { processed: 4, records: [
|
||||
{ group: 'errors', path: 'broken/truncated.png', root: 'output', detail: 'Auto-tagging failed. Check server logs for details.' },
|
||||
] });
|
||||
await expect(tiles(page).last().locator('dd')).toHaveText('1');
|
||||
await finish(request, job.id);
|
||||
await expect(toast(page)).toHaveText('Auto-tag images finished · 1 tagged · 1 error');
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Finished \d\d:\d\d · 1 tagged · 1 error$/);
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report']);
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('4 of 4 files processed');
|
||||
await page.goto(`${workspace}#/activity`);
|
||||
await expect(items(page).first().locator('h3')).toHaveText('✓ Auto-tag images');
|
||||
await expect(items(page).first().locator('p')).toHaveText(/^Finished \d\d:\d\d · 1 tagged · 1 error$/);
|
||||
});
|
||||
|
||||
test('cancel tagging stops at a file boundary', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request, { ...TAG_JOB, title: 'Add tags to images', mode: 'add', images: 1250, total: 1250 });
|
||||
await step(request, job.id, { processed: 2, counters: { tagged: 2 } });
|
||||
await page.goto(`${workspace}#/activity`);
|
||||
await expect(items(page).first().locator('p')).toHaveText(/^Started \d\d:\d\d · 1,250 images · closing this view does not cancel it$/);
|
||||
await openJob(page, job.id);
|
||||
await heading(page).getByRole('button', { name: 'Cancel tagging', exact: true }).focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(toast(page)).toHaveText('Add tags to images cancelled · 2 tagged');
|
||||
await expect(heading(page).locator('h1')).toBeFocused();
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Cancelled \d\d:\d\d at a file boundary · 2 tagged · completed work is kept$/);
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel tagging', exact: true })).toHaveCount(0);
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('Cancelled after 2 of 1,250 files');
|
||||
expect((await tileValues(page)).map(([, , value]) => value)).toEqual(['2', '0', '0', '0']);
|
||||
expect((await listed(request))[0].state).toBe('cancelled');
|
||||
});
|
||||
|
||||
test('the report download names the kind', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request, TAG_JOB);
|
||||
await step(request, job.id, { processed: 1, records: [
|
||||
{ group: 'tagged', path: 'sub/a.png', root: 'output', detail: 'Added 1 tag' },
|
||||
] });
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
const button = heading(page).getByRole('button', { name: 'Download report', exact: true });
|
||||
const [download] = await Promise.all([page.waitForEvent('download'), button.click()]);
|
||||
expect(download.suggestedFilename()).toMatch(/^prompt-manager-image-tags-\d{8}-\d{4}\.json$/);
|
||||
const report = JSON.parse(await (await import('node:fs/promises')).readFile(await download.path(), 'utf8'));
|
||||
expect(report.job.kind).toBe('image-tags');
|
||||
expect(report.job.scope).toEqual({ mode: 'auto', images: 4 });
|
||||
expect(Object.keys(report.outcomes)).toEqual(['tagged', 'unchanged', 'skipped', 'errors']);
|
||||
expect(report.outcomes.tagged.rows.map(row => [row.path, row.detail])).toEqual([['sub/a.png', 'Added 1 tag']]);
|
||||
await expect(toast(page)).toHaveText('Job report downloaded.');
|
||||
});
|
||||
|
||||
// D111 (System S5): each kind's own descriptor. A thumbnail job counts Generated (green), Skipped and Errors (red),
|
||||
// its pill says "Generating thumbnails" and its toasts "Thumbnails finished · N generated · M errors".
|
||||
const THUMB_JOB = { kind: 'thumbnails', total: 10, output_folders: 2 };
|
||||
|
||||
test('a thumbnail job shows Generated Skipped Errors, its pill verb and its toasts', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request, THUMB_JOB);
|
||||
await step(request, job.id, { processed: 4, records: [
|
||||
{ group: 'generated', path: 'sub/a.png', root: 'output', detail: 'Preview written' },
|
||||
{ group: 'skipped', path: 'b.png', root: 'input', detail: 'Preview already up to date' },
|
||||
{ group: 'errors', path: 'broken.png', root: 'output', detail: 'Not a valid image file' },
|
||||
] });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(pill(page)).toHaveText('Generating thumbnails · 40% · 4 / 10');
|
||||
await page.goto(`${workspace}#/activity`);
|
||||
await expect(items(page).first().locator('p')).toHaveText(/^Started \d\d:\d\d · 2 output folders · closing this view does not cancel it$/);
|
||||
await expect(items(page).first().locator('progress')).toHaveAttribute('value', '40');
|
||||
await openJob(page, job.id);
|
||||
await expect(heading(page).locator('h1')).toHaveText('Thumbnail generation');
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report', 'Cancel job']);
|
||||
expect(await tileValues(page)).toEqual([['stat added', 'Generated', '1'], ['stat', 'Skipped', '1'], ['stat bad', 'Errors', '1']]);
|
||||
await expect(outcomeButtons(page)).toHaveText(['Generated1', 'Errors1', 'Skipped1']);
|
||||
await expect(fileRows(page).first().locator('.path')).toHaveText('sub/a.png');
|
||||
expect(await fileRows(page).first().evaluate(node => node.localName)).toBe('button');
|
||||
await outcomeButtons(page).filter({ hasText: 'Errors' }).click();
|
||||
expect(await fileRows(page).first().evaluate(node => node.localName)).toBe('div');
|
||||
await expect(scanLog(page).locator('summary')).toHaveText('● Job log — raw detail, click to show');
|
||||
|
||||
await finish(request, job.id);
|
||||
await expect(toast(page)).toHaveText('Thumbnails finished · 1 generated · 1 error');
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Finished \d\d:\d\d · 1 generated · 1 error$/);
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report']);
|
||||
});
|
||||
|
||||
test('a held legacy job and the health check show no tiles, no Results and no made-up percentage', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const response = await request.post('/__test__/jobs/hold',
|
||||
{ data: { kind: 'thumbnails', title: 'Thumbnail clear (gallery page)', admission: 'guarded' } });
|
||||
await expect(response).toBeOK();
|
||||
const held = (await response.json()).job;
|
||||
await page.goto(`${workspace}#/images`);
|
||||
// D112: the pill names the held operation and shows no percentage; its bar is indeterminate, not an empty fill
|
||||
// (S5 review M5), and stays hidden from assistive tech, which reads the label.
|
||||
await expect(pill(page)).toHaveText('Thumbnail clear (gallery page)');
|
||||
await expect(pill(page)).toHaveAttribute('aria-label', 'Thumbnail clear (gallery page)');
|
||||
await expect(pill(page).locator('.activity-pill-bar')).toHaveClass('activity-pill-bar indeterminate');
|
||||
await expect(pill(page).locator('.activity-pill-bar')).toHaveAttribute('aria-hidden', 'true');
|
||||
expect(await pill(page).locator('.activity-pill-bar i').evaluate(node => node.style.width)).toBe('');
|
||||
await page.goto(`${workspace}#/activity`);
|
||||
const item = items(page).filter({ has: page.getByRole('link', { name: /Thumbnail clear \(gallery page\)$/ }) });
|
||||
await expect(item.locator('p')).toHaveText(/^Started \d\d:\d\d · closing this view does not cancel it$/);
|
||||
await expect(item.locator('progress')).toHaveCount(1);
|
||||
await expect(item.locator('progress')).not.toHaveAttribute('value');
|
||||
await expect(item.getByRole('button', { name: 'Cancel job', exact: true })).toHaveCount(0);
|
||||
await item.getByRole('link').click();
|
||||
await expect(heading(page).locator('h1')).toHaveText('Thumbnail clear (gallery page)');
|
||||
await expect(progressSection(page).locator('progress')).not.toHaveAttribute('value');
|
||||
await expect(progressSection(page).locator('dl.stat-row')).toHaveCount(0);
|
||||
await expect(resultsSection(page)).toHaveCount(0);
|
||||
await expect(scanLog(page)).toHaveCount(1);
|
||||
await finish(request, held.id);
|
||||
await expect(toast(page)).toHaveText('Thumbnail clear (gallery page) finished.');
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Finished \d\d:\d\d$/);
|
||||
await expect(progressSection(page).locator('progress')).toHaveAttribute('value', '100');
|
||||
|
||||
// The health check (a read job of its own kind): its status line, no tiles, no Cancel.
|
||||
const health = await startJob(request, { kind: 'health', total: 0 });
|
||||
await openJob(page, health.id);
|
||||
await expect(heading(page).locator('h1')).toHaveText('Health check');
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('Checking the database and image links');
|
||||
await expect(progressSection(page).locator('progress')).not.toHaveAttribute('value');
|
||||
await expect(progressSection(page).locator('dl.stat-row')).toHaveCount(0);
|
||||
await expect(resultsSection(page)).toHaveCount(0);
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report']);
|
||||
await expect(pill(page)).toHaveText('Health check');
|
||||
await finish(request, health.id);
|
||||
await expect(toast(page)).toHaveText('Health check finished.');
|
||||
});
|
||||
|
||||
// S5 review I3: a thumbnail job with nothing to generate ends on the server's final line ("No previews to generate · N
|
||||
// already up to date"), never on a stale "Processing file 0 of 0" under the Completed chip.
|
||||
test('a finished thumbnail job with nothing to do says so', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request, { ...THUMB_JOB, total: 0 });
|
||||
await step(request, job.id, { total: 0, processed: 0 }); // the stale line an older server left
|
||||
await openJob(page, job.id);
|
||||
const line = progressSection(page).locator('.panel-heading > span.small');
|
||||
await finish(request, job.id, 'finished', { status_line: 'No previews to generate · 24 already up to date' });
|
||||
await expect(line).toHaveText('No previews to generate · 24 already up to date');
|
||||
await expect(page.getByText('Processing file 0 of 0')).toHaveCount(0);
|
||||
await expect(progressSection(page).locator('progress')).toHaveAttribute('value', '100');
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {readFile,writeFile} from 'node:fs/promises';
|
||||
import {createHash} from 'node:crypto';
|
||||
|
||||
@@ -317,6 +317,19 @@ test('marker labels disclose exact observed dequeue and outcome including times
|
||||
}
|
||||
});
|
||||
|
||||
// A timeline scroll re-renders the markers on the next frame. On a slow runner that frame lands between the press and
|
||||
// the release of a marker tap (Playwright's own scroll-into-view scrolls the timeline first); the re-render must leave
|
||||
// the pressed marker's nodes alone, or Chromium touch drops the click. The scroll event is dispatched so the marker
|
||||
// stays under the pointer, and the two frames are the re-render itself.
|
||||
test('a timeline re-render between press and release keeps the marker tap',async({page})=>{
|
||||
const f=page.fixture;await open(page,f.detail_id);
|
||||
const target=marker(page,f.runs.shared,'z');await expect(target).toHaveAttribute('aria-disabled','false');await target.scrollIntoViewIfNeeded();
|
||||
const box=await target.boundingBox();await page.mouse.move(box.x+box.width/2,box.y+box.height/2);await page.mouse.down();
|
||||
await recorded(page).locator('.history-timeline').evaluate(timeline=>{timeline.dispatchEvent(new Event('scroll'));return new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)));});
|
||||
await page.mouse.up();
|
||||
await expect(dialog(page).locator('[data-receipt]')).toHaveCount(5);
|
||||
});
|
||||
|
||||
test('same-epoch refresh retains recorded context mode and exact receipt after fresh candidate reads',async({page})=>{
|
||||
const f=page.fixture;await open(page,f.detail_id);await marker(page,f.runs.shared,'z').click();await expect(dialog(page).locator('[data-receipt]')).toHaveCount(5);
|
||||
await dialog(page).getByRole('button',{name:'Effective',exact:true}).click();
|
||||
@@ -481,6 +494,23 @@ test('the inline current file names its prompts apart from the frozen run fields
|
||||
expect(await exact(dialog(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'))).toBe(before);
|
||||
});
|
||||
|
||||
// LI-4 (D117): the recorded history's current-file viewer is the shared full inspector, so it copies the legacy summary.
|
||||
test('the current file viewer copies its settings summary',async({page})=>{
|
||||
await page.addInitScript(()=>{window.__copied=[];Object.defineProperty(navigator,'clipboard',{configurable:true,value:{writeText:async text=>{window.__copied.push(text);}}});});
|
||||
await page.route(url=>url.pathname.endsWith('/inspection'),async route=>{const response=await route.fetch();const body=await response.json();
|
||||
body.prompts={positive:{status:'resolved',text:'current file positive',source:'graph-inferred'},negative:{status:'resolved',text:'current file negative',source:'graph-inferred'}};
|
||||
await route.fulfill({response,json:body});});
|
||||
await open(page,page.fixture.detail_id);await marker(page,page.fixture.runs.shared).click();await expect(dialog(page).locator('[data-receipt]')).toHaveCount(5);
|
||||
await dialog(page).locator('[data-receipt]').first().getByRole('button',{name:'View current file',exact:true}).click();
|
||||
const details=page.locator('dialog#viewer-dialog #viewer-details');await expect(details.locator('.panel-heading')).toBeVisible();
|
||||
const facts=await details.locator(':scope > div > dl.meta-grid dd').allTextContents();
|
||||
await details.getByRole('button',{name:'Copy settings summary',exact:true}).click();
|
||||
await expect(page.locator('#workspace-toast')).toHaveText('Settings summary copied.');
|
||||
const [seed,steps,cfg,sampler,,model]=facts;
|
||||
expect(await page.evaluate(()=>window.__copied)).toEqual([[`Checkpoint: ${model}`,'Positive Prompt: current file positive','Negative Prompt: current file negative',
|
||||
`Steps: ${steps}`,`CFG Scale: ${cfg}`,`Sampler: ${sampler}`,`Seed: ${seed}`].join('\n').replace(/: (Not recorded|Unresolved|Ambiguous|Unavailable within inspection limits)(?=\n|$)/g,': Unknown')]);
|
||||
});
|
||||
|
||||
test('the recorded run dialog with a resolved current file has no duplicate button or region names',async({page})=>{
|
||||
await page.route(url=>url.pathname.endsWith('/inspection'),async route=>{const response=await route.fetch();const body=await response.json();
|
||||
body.prompts={positive:{status:'resolved',text:'current file positive',source:'graph-inferred'},negative:{status:'resolved',text:'current file negative',source:'graph-inferred'}};
|
||||
@@ -1006,7 +1036,7 @@ test('shared gallery uses concept cards, image opens the viewer, empty copy',asy
|
||||
await expect(star).toHaveAccessibleName(`Favorite ${filename}`);await expect(star).toHaveAttribute('aria-pressed',String(image.favorite));
|
||||
await expect(star).toHaveText(image.favorite?'★':'☆');
|
||||
await expect(first.locator('.media-info h3')).toHaveCount(1);
|
||||
await expect(first.locator('.image-card-actions > button.icon-btn')).toHaveAccessibleName(`Inspect ${filename}`);
|
||||
await expect(first.locator('.image-card-actions > button.icon-btn:first-child')).toHaveAccessibleName(`Inspect ${filename}`);
|
||||
// C3 activation: the image opens the viewer and Escape returns focus to it.
|
||||
const openButton=first.locator('.image-open');await openButton.click();
|
||||
const viewer=page.locator('dialog#viewer-dialog');await expect(viewer).toBeVisible();
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept, openConcept, openProduction } from './concept-inventory.mjs';
|
||||
|
||||
const workspace = '/prompt_manager/workspace#/images';
|
||||
@@ -398,18 +398,20 @@ test('preview uses actual Viewer.js and cleans up when closing or leaving Images
|
||||
test('real video media has a usable card and native playback cleans up on close and navigation', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/reset', { data: { include_video: true } })).toBeOK();
|
||||
const media = await (await request.get('/prompt_manager/images/output?limit=40')).json();
|
||||
expect(media.images.find(image => image.filename === 'clip.webm').thumbnail_url).toBeNull();
|
||||
// #249: a video with no preview yet still lists its preview URL; the server makes the frame on first view.
|
||||
expect(media.images.find(image => image.filename === 'clip.webm').thumbnail_url).toMatch(/\/thumbnails\/clip_thumb\.jpg\?root=0/);
|
||||
await page.goto(workspace);
|
||||
await refine(page, { Media: 'video' });
|
||||
await expect(cards(page)).toHaveCount(1);
|
||||
const card = cards(page).first();
|
||||
await expect(card).toHaveAttribute('data-image-key', '0:clip.webm');
|
||||
await expect(card.locator('img')).toHaveCount(0);
|
||||
await expect(card.getByRole('button', { name: 'Open clip.webm', exact: true })).toBeVisible();
|
||||
// Legacy video art inside the concept card; D11 names the video's own extension.
|
||||
await expect(card.locator('.media-art > .image-open > video')).toHaveCount(1);
|
||||
// The card shows the preview frame, never the video itself; D11 names the video's own extension.
|
||||
await expect(card.locator('video')).toHaveCount(0);
|
||||
await expect(card.locator('.media-art > .image-open > img')).toHaveCount(1);
|
||||
await expect(card.locator('.media-info p')).toHaveText(/(?:^|· )WEBM$/);
|
||||
expect(await card.locator('video').evaluate(node => node.getBoundingClientRect().height)).toBeGreaterThan(20);
|
||||
await expect.poll(() => card.locator('img').evaluate(node => node.complete && node.naturalHeight > 0)).toBe(true);
|
||||
expect(await card.locator('img').evaluate(node => node.getBoundingClientRect().height)).toBeGreaterThan(20);
|
||||
await card.getByRole('button', { name: 'Open clip.webm', exact: true }).click();
|
||||
const video = page.locator('dialog[open] video');
|
||||
await expect(video).toBeVisible();
|
||||
@@ -463,6 +465,50 @@ test('collection viewer ignores delayed media callbacks and keeps inspection ret
|
||||
await page.evaluate(()=>{location.hash='/overview';});await expect(page.locator('.viewer-container,#viewer-dialog')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('collection viewer leaving an image Viewer.js has not built yet releases it at once',async({page})=>{
|
||||
await page.goto(workspace);await expect(cards(page)).toHaveCount(2);
|
||||
let release,held;const waiting=new Promise(resolve=>{held=resolve;});const pending=new Promise(resolve=>{release=resolve;});
|
||||
await page.route('**/images/serve/shared/same.png?*',async route=>{
|
||||
const url=new URL(route.request().url());
|
||||
if(url.searchParams.get('root')==='0'&&url.searchParams.has('_pm_viewer')) {held();await pending;}
|
||||
await route.continue().catch(()=>{});
|
||||
});
|
||||
await cards(page).first().getByRole('button',{name:'Open same.png',exact:true}).click();await waiting;
|
||||
const preview=page.locator('dialog#viewer-dialog');
|
||||
await page.evaluate(()=>{window.unbuiltImage=document.querySelector('#viewer-stage > img');});
|
||||
await preview.getByRole('button',{name:'Next media',exact:true}).click();
|
||||
await expect(preview.getByRole('button',{name:'Zoom in',exact:true})).toHaveAttribute('aria-disabled','false');
|
||||
// The left image is off the stage. Viewer.js has not built it, so the viewer is destroyed and the src released at
|
||||
// once: its late load must never build Viewer.js into an image without a parent (getComputedStyle(null)).
|
||||
release();
|
||||
await expect.poll(()=>page.evaluate(()=>[window.unbuiltImage.isConnected,window.unbuiltImage.hasAttribute('src'),!!window.unbuiltImage.viewer])).toEqual([false,false,false]);
|
||||
await page.evaluate(()=>{location.hash='/overview';});await expect(page.locator('.viewer-container,#viewer-dialog')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('collection viewer closed while its image finishes loading never builds Viewer.js afterwards',async({page})=>{
|
||||
await page.goto(workspace);await expect(cards(page)).toHaveCount(2);
|
||||
let held;const waiting=new Promise(resolve=>{held=resolve;});
|
||||
// The image is held for the whole test: its late load is delivered below, at the moment Viewer.js cannot cancel.
|
||||
await page.route('**/images/serve/shared/same.png?*',async route=>{
|
||||
const url=new URL(route.request().url());
|
||||
if(url.searchParams.get('root')==='0'&&url.searchParams.has('_pm_viewer')) {held();return;}
|
||||
await route.continue().catch(()=>{});
|
||||
});
|
||||
await cards(page).first().getByRole('button',{name:'Open same.png',exact:true}).click();await waiting;
|
||||
const preview=page.locator('dialog#viewer-dialog');
|
||||
// Viewer.js's own cancel removes its image listeners only while `image.complete` is false. An image that has
|
||||
// finished downloading but whose load event is still queued is already complete: close in that state, then deliver it.
|
||||
await page.evaluate(()=>{window.finishingImage=document.querySelector('#viewer-stage > img');Object.defineProperty(window.finishingImage,'complete',{get:()=>true});});
|
||||
await page.keyboard.press('Escape');await expect(preview).toHaveCount(0);
|
||||
const after=await page.evaluate(async()=>{
|
||||
const image=window.finishingImage;image.dispatchEvent(new Event('load'));
|
||||
// Viewer.js builds an inline viewer in a zero-delay timeout after its image loads; this runs after it.
|
||||
await new Promise(resolve=>setTimeout(resolve));
|
||||
return {src:image.hasAttribute('src'),built:!!image.parentNode?.querySelector('.viewer-container')};
|
||||
});
|
||||
expect(after).toEqual({src:false,built:false});
|
||||
});
|
||||
|
||||
test('collection viewer describes video transforms and releases native playback on navigation',async({page,request})=>{
|
||||
await expect(await request.post('/__test__/reset',{data:{include_video:true}})).toBeOK();await page.goto(workspace);
|
||||
await page.locator('article[data-image-key="0:clip.webm"] .image-open').click();
|
||||
@@ -545,7 +591,7 @@ async function staleSourceRecovery(page, request, roots) {
|
||||
await expect(await request.post('/prompt_manager/settings', { data: { gallery_root_paths: roots } })).toBeOK();
|
||||
const selected = initial.images.at(-1), key = `${selected.root_index}:${selected.path}`;
|
||||
const stale = page.waitForResponse(response => response.url().includes('/images/inspect/') && response.status() === 409);
|
||||
await page.locator(`article[data-image-key="${key}"] .image-card-actions button`).click();expect((await stale).status()).toBe(409);
|
||||
await page.locator(`article[data-image-key="${key}"] .image-card-actions button:first-child`).click();expect((await stale).status()).toBe(409);
|
||||
await expect(details(page)).toContainText('The source changed. Refresh the image collection before inspecting this file.');
|
||||
await page.locator(`article[data-image-key="${key}"] .image-open`).click();
|
||||
const viewer = page.locator('dialog#viewer-dialog');await expect(viewer).toContainText('The source changed.');
|
||||
@@ -669,16 +715,45 @@ test('selected layout stays blue with readable text on hover in either theme', a
|
||||
}
|
||||
});
|
||||
|
||||
test('refresh releases the previous inspector video source', async ({ page, request }) => {
|
||||
test('refresh releases the previous inspector cover source', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/reset', { data: { include_video: true } })).toBeOK();
|
||||
await page.goto(workspace);
|
||||
const card = cards(page).filter({ hasText: 'clip.webm' });
|
||||
await card.getByRole('button', { name: 'Inspect clip.webm', exact: true }).click();
|
||||
// #249: a video's cover is its preview frame, not the video.
|
||||
const cover = details(page).locator('img.detail-cover');
|
||||
await expect(cover).toHaveAttribute('src', /\/thumbnails\/clip_thumb\.jpg\?/);
|
||||
await expect(details(page).locator('video')).toHaveCount(0);
|
||||
await cover.evaluate(element => { window.__oldInspectorCover = element; });
|
||||
await rescan(page);
|
||||
await expect.poll(() => page.evaluate(() => !window.__oldInspectorCover.isConnected)).toBe(true);
|
||||
expect(await page.evaluate(() => window.__oldInspectorCover.getAttribute('src'))).toBeNull();
|
||||
});
|
||||
|
||||
/** A listing that names no preview for its videos (the fallback a card and the cover keep for one). */
|
||||
async function videosWithoutPreview(page) {
|
||||
await page.route(url => url.pathname === '/prompt_manager/images/output', async route => {
|
||||
const response = await route.fetch();
|
||||
const body = await response.json();
|
||||
if (Array.isArray(body.images)) body.images = body.images.map(image => image.is_video ? { ...image, thumbnail_url: null } : image);
|
||||
await route.fulfill({ response, json: body });
|
||||
});
|
||||
}
|
||||
|
||||
test('a video listed without a preview keeps its own element, released on refresh', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/reset', { data: { include_video: true } })).toBeOK();
|
||||
await videosWithoutPreview(page);
|
||||
await page.goto(workspace);
|
||||
const card = cards(page).filter({ hasText: 'clip.webm' });
|
||||
await expect(card.locator('.media-art > .image-open > video')).toHaveCount(1);
|
||||
await expect(card.locator('img')).toHaveCount(0);
|
||||
await card.getByRole('button', { name: 'Inspect clip.webm', exact: true }).click();
|
||||
await expect(details(page).locator('video')).toBeVisible();
|
||||
await details(page).locator('video').evaluate(element => { window.__oldInspectorVideo = element; });
|
||||
await card.locator('video').evaluate(element => { window.__oldCardVideo = element; });
|
||||
await rescan(page);
|
||||
await expect.poll(() => page.evaluate(() => !window.__oldInspectorVideo.isConnected)).toBe(true);
|
||||
expect(await page.evaluate(() => window.__oldInspectorVideo.getAttribute('src'))).toBeNull();
|
||||
await expect.poll(() => page.evaluate(() => !window.__oldInspectorVideo.isConnected && !window.__oldCardVideo.isConnected)).toBe(true);
|
||||
expect(await page.evaluate(() => [window.__oldInspectorVideo.getAttribute('src'), window.__oldCardVideo.getAttribute('src')])).toEqual([null, null]);
|
||||
});
|
||||
|
||||
test('Import Rescan rescans cached files and preserves the collection when rescan fails', async ({ page, request }) => {
|
||||
@@ -787,7 +862,12 @@ test('density changes and resize preserve identity focus and selection', async (
|
||||
await page.getByRole('button',{name:'Grid',exact:true}).click();
|
||||
for (const [width,columns] of [[951,3],[950,3],[700,2],[390,2]]) {
|
||||
await page.setViewportSize({width,height:844});
|
||||
await expect.poll(()=>cards(page).nth(columns).evaluate(node=>node.offsetLeft)).toBe(0);
|
||||
// 700px and 390px both lay out two columns, so a card at the row start alone is also true of the previous
|
||||
// width's cards before WebKit runs the first frame after the resize; wait for the row to fit the new grid.
|
||||
await expect.poll(()=>page.locator('.image-grid').evaluate((grid,columns)=>{
|
||||
const row=[...grid.querySelectorAll(':scope > article[data-image-key]')];
|
||||
return row[columns].offsetLeft===0&&row[columns-1].getBoundingClientRect().right<=grid.getBoundingClientRect().right+1;
|
||||
},columns)).toBe(true);
|
||||
await first.getByRole('button',{name:/^Open /}).focus();await page.keyboard.press('ArrowDown');
|
||||
await expect(cards(page).nth(columns).getByRole('button',{name:/^Open /})).toBeFocused();
|
||||
expect(await page.evaluate(()=>document.documentElement.scrollWidth-document.documentElement.clientWidth)).toBeLessThanOrEqual(1);
|
||||
@@ -902,6 +982,10 @@ test('browser Back closes the open Images dialog and leaves the next page usable
|
||||
|
||||
test('count line shows images and positive prompts', async ({ page, request }) => {
|
||||
await page.addInitScript(() => { window.IntersectionObserver = undefined; });
|
||||
// D17's "Today" is the newest file's day: the page's clock stands at that file's mtime, so the fixture's day and
|
||||
// the page's agree on either side of midnight.
|
||||
const { images } = await (await request.get('/prompt_manager/images/output?mode=workspace&limit=40')).json();
|
||||
await page.clock.setFixedTime(new Date(Math.max(...images.map(image => image.modified_time)) * 1000));
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
await expect(caption(page).locator('#result-count')).toHaveText('2 of 2 images · 2 positive prompts');
|
||||
@@ -1028,8 +1112,7 @@ async function cardInventory(page, concept) {
|
||||
const walk = new Function(`return (${walkSource})()`)();
|
||||
return nodes.map(card => {
|
||||
const copy = card.cloneNode(true);
|
||||
// D9: no tag button until image annotations. The D8 checkbox and the D7 star are compared.
|
||||
if (concept) for (const node of copy.querySelectorAll('[aria-label^="Tag or auto-tag "]')) node.remove();
|
||||
// Info then Tag (#249 C2 retired D9). The D8 checkbox and the D7 star are compared.
|
||||
const open = card.querySelector('.image-open'), img = open.querySelector('img');
|
||||
const star = card.querySelector('.image-overlays > button.star-button');
|
||||
const box = card.querySelector('.image-overlays > label > input');
|
||||
@@ -1039,7 +1122,7 @@ async function cardInventory(page, concept) {
|
||||
size: [img.getAttribute('width'), img.getAttribute('height')],
|
||||
title: card.querySelector('.media-info h3').textContent, subtitle: card.querySelector('.media-info p').textContent,
|
||||
ratio: card.style.getPropertyValue('--media-ratio').replace(/\s/g, ''),
|
||||
inspect: card.querySelector('.image-card-actions > button').getAttribute('aria-label'),
|
||||
actions: [...card.querySelectorAll('.image-card-actions > button')].map(button => button.getAttribute('aria-label')),
|
||||
star: star && { text: star.textContent, pressed: star.getAttribute('aria-pressed'), label: star.getAttribute('aria-label') },
|
||||
select: box && { type: box.type, checked: box.checked, label: box.getAttribute('aria-label') },
|
||||
paths: walk(copy, `article.${[...card.classList].filter(value => value !== 'selected').sort().join('.')}`, []) };
|
||||
@@ -1209,7 +1292,7 @@ async function zoomedGallery(page, request, { cssZoom }) {
|
||||
await page.keyboard.press('Tab');
|
||||
await expect(page.locator(`article[aria-posinset="${columns + 1}"] .image-overlays .star-button`)).toBeFocused();
|
||||
await page.keyboard.press('Tab');
|
||||
await expect(page.locator(`article[aria-posinset="${columns + 1}"] .image-card-actions button`)).toBeFocused();
|
||||
await expect(page.locator(`article[aria-posinset="${columns + 1}"] .image-card-actions button:first-child`)).toBeFocused();
|
||||
await page.evaluate(() => {
|
||||
const card = document.querySelector('article[aria-posinset="1"]');
|
||||
const grid = document.querySelector('.image-grid');
|
||||
@@ -1512,6 +1595,25 @@ test('Refine opened before discovery settles lists the discovered sources and fo
|
||||
});
|
||||
|
||||
// #249 final review m1: as in the old field row, a Source offers only its own folders.
|
||||
test('a search reload keeps the Filters label node, so a click landing during the reload is not lost', async ({ page, request }) => {
|
||||
// WebKit drops a click whose mousedown hit a text node that was replaced before mouseup; the reload reports the
|
||||
// (unchanged) active count, so the label must keep its node.
|
||||
await expect(await request.post('/__test__/reset', { data: { extra_images: 80 } })).toBeOK();
|
||||
await page.goto(workspace);
|
||||
const button = filtersButton(page);
|
||||
await button.click();await expect(refineDialog(page).getByLabel('Folder', { exact: true }).locator('option[value="bulk"]')).toHaveCount(1);await page.keyboard.press('Escape');
|
||||
await refine(page, { Source: '0', Folder: 'bulk' });
|
||||
await expect(button).toHaveText('Filters · 2');
|
||||
const label = await button.evaluateHandle(node => node.firstChild);
|
||||
const reloaded = page.waitForResponse(response => {
|
||||
const url = new URL(response.url());return imagePages(url) && url.searchParams.get('q') === 'png';
|
||||
});
|
||||
await search(page).fill('png');
|
||||
await reloaded;
|
||||
await expect(button).toHaveText('Filters · 2');
|
||||
expect(await button.evaluate((node, text) => node.firstChild === text && text.isConnected, label)).toBe(true);
|
||||
});
|
||||
|
||||
test('Refine offers only the chosen source folders', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/add-nested-image')).toBeOK();
|
||||
await page.goto(workspace);
|
||||
@@ -1684,7 +1786,8 @@ test('card checkboxes select images into the concept bar and Clear empties it',
|
||||
const selectedBefore = await cards(page).evaluateAll(nodes => nodes.map(node => node.classList.contains('selected')));
|
||||
await checkbox(first, 'same.png').check();
|
||||
await expect(bulkbar(page).locator('.bulkbar > strong')).toHaveText('1 selected');
|
||||
await expect(barButtons(page)).toHaveText(['Set category', 'Export metadata', 'Clear']);
|
||||
// D85: Add tags (the concept's) and Auto-tag selected lead the real actions.
|
||||
await expect(barButtons(page)).toHaveText(['Add tags', 'Auto-tag selected', 'Set category', 'Export metadata', 'Clear']);
|
||||
// Toggling neither opens the viewer, nor inspects, nor moves focus.
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
await expect(checkbox(first, 'same.png')).toBeFocused();
|
||||
@@ -2060,7 +2163,7 @@ test('scan for metadata opens the review dialog from Import', async ({ page, req
|
||||
await openScanReview(page);
|
||||
const dialog = scanReview(page);
|
||||
await expect(dialog.locator('dl.meta-grid > div > dt')).toHaveText(['Sources', 'Scope']);
|
||||
await expect(dialog.locator('dl.meta-grid > div > dd')).toHaveText(['2 output folders', 'All files']);
|
||||
await expect(dialog.locator('dl.meta-grid > div > dd')).toHaveText(['2 output folders', 'New and unresolved files']);
|
||||
await expect(dialog.locator('#modal-body > p.small')).toHaveText('Enrich positive / negative associations, reuse canonical text and report unchanged files. Do not inflate run counts or recreate existing thumbnails.');
|
||||
await expect(dialog.locator('.notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
await expect(dialog.locator('.dialog-actions button')).toHaveText(['Cancel', 'Start scan']);
|
||||
|
||||
@@ -0,0 +1,857 @@
|
||||
// The native "Inspect a file" page (#249 local Inspect, PR L2 Task 6): one chosen or dropped local file previews
|
||||
// inline, and the heading's one action opens the shared viewer with the full inspector on a record read entirely in
|
||||
// this browser. The concept is docs/concepts/workspace/app.js renderMetadata (740–748) and its selection (1715–1729);
|
||||
// the production differences are D60–D66 in TESTING.md.
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { parityPng, localFixture, tinyWebm, sendablePng, corruptPng, disguised, undecodableMp4, POSITIVE_LINES } from './local-inspect-files.mjs';
|
||||
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const TOAST = 'Local preview only. No file uploaded; metadata is available above.';
|
||||
const ACTION = 'Open file metadata';
|
||||
const COPY = {
|
||||
intro: 'Read embedded metadata in this browser. The file is not uploaded or added to your library.',
|
||||
note: 'Your file stays in this browser. Its metadata opens with “Open file metadata”.',
|
||||
chooser: 'Choose an image or video for a local preview',
|
||||
multiple: 'Drop one file at a time.',
|
||||
directory: 'Folders cannot be inspected. Choose one image or video file.',
|
||||
unsupported: 'This file type cannot be previewed or inspected here. Choose an image or video file.',
|
||||
previewFailed: 'The preview could not be displayed. Metadata and the original remain available in the viewer.',
|
||||
};
|
||||
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC', reducedMotion: 'reduce' });
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await expect(await request.post('/__test__/reset')).toBeOK();
|
||||
page.inspectErrors = [];
|
||||
page.on('pageerror', error => page.inspectErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, browserName }) => {
|
||||
await page.unrouteAll({ behavior: 'ignoreErrors' });
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.inspectErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const main = page => page.locator('#workspace-main');
|
||||
const action = page => main(page).locator('.heading-actions').getByRole('button', { name: ACTION, exact: true });
|
||||
const chooser = page => page.locator('#local-file');
|
||||
const preview = page => page.locator('#local-preview');
|
||||
const status = page => preview(page).locator('p.small');
|
||||
const viewer = page => page.locator('dialog#viewer-dialog');
|
||||
const rail = page => viewer(page).locator('#viewer-details');
|
||||
|
||||
async function openInspect(page) {
|
||||
await page.goto(`${workspace}#/inspect`);
|
||||
await expect(main(page).getByRole('heading', { level: 1, name: 'Inspect a file', exact: true })).toBeVisible();
|
||||
await expect(chooser(page)).toBeAttached();
|
||||
}
|
||||
|
||||
/** Selects through the real chooser, then waits for the page to classify the file. */
|
||||
async function choose(page, file) {
|
||||
await chooser(page).setInputFiles(file);
|
||||
}
|
||||
|
||||
async function openViewer(page) {
|
||||
await expect(action(page)).toHaveAttribute('aria-disabled', 'false');
|
||||
await action(page).click();
|
||||
await expect(viewer(page)).toBeVisible();
|
||||
await expect(rail(page).locator('.panel-heading')).toBeVisible();
|
||||
}
|
||||
|
||||
/** Dispatches dragover + drop on `selector` with Files built in the page (and optional string items). */
|
||||
function drop(page, selector, { files = [], strings = [] } = {}) {
|
||||
return page.evaluate(async ({ selector, files, strings }) => {
|
||||
const transfer = new DataTransfer();
|
||||
for (const file of files) transfer.items.add(new File([new Uint8Array(file.bytes)], file.name, { type: file.mimeType }));
|
||||
for (const [type, value] of strings) transfer.items.add(value, type);
|
||||
const target = document.querySelector(selector);
|
||||
const over = new DragEvent('dragover', { bubbles: true, cancelable: true, dataTransfer: transfer });
|
||||
const dropped = new DragEvent('drop', { bubbles: true, cancelable: true, dataTransfer: transfer });
|
||||
target.dispatchEvent(over);
|
||||
target.dispatchEvent(dropped);
|
||||
return { over: over.defaultPrevented, drop: dropped.defaultPrevented };
|
||||
}, { selector, files: files.map(file => ({ name: file.name, mimeType: file.mimeType, bytes: [...file.buffer] })), strings });
|
||||
}
|
||||
|
||||
/** The rail's comparable content: prompts, the six facts, the raw records and the two downloads (D66 aside). */
|
||||
const railContent = locator => locator.evaluate(node => {
|
||||
const raw = [...node.querySelectorAll('details')].find(item => item.querySelector('summary')?.textContent === 'Workflow & raw metadata');
|
||||
const facts = node.querySelector(':scope .meta-grid:not(details .meta-grid)');
|
||||
return {
|
||||
positive: node.querySelector('.prompt-section.positive p.prompt-text')?.textContent ?? null,
|
||||
negative: node.querySelector('.prompt-section.negative p.prompt-text')?.textContent ?? null,
|
||||
facts: facts ? [...facts.querySelectorAll(':scope > div')].map(item => [item.querySelector('dt').textContent, item.querySelector('dd').textContent]) : null,
|
||||
raw: raw ? [...raw.querySelectorAll('p.small, pre')].map(item => item.textContent) : null,
|
||||
downloads: raw ? [...raw.querySelectorAll('button')].map(item => [item.textContent, item.getAttribute('aria-disabled')]) : null,
|
||||
};
|
||||
});
|
||||
|
||||
/** Counts storage writes and object URLs created and revoked for Files (download blobs are not the session's). */
|
||||
async function instrument(page) {
|
||||
await page.addInitScript(() => {
|
||||
window.__inspect = { writes: [], created: [], revoked: [], files: new Set() };
|
||||
for (const name of ['setItem', 'removeItem', 'clear']) {
|
||||
const original = Storage.prototype[name];
|
||||
Storage.prototype[name] = function (...args) { window.__inspect.writes.push([name, args[0]]); return original.apply(this, args); };
|
||||
}
|
||||
const open = indexedDB.open.bind(indexedDB);
|
||||
indexedDB.open = (...args) => { window.__inspect.writes.push(['indexedDB.open', args[0]]); return open(...args); };
|
||||
const create = URL.createObjectURL, revoke = URL.revokeObjectURL;
|
||||
URL.createObjectURL = object => {
|
||||
const url = create(object);
|
||||
if (object instanceof File) { window.__inspect.created.push(url); window.__inspect.files.add(url); }
|
||||
return url;
|
||||
};
|
||||
URL.revokeObjectURL = url => { if (window.__inspect.files.has(url)) window.__inspect.revoked.push(url); return revoke(url); };
|
||||
});
|
||||
}
|
||||
const urls = page => page.evaluate(() => ({ created: [...window.__inspect.created], revoked: [...window.__inspect.revoked], writes: [...window.__inspect.writes] }));
|
||||
|
||||
test('chooser opens the shared full inspector without a second read', async ({ page, request }) => {
|
||||
// The Images viewer's rail for the same bytes, served from the output folder.
|
||||
await expect(await request.post('/__test__/add-local-inspect-file', { data: { name: 'f02.png' } })).toBeOK();
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await page.locator('article[data-image-key="0:local-inspect/f02.png"] .image-open').click();
|
||||
await expect(rail(page).locator('.prompt-section.positive p.prompt-text')).toBeVisible();
|
||||
const served = await railContent(rail(page));
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer(page)).toHaveCount(0);
|
||||
|
||||
await page.locator('aside.sidebar nav').getByRole('link', { name: 'Inspect a file', exact: true }).click();
|
||||
await expect(main(page).getByRole('heading', { level: 1, name: 'Inspect a file', exact: true })).toBeVisible();
|
||||
await expect(main(page).locator('.heading p.muted')).toHaveText('A standalone file uses the same image viewer and prompt actions.');
|
||||
// The concept DOM: one surface, the drop area with its icon, label, input and note, then the preview slot.
|
||||
const surface = main(page).locator('section.surface');
|
||||
await expect(surface).toHaveCount(1);
|
||||
await expect(surface.locator('h2')).toHaveText('Inspect without importing');
|
||||
await expect(surface.locator(':scope > p')).toHaveText(COPY.intro);
|
||||
const area = surface.locator('div.file-drop.spaced-20');
|
||||
await expect(area.locator(':scope > svg')).toHaveCount(1);
|
||||
await expect(area.locator('label[for="local-file"]')).toHaveText(COPY.chooser);
|
||||
await expect(chooser(page)).toHaveAttribute('type', 'file');
|
||||
await expect(chooser(page)).toHaveAttribute('accept', 'image/png,image/jpeg,image/webp,video/*,.mp4,.mov,.m4v,.webm,.mkv,.avi,.wmv');
|
||||
await expect(chooser(page)).not.toHaveAttribute('multiple', /.*/);
|
||||
await expect(area.locator('span.small#local-file-note')).toHaveText(COPY.note);
|
||||
await expect(surface.locator(':scope > #local-preview')).toBeEmpty();
|
||||
// D60: the one heading action is described by the note until a file is chosen.
|
||||
await expect(main(page).locator('.heading-actions button')).toHaveCount(1);
|
||||
await expect(action(page)).toHaveClass(/\bca-primary\b/);
|
||||
await expect(action(page)).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(action(page)).toHaveAccessibleDescription(COPY.note);
|
||||
await action(page).dispatchEvent('click');
|
||||
await expect(viewer(page)).toHaveCount(0);
|
||||
|
||||
await choose(page, parityPng('f02.png'));
|
||||
const image = preview(page).locator('img.local-file-preview');
|
||||
await expect(image).toHaveAttribute('alt', 'Local file preview');
|
||||
await expect(image).toHaveJSProperty('complete', true);
|
||||
expect(await image.evaluate(node => node.naturalWidth)).toBe(64);
|
||||
expect(await image.getAttribute('src')).toMatch(/^blob:/);
|
||||
const box = await image.evaluate(node => ({ height: node.getBoundingClientRect().height, fit: getComputedStyle(node).objectFit, margin: getComputedStyle(node).marginTop }));
|
||||
expect(box.height).toBeLessThanOrEqual(400);
|
||||
expect(box).toMatchObject({ fit: 'contain', margin: '20px' });
|
||||
await expect(page.locator('#workspace-toast')).toHaveText(TOAST);
|
||||
await expect(action(page)).toHaveAttribute('aria-disabled', 'false');
|
||||
await expect(action(page)).not.toHaveAttribute('aria-describedby', /.*/);
|
||||
|
||||
await openViewer(page);
|
||||
await expect(viewer(page).locator('#viewer-title')).toHaveText('f02.png');
|
||||
const position = viewer(page).locator('.media-position');
|
||||
await expect(position).toHaveText('1 of 1 loaded: f02.png');
|
||||
for (const name of ['Previous media', 'Next media', 'Play slideshow']) {
|
||||
await expect(viewer(page).getByRole('button', { name, exact: true })).toHaveAttribute('aria-disabled', 'true');
|
||||
}
|
||||
await expect(viewer(page).locator('.viewer-canvas img')).toBeVisible();
|
||||
for (const name of ['Fit image', 'Rotate image 90 degrees', 'Flip image horizontally']) {
|
||||
await expect(viewer(page).getByRole('button', { name, exact: true })).toHaveAttribute('aria-disabled', 'false');
|
||||
}
|
||||
const canvas = viewer(page).locator('.viewer-canvas img');
|
||||
await viewer(page).getByRole('button', { name: 'Rotate image 90 degrees', exact: true }).click();
|
||||
await expect(canvas).toHaveAttribute('style', /rotate\(90deg\)/);
|
||||
await viewer(page).getByRole('button', { name: 'Flip image horizontally', exact: true }).click();
|
||||
await expect(canvas).toHaveAttribute('style', /scaleX\(-1\)/);
|
||||
await viewer(page).getByRole('button', { name: 'Fit image', exact: true }).click();
|
||||
await expect(canvas).not.toHaveAttribute('style', /rotate\(90deg\)/);
|
||||
|
||||
// The same rail as Images for the same bytes, except the D66 provenance.
|
||||
const local = await railContent(rail(page));
|
||||
expect(local).toEqual(served);
|
||||
expect(local.positive).toBe('a red fox in snow, detailed fur');
|
||||
expect(local.negative).toBe('blurry, lowres');
|
||||
expect(local.facts.map(([label]) => label)).toEqual(['Seed', 'Steps', 'CFG', 'Sampler', 'Scheduler', 'Model']);
|
||||
expect(local.downloads).toEqual([['Download metadata JSON', null], ['Download workflow JSON', null]]);
|
||||
const provenance = rail(page).locator('details').filter({ has: page.locator('summary', { hasText: 'File & provenance' }) });
|
||||
await expect(provenance).toContainText('Selected local file · read in this browser');
|
||||
await expect(provenance).toContainText('Local file; no saved library association was checked.');
|
||||
await expect(provenance.locator('dt', { hasText: /^Path$/ })).toHaveCount(0);
|
||||
await expect(rail(page).getByRole('button', { name: 'Show in folder', exact: true })).toHaveCount(0);
|
||||
await expect(rail(page).getByRole('button', { name: /Prompt history/ })).toHaveCount(0);
|
||||
await expect(rail(page).locator('.chip-row button.chip')).toHaveCount(0);
|
||||
const original = viewer(page).getByRole('link', { name: 'Download original', exact: true });
|
||||
await expect(original).toHaveAttribute('href', /^blob:/);
|
||||
await expect(original).toHaveAttribute('download', 'f02.png');
|
||||
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer(page)).toHaveCount(0);
|
||||
await expect(action(page)).toBeFocused();
|
||||
// Opening the viewer again shows the same record without reading the File a second time.
|
||||
await page.evaluate(() => {
|
||||
window.__slices = 0;
|
||||
const slice = Blob.prototype.slice;
|
||||
Blob.prototype.slice = function (...args) { if (this.name === 'f02.png') window.__slices++; return slice.apply(this, args); };
|
||||
});
|
||||
await openViewer(page);
|
||||
expect(await railContent(rail(page))).toEqual(served);
|
||||
expect(await page.evaluate(() => window.__slices)).toBe(0);
|
||||
});
|
||||
|
||||
// PR L3 Task 10: local roles are the server inspector's (local-resolver.mjs), so for every parity PNG the rail shows the
|
||||
// prompts, their status lines and the facts Images shows for the same bytes in the output folder.
|
||||
const PARITY_PNGS = ['f01b', 'f02', 'f03', 'f06a', 'f06b', 'f06c', 'f06d', 'f06e', 'f07a', 'f07b', 'f08', 'f09', 'f10', 'f11', 'f12',
|
||||
'f14', 'f15', 'f16', 'f17', 'f19', 'f20', 'f21', 'f22', 'f23', 'f24'];
|
||||
const roleSections = locator => locator.evaluate(node => [...node.querySelectorAll('.prompt-section')].map(item => item.textContent));
|
||||
|
||||
test('chooser shows the Images roles and resolution for every parity PNG', async ({ page, request }) => {
|
||||
test.slow();
|
||||
for (const name of PARITY_PNGS) {
|
||||
await expect(await request.post('/__test__/add-local-inspect-file', { data: { name: `${name}.png` } })).toBeOK();
|
||||
}
|
||||
const served = {};
|
||||
await page.goto(`${workspace}#/images`);
|
||||
for (const name of PARITY_PNGS) {
|
||||
await page.locator(`article[data-image-key="0:local-inspect/${name}.png"] .image-open`).click();
|
||||
await expect(viewer(page).locator('#viewer-title')).toHaveText(`${name}.png`);
|
||||
await expect(rail(page).locator('.panel-heading')).toBeVisible();
|
||||
await expect(rail(page).locator('.prompt-section')).not.toHaveCount(0);
|
||||
served[name] = { content: await railContent(rail(page)), roles: await roleSections(rail(page)) };
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer(page)).toHaveCount(0);
|
||||
}
|
||||
await openInspect(page);
|
||||
for (const name of PARITY_PNGS) {
|
||||
await choose(page, parityPng(`${name}.png`));
|
||||
await openViewer(page);
|
||||
await expect(viewer(page).locator('#viewer-title')).toHaveText(`${name}.png`);
|
||||
await expect(rail(page).locator('.prompt-section')).not.toHaveCount(0);
|
||||
expect({ content: await railContent(rail(page)), roles: await roleSections(rail(page)) }, name).toEqual(served[name]);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer(page)).toHaveCount(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('Import → Inspect a local image lands here', async ({ page }) => {
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await page.getByRole('button', { name: 'Import', exact: true }).click();
|
||||
const dialog = page.getByRole('dialog', { name: 'Import & inspect', exact: true });
|
||||
await dialog.getByRole('button', { name: 'Open inspector', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/inspect$/);
|
||||
await expect(main(page).getByRole('heading', { level: 1, name: 'Inspect a file', exact: true })).toBeVisible();
|
||||
await expect(chooser(page)).toBeAttached();
|
||||
await expect(main(page).locator('.handoff')).toHaveCount(0);
|
||||
await expect(page.getByRole('dialog')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('drop accepts exactly one File and never navigates', async ({ page }) => {
|
||||
await openInspect(page);
|
||||
// A drop that misses the area is cancelled, so the browser never opens the file in place of the page.
|
||||
expect(await drop(page, 'body', { files: [parityPng('f02.png')] })).toEqual({ over: true, drop: true });
|
||||
await expect(preview(page)).toBeEmpty();
|
||||
expect(await drop(page, '.file-drop', { files: [parityPng('f02.png')] })).toEqual({ over: true, drop: true });
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
await expect(page.locator('#workspace-toast')).toHaveText(TOAST);
|
||||
// The chooser names the dropped file, never the one it replaced and never "No file chosen".
|
||||
expect(await chooser(page).evaluate(node => [...node.files].map(file => file.name))).toEqual(['f02.png']);
|
||||
const first = await preview(page).locator('img').getAttribute('src');
|
||||
// Two files, a link and markup keep the selection and say why.
|
||||
await drop(page, '.file-drop', { files: [parityPng('f02.png'), parityPng('f03.png')] });
|
||||
await expect(status(page)).toHaveText(COPY.multiple);
|
||||
await expect(status(page)).toHaveAttribute('role', 'alert');
|
||||
await expect(preview(page).locator('img')).toHaveAttribute('src', first);
|
||||
for (const strings of [[['text/uri-list', 'https://example.invalid/a.png']], [['text/html', '<img src="https://example.invalid/a.png">']]]) {
|
||||
await drop(page, '.file-drop', { strings });
|
||||
await expect(status(page)).toHaveText(COPY.unsupported);
|
||||
await expect(preview(page).locator('img')).toHaveAttribute('src', first);
|
||||
}
|
||||
await expect(action(page)).toHaveAttribute('aria-disabled', 'false');
|
||||
await openViewer(page);
|
||||
await expect(viewer(page).locator('#viewer-title')).toHaveText('f02.png');
|
||||
await expect(page).toHaveURL(/#\/inspect$/);
|
||||
});
|
||||
|
||||
test('cancel and same-file reselect', async ({ page }) => {
|
||||
await instrument(page);
|
||||
await openInspect(page);
|
||||
await choose(page, parityPng('f02.png'));
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
const first = await preview(page).locator('img').getAttribute('src');
|
||||
// The chooser still names the chosen file, as in the concept; a cancel (no files) keeps the page as it is.
|
||||
expect(await chooser(page).evaluate(node => node.files[0]?.name)).toBe('f02.png');
|
||||
await chooser(page).setInputFiles([]);
|
||||
await expect(preview(page).locator('img')).toHaveAttribute('src', first);
|
||||
await expect(status(page)).toHaveCount(0);
|
||||
await expect(action(page)).toHaveAttribute('aria-disabled', 'false');
|
||||
// Opening the picker and cancelling it keeps naming the live selection (modern engines fire `cancel`, not `change`).
|
||||
await chooser(page).dispatchEvent('click');
|
||||
await chooser(page).dispatchEvent('cancel');
|
||||
expect(await chooser(page).evaluate(node => [...node.files].map(file => file.name))).toEqual(['f02.png']);
|
||||
await expect(preview(page).locator('img')).toHaveAttribute('src', first);
|
||||
await expect(action(page)).toHaveAttribute('aria-disabled', 'false');
|
||||
// The same file again is a new selection with its own URL; the old one is released. Opening the picker clears
|
||||
// the chooser, so the browser reports the same file as a change.
|
||||
// (Read in the same task: a picker the click opens is cancelled by the harness, which names the file again.)
|
||||
expect(await chooser(page).evaluate(node => { node.dispatchEvent(new MouseEvent('click')); return node.value; })).toBe('');
|
||||
await choose(page, parityPng('f02.png'));
|
||||
await expect(preview(page).locator('img')).not.toHaveAttribute('src', first);
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
const state = await urls(page);
|
||||
expect(state.created).toHaveLength(2);
|
||||
expect(state.revoked).toEqual([first]);
|
||||
await openViewer(page);
|
||||
await expect(viewer(page).locator('.media-position')).toHaveText('1 of 1 loaded: f02.png');
|
||||
});
|
||||
|
||||
test('preview does not imply metadata support', async ({ page }) => {
|
||||
await instrument(page);
|
||||
await openInspect(page);
|
||||
// JPEG and WebP preview and report no supported metadata, exactly as Images reports them.
|
||||
for (const [file, format] of [[localFixture('tiny.jpg', 'image/jpeg'), 'JPEG'], [localFixture('tiny.webp', 'image/webp'), 'WEBP']]) {
|
||||
await choose(page, file);
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
await openViewer(page);
|
||||
await expect(rail(page)).toContainText('No supported embedded metadata was recorded in this file.');
|
||||
await expect(rail(page).locator('details').first()).toContainText(format);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer(page)).toHaveCount(0);
|
||||
}
|
||||
// A PNG whose text chunk fails its CRC: unreadable metadata, the original still downloads.
|
||||
await choose(page, corruptPng());
|
||||
await expect(action(page)).toHaveAttribute('aria-disabled', 'false');
|
||||
await openViewer(page);
|
||||
await expect(rail(page)).toContainText('Embedded metadata could not be read.');
|
||||
const [download] = await Promise.all([page.waitForEvent('download'), viewer(page).getByRole('link', { name: 'Download original', exact: true }).click()]);
|
||||
expect(download.suggestedFilename()).toBe('corrupt.png');
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer(page)).toHaveCount(0);
|
||||
// Text, markup, SVG and a playlist under media names are unsupported and never get a URL.
|
||||
const before = (await urls(page)).created.length;
|
||||
for (const file of disguised()) {
|
||||
await choose(page, file);
|
||||
await expect(status(page)).toHaveText(COPY.unsupported);
|
||||
await expect(status(page)).toHaveAttribute('role', 'alert');
|
||||
await expect(preview(page).locator('img, video')).toHaveCount(0);
|
||||
await expect(action(page)).toHaveAttribute('aria-disabled', 'true');
|
||||
}
|
||||
expect((await urls(page)).created).toHaveLength(before);
|
||||
});
|
||||
|
||||
test('local video plays inline and in the viewer with real tags', async ({ page, browserName }) => {
|
||||
await openInspect(page);
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
const video = preview(page).locator('video.local-file-preview');
|
||||
await expect(video).toBeVisible();
|
||||
expect(await video.evaluate(node => ({ controls: node.controls, playsInline: node.playsInline, preload: node.preload, muted: node.muted, autoplay: node.autoplay, paused: node.paused })))
|
||||
.toEqual({ controls: true, playsInline: true, preload: 'metadata', muted: true, autoplay: false, paused: true });
|
||||
// WebKit may not decode VP8: whichever state it reports, metadata is still read.
|
||||
await expect.poll(() => video.evaluate(node => node.readyState >= 1 || !!node.error)).toBe(true);
|
||||
await openViewer(page);
|
||||
await expect(rail(page).locator('.prompt-section.positive p.prompt-text')).toHaveText('a red fox in snow, detailed fur');
|
||||
await expect(rail(page).locator('.meta-grid').first().locator('dd').first()).toHaveText('7');
|
||||
await expect(viewer(page).locator('#viewer-stage video')).toHaveCount(1);
|
||||
await page.keyboard.press('Escape');
|
||||
await choose(page, tinyWebm());
|
||||
await expect(preview(page).locator('video.local-file-preview')).toBeVisible();
|
||||
await openViewer(page);
|
||||
await expect(rail(page)).toContainText('No supported embedded metadata was recorded in this file.');
|
||||
if (browserName !== 'webkit') await expect.poll(() => viewer(page).locator('#viewer-stage video').evaluate(node => node.readyState >= 1)).toBe(true);
|
||||
});
|
||||
|
||||
test('the toast and a rejected drop never contradict the status line', async ({ page }) => {
|
||||
// A file whose bytes are gone after choosing: the status says its metadata cannot be read, so no toast says it is
|
||||
// available.
|
||||
await page.addInitScript(() => {
|
||||
const slice = Blob.prototype.slice;
|
||||
Blob.prototype.slice = function (...args) {
|
||||
if (this.name === 'gone.png') return { arrayBuffer: () => Promise.reject(new DOMException('The file is gone', 'NotReadableError')) };
|
||||
return slice.apply(this, args);
|
||||
};
|
||||
});
|
||||
await openInspect(page);
|
||||
await choose(page, parityPng('f02.png', 'gone.png'));
|
||||
await expect(status(page)).toHaveText('Embedded metadata could not be read.');
|
||||
await expect(action(page)).toHaveAttribute('aria-disabled', 'false');
|
||||
await expect(page.locator('#workspace-toast')).toHaveText('');
|
||||
// A rejected drop is its own alert beside the selection's status, which it never replaces, and it ends with the
|
||||
// concept's toast interval.
|
||||
await choose(page, undecodableMp4());
|
||||
const failed = status(page).filter({ hasText: COPY.previewFailed });
|
||||
await expect(failed).toHaveCount(1);
|
||||
await drop(page, '.file-drop', { files: [parityPng('f02.png'), parityPng('f03.png')] });
|
||||
const rejected = status(page).filter({ hasText: COPY.multiple });
|
||||
await expect(rejected).toHaveAttribute('role', 'alert');
|
||||
await expect(failed).toHaveCount(1);
|
||||
await expect(rejected).toHaveCount(0, { timeout: 15000 });
|
||||
await expect(status(page)).toHaveText([COPY.previewFailed]);
|
||||
// The next selection attempt ends a rejection at once.
|
||||
await drop(page, '.file-drop', { files: [parityPng('f02.png'), parityPng('f03.png')] });
|
||||
await expect(rejected).toHaveCount(1);
|
||||
await chooser(page).dispatchEvent('click');
|
||||
await chooser(page).dispatchEvent('cancel');
|
||||
await expect(status(page)).toHaveText([COPY.previewFailed]);
|
||||
});
|
||||
|
||||
test('undecodable video keeps container metadata', async ({ page }) => {
|
||||
await openInspect(page);
|
||||
// A real MP4 whose sample entry names a codec no engine decodes; its container tags are intact.
|
||||
await choose(page, undecodableMp4());
|
||||
await expect(status(page)).toHaveText(COPY.previewFailed);
|
||||
await expect(status(page)).toHaveAttribute('role', 'alert');
|
||||
await expect(preview(page).locator('video')).toHaveCount(0);
|
||||
await openViewer(page);
|
||||
await expect(rail(page).locator('.prompt-section.positive p.prompt-text')).toHaveText('a red fox in snow, detailed fur');
|
||||
await expect(rail(page).locator('.meta-grid').first().locator('dd').first()).toHaveText('7');
|
||||
});
|
||||
|
||||
/** Records, for each revoked URL, how many elements (inline or in the viewer) still held it, and counts resets. */
|
||||
const trackReleases = page => page.evaluate(() => {
|
||||
window.__released = [];
|
||||
const revoke = URL.revokeObjectURL;
|
||||
URL.revokeObjectURL = url => {
|
||||
const live = [...document.querySelectorAll('img, video')].filter(node => node.getAttribute('src') === url);
|
||||
window.__released.push({ url, live: live.length });
|
||||
return revoke(url);
|
||||
};
|
||||
const load = HTMLMediaElement.prototype.load;
|
||||
window.__loads = 0;
|
||||
HTMLMediaElement.prototype.load = function () { if (!this.hasAttribute('src')) window.__loads++; return load.call(this); };
|
||||
});
|
||||
|
||||
test('URLs and media end with their owning session', async ({ page }) => {
|
||||
await instrument(page);
|
||||
await openInspect(page);
|
||||
// Every release pauses the media and drops its src before the URL is revoked.
|
||||
await trackReleases(page);
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(preview(page).locator('video')).toBeVisible();
|
||||
// A settled video (metadata read, or failed) is reset with load() when it is released.
|
||||
await expect.poll(() => preview(page).locator('video').evaluate(node => node.readyState >= 1 || !!node.error)).toBe(true);
|
||||
// Replacement.
|
||||
await choose(page, parityPng('f02.png'));
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
let state = await urls(page);
|
||||
expect(state.revoked).toEqual([state.created[0]]);
|
||||
expect(await page.evaluate(() => window.__loads)).toBeGreaterThanOrEqual(1);
|
||||
// Route change.
|
||||
await page.locator('aside.sidebar nav').getByRole('link', { name: 'Overview', exact: true }).click();
|
||||
await expect(main(page).getByRole('heading', { level: 1, name: 'Overview', exact: true })).toBeVisible();
|
||||
state = await urls(page);
|
||||
expect(state.revoked).toEqual(state.created);
|
||||
// Five mount/unmount cycles with a selection and an opened viewer leave no live URL and no dialog.
|
||||
for (let cycle = 0; cycle < 5; cycle++) {
|
||||
await page.locator('aside.sidebar nav').getByRole('link', { name: 'Inspect a file', exact: true }).click();
|
||||
await choose(page, parityPng('f02.png', `cycle-${cycle}.png`));
|
||||
await openViewer(page);
|
||||
// The modal viewer makes the sidebar inert, so the route changes underneath it.
|
||||
await page.evaluate(() => { location.hash = '#/overview'; });
|
||||
await expect(main(page).getByRole('heading', { level: 1, name: 'Overview', exact: true })).toBeVisible();
|
||||
}
|
||||
state = await urls(page);
|
||||
expect(state.created).toHaveLength(7);
|
||||
expect([...state.revoked].sort()).toEqual([...state.created].sort());
|
||||
expect(new Set(state.revoked).size).toBe(state.revoked.length);
|
||||
expect((await page.evaluate(() => window.__released)).every(entry => entry.live === 0)).toBe(true);
|
||||
await expect(page.locator('dialog')).toHaveCount(0);
|
||||
// Page leave releases the selection once.
|
||||
await page.locator('aside.sidebar nav').getByRole('link', { name: 'Inspect a file', exact: true }).click();
|
||||
await choose(page, parityPng('f02.png', 'leave.png'));
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
await page.evaluate(() => window.dispatchEvent(new PageTransitionEvent('pagehide', { persisted: false })));
|
||||
state = await urls(page);
|
||||
expect(state.created).toHaveLength(8);
|
||||
expect(state.revoked.filter(url => url === state.created[7])).toHaveLength(1);
|
||||
// Page leave with the viewer open, on a page loaded at Inspect (so its listeners precede the shell's): the viewer
|
||||
// closes and releases its media before the URL is revoked, and a page restored from the back-forward cache shows
|
||||
// an empty chooser beside the empty selection.
|
||||
await page.reload();
|
||||
await expect(chooser(page)).toBeAttached();
|
||||
await trackReleases(page);
|
||||
await choose(page, parityPng('f02.png', 'leave-open.png'));
|
||||
await openViewer(page);
|
||||
await expect(viewer(page).getByRole('button', { name: 'Zoom in', exact: true })).toHaveAttribute('aria-disabled', 'false');
|
||||
await page.evaluate(() => window.dispatchEvent(new PageTransitionEvent('pagehide', { persisted: true })));
|
||||
await expect(page.locator('dialog')).toHaveCount(0);
|
||||
state = await urls(page);
|
||||
expect(state.created).toHaveLength(1);
|
||||
expect(state.revoked).toEqual(state.created);
|
||||
expect(await page.evaluate(() => window.__released)).toEqual([{ url: state.created[0], live: 0 }]);
|
||||
await page.evaluate(() => window.dispatchEvent(new PageTransitionEvent('pageshow', { persisted: true })));
|
||||
await expect(chooser(page)).toBeAttached();
|
||||
expect(await chooser(page).evaluate(node => ({ value: node.value, files: node.files.length }))).toEqual({ value: '', files: 0 });
|
||||
await expect(preview(page).locator('img, video')).toHaveCount(0);
|
||||
await expect(action(page)).toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
|
||||
test('replacing a video that just started loading never blocks the page', async ({ context, browserName }) => {
|
||||
test.setTimeout(120000);
|
||||
// WebKit (GStreamer) blocked the page when a video whose pipeline was still starting was reset with load(): the
|
||||
// next chooser selection, or the steps after it, never completed. Each fresh page replaces its first video as
|
||||
// soon as it shows.
|
||||
for (let attempt = 0; attempt < 15; attempt++) {
|
||||
const page = await context.newPage();
|
||||
try {
|
||||
if (attempt === 0) await startCoverage(page, browserName);
|
||||
await openInspect(page);
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(preview(page).locator('video.local-file-preview')).toBeVisible();
|
||||
await chooser(page).setInputFiles(parityPng('f02.png'), { timeout: 8000 });
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
await expect(preview(page).locator('video')).toHaveCount(0);
|
||||
if (attempt === 0) await saveCoverage(page, browserName);
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('a video replaced while still starting is reset once it settles', async ({ context, browserName }) => {
|
||||
test.setTimeout(180000);
|
||||
// The video is replaced from its own `loadstart` (readyState HAVE_NOTHING), or a few milliseconds after it. On
|
||||
// WebKit (GStreamer) a video that left the page or the viewport before its pipeline had started deadlocked the
|
||||
// page (TESTING.md D62), so it stays in place, inert and still playing its URL, with the new preview after it.
|
||||
// Once it reports metadata or an error it is paused, loses its src, is reset (load() with no src, which frees its
|
||||
// decoder) and leaves the page, and only then is its URL revoked. Fresh pages share one browser process.
|
||||
for (let attempt = 0; attempt < 15; attempt++) {
|
||||
const page = await context.newPage();
|
||||
try {
|
||||
if (attempt === 0) await startCoverage(page, browserName);
|
||||
await page.addInitScript(() => {
|
||||
const load = HTMLMediaElement.prototype.load;
|
||||
HTMLMediaElement.prototype.load = function () { if (!this.hasAttribute('src')) this.__resets = (this.__resets ?? 0) + 1; return load.call(this); };
|
||||
const revoke = URL.revokeObjectURL;
|
||||
URL.revokeObjectURL = url => {
|
||||
const starting = window.__starting;
|
||||
if (url === starting?.url) starting.revoked = { src: starting.node.hasAttribute('src'), connected: starting.node.isConnected, resets: starting.node.__resets ?? 0 };
|
||||
return revoke(url);
|
||||
};
|
||||
});
|
||||
await openInspect(page);
|
||||
await page.evaluate(() => {
|
||||
const input = document.querySelector('#local-file'), slot = document.querySelector('#local-preview');
|
||||
new MutationObserver((records, observer) => {
|
||||
const video = slot.querySelector('video');
|
||||
if (!video) return;
|
||||
observer.disconnect();
|
||||
const replace = () => {
|
||||
window.__starting = { node: video, url: video.getAttribute('src'), readyState: video.readyState };
|
||||
input.files = window.__pngFiles; input.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
window.__starting.replaced = { connected: video.isConnected, src: video.hasAttribute('src'), inert: video.inert, resets: video.__resets ?? 0 };
|
||||
};
|
||||
video.addEventListener('loadstart', () => { if (window.__delay) setTimeout(replace, window.__delay); else replace(); }, { once: true });
|
||||
}).observe(slot, { childList: true });
|
||||
});
|
||||
const png = parityPng('f02.png');
|
||||
await page.evaluate(bytes => {
|
||||
const transfer = new DataTransfer();
|
||||
transfer.items.add(new File([new Uint8Array(bytes)], 'f02.png', { type: 'image/png' }));
|
||||
window.__pngFiles = transfer.files;
|
||||
}, [...png.buffer]);
|
||||
await page.evaluate(delay => { window.__delay = delay; }, attempt % 5);
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible({ timeout: 8000 });
|
||||
await expect(preview(page).locator('video')).toHaveCount(0);
|
||||
// Replaced while starting (always so at `loadstart` itself): kept in place and untouched. A video that had
|
||||
// settled is released at once. Either way it is reset once, and its URL goes last.
|
||||
await expect.poll(() => page.evaluate(() => window.__starting.revoked ?? null)).toEqual({ src: false, connected: false, resets: 1 });
|
||||
const { readyState, replaced } = await page.evaluate(() => ({ readyState: window.__starting.readyState, replaced: window.__starting.replaced }));
|
||||
if (attempt % 5 === 0) expect(readyState).toBe(0);
|
||||
expect(replaced).toEqual(readyState === 0 ? { connected: true, src: true, inert: true, resets: 0 } : { connected: false, src: false, inert: false, resets: 1 });
|
||||
expect(await page.evaluate(() => window.__starting.node.__resets)).toBe(1);
|
||||
// The page still answers.
|
||||
await chooser(page).setInputFiles(parityPng('f02.png', 'after.png'), { timeout: 8000 });
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible({ timeout: 8000 });
|
||||
if (attempt === 0) await saveCoverage(page, browserName);
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('leaving for another route while a video starts waits for it to settle', async ({ context, browserName }) => {
|
||||
test.setTimeout(180000);
|
||||
// A WebKit (GStreamer) video detached before its pipeline had started deadlocked the page (TESTING.md D62); a
|
||||
// route change detaches the whole page, so the shell renders the next route once the video reports metadata or an
|
||||
// error. The route changes from the video's own `loadstart`, or a few milliseconds after it.
|
||||
for (let attempt = 0; attempt < 15; attempt++) {
|
||||
const page = await context.newPage();
|
||||
try {
|
||||
if (attempt === 0) await startCoverage(page, browserName);
|
||||
await page.addInitScript(() => {
|
||||
const load = HTMLMediaElement.prototype.load;
|
||||
HTMLMediaElement.prototype.load = function () {
|
||||
if (!this.hasAttribute('src') && window.__starting?.node === this) window.__starting.reset = { readyState: this.readyState, failed: this.error !== null, connected: this.isConnected };
|
||||
return load.call(this);
|
||||
};
|
||||
});
|
||||
await openInspect(page);
|
||||
await page.evaluate(delay => {
|
||||
const slot = document.querySelector('#local-preview');
|
||||
new MutationObserver((records, observer) => {
|
||||
const video = slot.querySelector('video');
|
||||
if (!video) return;
|
||||
observer.disconnect();
|
||||
const leave = () => {
|
||||
window.__starting = { node: video, readyState: video.readyState };
|
||||
location.hash = '#/overview';
|
||||
};
|
||||
video.addEventListener('loadstart', () => { if (delay) setTimeout(leave, delay); else leave(); }, { once: true });
|
||||
}).observe(slot, { childList: true });
|
||||
}, attempt % 5);
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(main(page).getByRole('heading', { level: 1, name: 'Overview', exact: true })).toBeVisible({ timeout: 8000 });
|
||||
const { readyState, reset } = await page.evaluate(() => ({ readyState: window.__starting.readyState, reset: window.__starting.reset }));
|
||||
if (attempt % 5 === 0) expect(readyState).toBe(0);
|
||||
// The video was reset and left the page only once it had settled.
|
||||
expect(reset.readyState >= 1 || reset.failed).toBe(true);
|
||||
expect(reset.connected).toBe(true);
|
||||
// The page still answers.
|
||||
await page.locator('aside.sidebar nav').getByRole('link', { name: 'Inspect a file', exact: true }).click();
|
||||
await chooser(page).setInputFiles(parityPng('f02.png'), { timeout: 8000 });
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible({ timeout: 8000 });
|
||||
if (attempt === 0) await saveCoverage(page, browserName);
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('closing the viewer on a video that just started never blocks the page', async ({ context, browserName }) => {
|
||||
test.setTimeout(180000);
|
||||
// The viewer's video teardown resets its element; on WebKit a reset of a still-starting pipeline blocked the page.
|
||||
for (let attempt = 0; attempt < 15; attempt++) {
|
||||
const page = await context.newPage();
|
||||
try {
|
||||
if (attempt === 0) await startCoverage(page, browserName);
|
||||
await openInspect(page);
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await openViewer(page);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer(page)).toHaveCount(0, { timeout: 8000 });
|
||||
await chooser(page).setInputFiles(parityPng('f02.png'), { timeout: 8000 });
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible({ timeout: 8000 });
|
||||
if (attempt === 0) await saveCoverage(page, browserName);
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('replacing a file mid-read never shows or sends the old record', async ({ page, context }) => {
|
||||
const host = await bridgeHost(context);
|
||||
try {
|
||||
await openInspect(page);
|
||||
// Hold every read of held.png after its classification prefix.
|
||||
await page.evaluate(() => {
|
||||
let release;
|
||||
window.__gate = new Promise(resolve => { release = resolve; });
|
||||
window.__releaseGate = release;
|
||||
window.__held = 0;
|
||||
const slice = Blob.prototype.slice;
|
||||
Blob.prototype.slice = function (...args) {
|
||||
const part = slice.apply(this, args);
|
||||
if (this.name !== 'held.png' || (args[0] ?? 0) === 0 && (args[1] ?? this.size) <= 32) return part;
|
||||
const read = part.arrayBuffer.bind(part);
|
||||
part.arrayBuffer = async () => { window.__held++; await window.__gate; return read(); };
|
||||
return part;
|
||||
};
|
||||
});
|
||||
await choose(page, parityPng('f03.png', 'held.png'));
|
||||
await expect.poll(() => page.evaluate(() => window.__held)).toBeGreaterThan(0);
|
||||
await choose(page, sendablePng('fresh.png'));
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
await page.evaluate(() => window.__releaseGate());
|
||||
await openViewer(page);
|
||||
await expect(viewer(page).locator('#viewer-title')).toHaveText('fresh.png');
|
||||
await expect(rail(page).locator('.prompt-section.positive p.prompt-text')).toHaveText(POSITIVE_LINES);
|
||||
await sendPair(page);
|
||||
expect(await host.evaluate(() => window.inspectApplied)).toEqual([{ positive: POSITIVE_LINES, negative: '' }]);
|
||||
} finally {
|
||||
await host.evaluate(() => window.inspectHost.dispose());
|
||||
await host.close();
|
||||
}
|
||||
});
|
||||
|
||||
/** A second tab acting as a ComfyUI host with one Prompt Manager pair destination. */
|
||||
async function bridgeHost(context) {
|
||||
const host = await context.newPage();
|
||||
await host.goto(`${workspace}#/overview`);
|
||||
await host.evaluate(async () => {
|
||||
const { createBridgeHost } = await import('/prompt_manager/js/shared/toolkit/prompt-bridge.js');
|
||||
const target = { id: 'pair-1', label: 'Inspect destination', type: 'PromptManagerPair', roles: ['positive', 'negative'] };
|
||||
window.inspectApplied = [];
|
||||
window.inspectPreviews = [];
|
||||
window.inspectHost = createBridgeHost({ origin: location.origin, namespace: '/', label: 'Inspect ComfyUI', adapter: {
|
||||
listTargets: () => [target],
|
||||
preview: (_id, roles) => {
|
||||
window.inspectPreviews.push(roles);
|
||||
return { preview: { target, fields: Object.entries(roles).map(([role, after]) => ({ role, before: `Previous ${role}`, after })),
|
||||
composition: { prepend: '', append: '', prependConnected: false, appendConnected: false, before: 'Previous positive', after: roles.positive ?? 'Previous positive' } }, snapshot: roles };
|
||||
},
|
||||
apply: roles => { window.inspectApplied.push(roles); return { targetId: target.id, values: roles }; },
|
||||
} });
|
||||
});
|
||||
return host;
|
||||
}
|
||||
|
||||
async function sendPair(page) {
|
||||
await rail(page).getByRole('button', { name: /^Send prompt pair to ComfyUI: / }).click();
|
||||
const sender = page.getByRole('dialog', { name: 'Send to ComfyUI', exact: true });
|
||||
await sender.getByRole('combobox', { name: 'ComfyUI window', exact: true }).selectOption({ label: 'Inspect ComfyUI' });
|
||||
await sender.getByRole('combobox', { name: 'Destination node', exact: true }).selectOption({ label: 'Inspect destination' });
|
||||
await sender.getByRole('button', { name: 'Preview changes', exact: true }).click();
|
||||
await sender.getByRole('button', { name: 'Apply to ComfyUI', exact: true }).click();
|
||||
await expect(sender).toContainText('Sent to ComfyUI.');
|
||||
await sender.getByRole('button', { name: 'Close', exact: true }).click();
|
||||
}
|
||||
|
||||
test('local roles send exactly', async ({ page, context }) => {
|
||||
const host = await bridgeHost(context);
|
||||
try {
|
||||
await openInspect(page);
|
||||
await choose(page, sendablePng());
|
||||
await openViewer(page);
|
||||
await expect(rail(page).locator('.prompt-section.positive p.prompt-text')).toHaveText(POSITIVE_LINES);
|
||||
await expect(rail(page).locator('.prompt-section.negative p.prompt-text')).toHaveText('No negative prompt (explicitly empty)');
|
||||
await sendPair(page);
|
||||
const applied = await host.evaluate(() => window.inspectApplied);
|
||||
expect(applied).toEqual([{ positive: POSITIVE_LINES, negative: '' }]);
|
||||
const payload = JSON.stringify([applied, await host.evaluate(() => window.inspectPreviews)]);
|
||||
expect(payload).not.toContain('blob:');
|
||||
expect(payload).not.toContain('send-me');
|
||||
// The viewer is still open, focus came back to the send action.
|
||||
await expect(viewer(page)).toBeVisible();
|
||||
} finally {
|
||||
await host.evaluate(() => window.inspectHost.dispose());
|
||||
await host.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('local inspection makes no content request or storage write', async ({ page }) => {
|
||||
await instrument(page);
|
||||
const requests = [];
|
||||
page.on('request', request => requests.push({ url: request.url(), method: request.method(), body: request.postData() }));
|
||||
page.on('websocket', socket => requests.push({ url: socket.url(), method: 'WS', body: null }));
|
||||
const table = /\/prompt_manager\/js\/inspection-semantics\.json$/;
|
||||
const tableRead = page.waitForResponse(response => table.test(response.url()));
|
||||
await openInspect(page);
|
||||
// The one static file Inspect reads, its node table, is fetched as the page mounts: before any selection.
|
||||
expect((await tableRead).ok()).toBe(true);
|
||||
// The shell's own reads (counts, Activity) settle before the baseline.
|
||||
await expect(page.locator('aside.sidebar nav [data-route="images"] .nav-count')).toBeAttached();
|
||||
const baseline = (await urls(page)).writes.length;
|
||||
const before = requests.length;
|
||||
expect(requests.filter(entry => table.test(entry.url))).toHaveLength(1);
|
||||
for (const file of [sendablePng('private-name.png'), localFixture('core-tags.webm', 'video/webm', 'private-clip.webm')]) {
|
||||
await choose(page, file);
|
||||
await openViewer(page);
|
||||
await rail(page).locator('summary', { hasText: 'Workflow & raw metadata' }).click();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer(page)).toHaveCount(0);
|
||||
}
|
||||
const network = requests.slice(before).filter(entry => !entry.url.startsWith('blob:') && !entry.url.startsWith('data:'));
|
||||
expect(requests.filter(entry => table.test(entry.url)), 'the table is read once per page').toHaveLength(1);
|
||||
expect(network.filter(entry => /private|lighthouse|fox/.test(`${entry.url} ${entry.body ?? ''}`))).toEqual([]);
|
||||
expect(network.filter(entry => entry.method !== 'GET')).toEqual([]);
|
||||
expect(network.filter(entry => !/\/prompt_manager\/(activity|images\/output|library|vocabulary)/.test(entry.url))).toEqual([]);
|
||||
expect((await urls(page)).writes.slice(baseline)).toEqual([]);
|
||||
});
|
||||
|
||||
test('touch layout at 390 has no horizontal overflow at 200% zoom', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
await openInspect(page);
|
||||
await choose(page, parityPng('f02.png'));
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
for (const zoom of ['1', '2']) {
|
||||
await page.evaluate(value => { document.documentElement.style.zoom = value; }, zoom);
|
||||
await expect.poll(() => page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true);
|
||||
const area = await page.locator('.file-drop').evaluate(node => node.getBoundingClientRect().right <= document.documentElement.clientWidth + 0.5);
|
||||
expect(area).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
// LI-4 (D117): "Copy settings summary" after the six facts is the legacy metadata page's COPY ALL, in the shared full
|
||||
// inspector, so a file served in Images and the same bytes chosen here copy the same summary of what the rail shows.
|
||||
/** Records clipboard writes in the page, or (`clipboard: false`) leaves the page without a clipboard. */
|
||||
async function stubClipboard(page, { clipboard = true } = {}) {
|
||||
await page.addInitScript(available => {
|
||||
window.__copied = [];
|
||||
Object.defineProperty(navigator, 'clipboard', { configurable: true,
|
||||
value: available ? { writeText: async text => { window.__copied.push(text); } } : undefined });
|
||||
}, clipboard);
|
||||
}
|
||||
/** The summary the rail's own prompts and facts make, in the legacy COPY ALL lines and placeholders. */
|
||||
const expectedSummary = locator => locator.evaluate(node => {
|
||||
const facts = Object.fromEntries([...node.querySelectorAll(':scope .meta-grid:not(details .meta-grid) > div')]
|
||||
.map(item => [item.querySelector('dt').textContent, item.querySelector('dd').textContent]));
|
||||
// A confirmed prompt has its Copy action; an explicitly empty one takes the legacy placeholder too.
|
||||
const prompt = role => {
|
||||
const text = node.querySelector(`.prompt-section.${role} [data-copy-role]`) ? node.querySelector(`.prompt-section.${role} p.prompt-text`).textContent : null;
|
||||
return text === `No ${role} prompt (explicitly empty)` ? null : text;
|
||||
};
|
||||
return [`Checkpoint: ${facts.Model}`, `Positive Prompt: ${prompt('positive') ?? 'No prompt found'}`,
|
||||
`Negative Prompt: ${prompt('negative') ?? 'No negative prompt found'}`, `Steps: ${facts.Steps}`, `CFG Scale: ${facts.CFG}`,
|
||||
`Sampler: ${facts.Sampler}`, `Seed: ${facts.Seed}`].join('\n');
|
||||
});
|
||||
async function copySummary(page) {
|
||||
const button = rail(page).getByRole('button', { name: 'Copy settings summary', exact: true });
|
||||
// It follows the six facts, before "File & provenance".
|
||||
expect(await button.evaluate(node => [node.previousElementSibling?.localName, node.nextElementSibling?.localName])).toEqual(['dl', 'details']);
|
||||
await button.click();
|
||||
}
|
||||
|
||||
test('Copy settings summary copies the same legacy summary in Images and Inspect', async ({ page, request }) => {
|
||||
await stubClipboard(page);
|
||||
await expect(await request.post('/__test__/add-local-inspect-file', { data: { name: 'f02.png' } })).toBeOK();
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await page.locator('article[data-image-key="0:local-inspect/f02.png"] .image-open').click();
|
||||
await expect(rail(page).locator('.prompt-section.positive p.prompt-text')).toBeVisible();
|
||||
const served = await expectedSummary(rail(page));
|
||||
expect(served).toMatch(/^Checkpoint: .+\nPositive Prompt: [\s\S]+\nNegative Prompt: [\s\S]+\nSteps: .+\nCFG Scale: .+\nSampler: .+\nSeed: .+$/);
|
||||
expect(served).not.toContain('Unknown');
|
||||
await copySummary(page);
|
||||
await expect(page.locator('#workspace-toast')).toHaveText('Settings summary copied.');
|
||||
expect(await page.evaluate(() => window.__copied)).toEqual([served]);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer(page)).toHaveCount(0);
|
||||
|
||||
await openInspect(page);
|
||||
await choose(page, parityPng('f02.png'));
|
||||
await openViewer(page);
|
||||
expect(await expectedSummary(rail(page))).toBe(served);
|
||||
await copySummary(page);
|
||||
await expect.poll(() => page.evaluate(() => window.__copied)).toEqual([served, served]);
|
||||
// The panel variant (the Images side panel) keeps the concept's footer: no summary action there.
|
||||
await page.keyboard.press('Escape');
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(page.locator('#image-inspector .panel-heading')).toBeVisible();
|
||||
await expect(page.locator('#image-inspector').getByRole('button', { name: 'Copy settings summary', exact: true })).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('Copy settings summary without a clipboard opens the select-to-copy dialog', async ({ page }) => {
|
||||
await stubClipboard(page, { clipboard: false });
|
||||
await openInspect(page);
|
||||
await choose(page, sendablePng());
|
||||
await openViewer(page);
|
||||
const summary = await expectedSummary(rail(page));
|
||||
expect(summary).toContain(`Positive Prompt: ${POSITIVE_LINES}`);
|
||||
await copySummary(page);
|
||||
const fallback = page.getByRole('dialog', { name: 'Copy settings summary', exact: true });
|
||||
await expect(fallback).toBeVisible();
|
||||
await expect(fallback.getByRole('textbox', { name: 'Select and copy this text', exact: true })).toHaveValue(summary);
|
||||
await expect(fallback.getByRole('textbox', { name: 'Select and copy this text', exact: true })).toBeFocused();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(fallback).toHaveCount(0);
|
||||
// Escape closes only the copy dialog; focus returns to the action in the still-open viewer.
|
||||
await expect(viewer(page)).toBeVisible();
|
||||
await expect(rail(page).getByRole('button', { name: 'Copy settings summary', exact: true })).toBeFocused();
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {inventoryOf,mapConcept,diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
|
||||
const route='/prompt_manager/workspace#/library';
|
||||
const cards=page=>page.locator('article[data-prompt-id]');
|
||||
@@ -25,6 +25,8 @@ async function loadShelfPages(page,more,pages) {
|
||||
await more.evaluate(node=>node.click());await response;await expect(more).toHaveAttribute('aria-disabled','false');
|
||||
}
|
||||
}
|
||||
/** Resolves in the rendering frame after every frame callback the page had already queued (callbacks run in request order). */
|
||||
const nextFrame=page=>page.evaluate(()=>new Promise(resolve=>requestAnimationFrame(()=>resolve())));
|
||||
async function openHistory(page,text='Cat portrait') {
|
||||
await cards(page).filter({has:page.getByText(text,{exact:true})}).getByRole('button',{name:'Explore history',exact:true}).click();
|
||||
await expect(history(page)).toBeVisible();
|
||||
@@ -103,7 +105,8 @@ test('library infinite loading keeps unique parents and keyboard-accessible hist
|
||||
const appended=await (await nextPage).json();expect(appended.items.length).toBeGreaterThan(0);hasMore=appended.has_more;
|
||||
if(hasMore)await expect(more).toHaveAttribute('aria-disabled','false');
|
||||
}
|
||||
await expect(more).toBeHidden();expect(await cards(page).count()).toBeLessThan(80);
|
||||
// The last page's render queued the window update for the next frame; count after that frame, not before it.
|
||||
await expect(more).toBeHidden();await nextFrame(page);expect(await cards(page).count()).toBeLessThan(80);
|
||||
const ids=await cards(page).evaluateAll(nodes=>nodes.map(node=>node.dataset.promptId));expect(new Set(ids).size).toBe(ids.length);
|
||||
await expect(page.getByRole('button',{name:/next page|previous page/i})).toHaveCount(0);
|
||||
const firstId=await cards(page).first().getAttribute('data-prompt-id');
|
||||
@@ -253,6 +256,19 @@ test('returning from history restores library context and mobile content stays i
|
||||
await expect(history(page)).toHaveCount(0);await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('leaving the shelf before its first page answers and returning reads page one without a page error',async({page})=>{
|
||||
// Hold the first shelf page so the route changes while the shelf is still loading (a slow CI runner's natural order).
|
||||
let reached;const firstRead=new Promise(resolve=>{reached=resolve;});
|
||||
await page.route(url=>url.pathname==='/prompt_manager/library'&&!url.searchParams.has('cursor'),route=>{reached();});
|
||||
await page.goto(route);await firstRead;await expect(page.locator('.library-status')).toHaveText('Loading prompt library…');
|
||||
await page.evaluate(()=>{window.location.hash='#/tags';});
|
||||
await expect(page.locator('#workspace-main > .heading h1')).toHaveText('Tags & categories');
|
||||
await page.unrouteAll({behavior:'ignoreErrors'});
|
||||
await page.evaluate(()=>{window.location.hash='#/library';});
|
||||
await expect(page.locator('#workspace-main > .heading h1')).toHaveText('Prompt library');
|
||||
await expect(cards(page)).toHaveCount(3);await expect(page.locator('.section-caption')).toContainText('3 positive prompts');
|
||||
expect(page.libraryErrors).toEqual([]);
|
||||
});
|
||||
test('empty shelf visits restore saved scroll without inventing a parent anchor',async({page})=>{
|
||||
await page.setViewportSize({width:1600,height:300});await page.goto(route);
|
||||
await page.getByRole('searchbox',{name:'Search workspace',exact:true}).fill('No matching saved positive');
|
||||
@@ -376,14 +392,18 @@ async function observeLibrary(page, hold=false) {
|
||||
holdFrames(){holdingFrames=true;},
|
||||
releaseFrames(){holdingFrames=false;const callbacks=[...heldFrames.values()];heldFrames.clear();for(const callback of callbacks)callback();},
|
||||
equal(){for(const record of records){const entries=[...record.latest.values()].filter(entry=>record.targets.has(entry.target));if(entries.length)record.callback(entries,record.native);}},
|
||||
// Resolves once two consecutive rendering frames find no Library frame queued. Each check runs inside a native
|
||||
// frame callback, so a frame the page queues later in that update (a scroll event, a ResizeObserver delivery) is
|
||||
// still pending there. A timer is not a frame: under CPU load a 60 ms wait could end before the scroll's frame.
|
||||
// A frame that never drains (a rAF loop, or settling while frames are held) fails after `limit` frames with the stats.
|
||||
settled(limit=600){return new Promise((resolve,reject)=>{let quiet=0,frames=0;const check=()=>{frames++;quiet=pending.size?0:quiet+1;
|
||||
if(quiet>=2)resolve();else if(frames>=limit)reject(new Error(`Library never settled within ${limit} frames: ${JSON.stringify(window.libraryObservation.stats())}`));else raf(check);};raf(check);});},
|
||||
stats(){return {requested,pending:pending.size,heldFrames:heldFrames.size,measurements:records.reduce((sum,record)=>sum+record.entries.size,0),mutations,unmounted,targets:records.reduce((sum,record)=>sum+record.targets.size,0)};},
|
||||
};
|
||||
},{hold});
|
||||
}
|
||||
async function settleLibrary(page) {
|
||||
await expect.poll(async()=>page.evaluate(()=>window.libraryObservation.stats().pending)).toBe(0);
|
||||
await page.evaluate(()=>new Promise(resolve=>setTimeout(resolve,60)));
|
||||
await expect.poll(async()=>page.evaluate(()=>window.libraryObservation.stats().pending)).toBe(0);
|
||||
await page.evaluate(()=>window.libraryObservation.settled());
|
||||
}
|
||||
async function firstVisibleParent(page) {
|
||||
return cards(page).evaluateAll(nodes=>{
|
||||
@@ -590,7 +610,9 @@ test('card matches the concept and never wraps the category mid-word',async({pag
|
||||
await expect(await request.post('/prompt_manager/bulk/category',{data:{prompt_ids:[dual.id],category:''}})).toBeOK();
|
||||
// A long saved title fills the title row, so only the chip's own min-content keeps "Uncategorized" and "asian" whole.
|
||||
const wide='A remarkably long saved positive prompt title that fills the whole card title row';
|
||||
await patchShelf(page,item=>item.id===cat.id?{run_count:4,last_used_at:new Date().toISOString(),rating:5}:item.id===dual.id||item.id===dog.id?{title:wide,run_count:1,last_used_at:null,rating:null}:{});
|
||||
// One "now" for the patched last use and the page's clock, so it reads "Used today" on either side of midnight.
|
||||
const usedAt=new Date();await page.clock.setFixedTime(usedAt);
|
||||
await patchShelf(page,item=>item.id===cat.id?{run_count:4,last_used_at:usedAt.toISOString(),rating:5}:item.id===dual.id||item.id===dog.id?{title:wide,run_count:1,last_used_at:null,rating:null}:{});
|
||||
for(const [width,height] of [[1600,1100],[390,844]]) {
|
||||
await page.setViewportSize({width,height});await page.goto(route);await expect(cards(page)).toHaveCount(3);
|
||||
const card=page.locator(`article[data-prompt-id="${cat.id}"]`);
|
||||
@@ -683,6 +705,17 @@ test('title and Explore history are buttons that open history',async({page,reque
|
||||
await page.keyboard.press('ArrowLeft');await expect(cards(page).nth(1).locator('.card-title > button')).toBeFocused();
|
||||
});
|
||||
|
||||
test('settling fails with the observation stats when a Library frame stays queued',async({page,request})=>{
|
||||
await request.post('/__test__/reset',{data:{native_library:true,extra_prompts:200}});
|
||||
await observeLibrary(page);await page.addInitScript(()=>{window.IntersectionObserver=undefined;});
|
||||
await page.goto(route);await expect(cards(page).first()).toBeVisible();await settleLibrary(page);
|
||||
await page.evaluate(()=>{window.libraryObservation.holdFrames();dispatchEvent(new Event('scroll'));});
|
||||
await expect.poll(()=>page.evaluate(()=>window.libraryObservation.stats().heldFrames)).toBeGreaterThan(0);
|
||||
const failure=await page.evaluate(()=>window.libraryObservation.settled(30).then(()=>'settled',error=>error.message));
|
||||
expect(failure).toMatch(/^Library never settled within 30 frames: \{.*"heldFrames":[1-9]/);
|
||||
await page.evaluate(()=>window.libraryObservation.releaseFrames());await settleLibrary(page);
|
||||
});
|
||||
|
||||
test('virtualized rows keep measured heights with the clamp',async({page,request,isMobile})=>{
|
||||
await request.post('/__test__/reset',{data:{native_library:true,extra_prompts:200}});
|
||||
await observeLibrary(page);await page.addInitScript(()=>{window.IntersectionObserver=undefined;});
|
||||
|
||||
@@ -0,0 +1,588 @@
|
||||
// Needs review (#249 PR F, plan Tasks F1 and F2, Mock inventory M4): the SAVED VIEWS row and its count, the Metadata
|
||||
// review page over the real review routes, and Activity's Review metadata. Items are recorded by the real recorder
|
||||
// (tests/browser/workspace_review_fixture.py) through the scan's recorder (only scans add items); the conflict item's file was linked
|
||||
// to another prompt after it was queued. An open page re-reads its queue from the review change stream (F2).
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
const review = '/prompt_manager/workspace#/review';
|
||||
const main = page => page.locator('#workspace-main');
|
||||
const savedNav = page => page.getByRole('navigation', { name: 'Saved views', exact: true });
|
||||
const reviewRow = page => savedNav(page).locator('a[data-route="review"]');
|
||||
const panel = page => main(page).getByRole('region', { name: 'Resolve the association', exact: true });
|
||||
const context = page => panel(page).getByRole('combobox', { name: 'Sampler context', exact: true });
|
||||
const cards = page => main(page).locator('.review-queue article.media-card');
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
const NOTICE = 'Multiple sampler paths were found. The existing image links were preserved. Choose the correct context or leave them unresolved.';
|
||||
const EMPTY = 'Ambiguous images are added when a metadata scan or a new generation finds more than one sampler path.';
|
||||
const OPTIONS = ['Sampler 11 · Golden dunes at dusk / Soft negative, blur', 'Sampler 13 · Harbor lights at night / Soft negative, blur',
|
||||
'Leave unresolved'];
|
||||
|
||||
async function seed(request, items = 3, special = {}) {
|
||||
const response = await request.post('/__test__/review/reset', { data: { items, ...special } });
|
||||
await expect(response).toBeOK();
|
||||
return response.json();
|
||||
}
|
||||
async function pending(request) {
|
||||
return (await (await request.get('/prompt_manager/review/items?limit=1')).json()).total;
|
||||
}
|
||||
const links = async request => (await (await request.get('/__test__/review/links')).json()).links;
|
||||
const streams = async request => (await (await request.get('/__test__/review/streams')).json());
|
||||
/**
|
||||
* Wait until the server has dropped every stream a page closed. It notices a closed stream only when a write to it
|
||||
* fails, and the close can reach it after any one write (WebKit under load): each poll sends one more change notice,
|
||||
* which changes nothing in the queue, until the count is 0.
|
||||
*/
|
||||
async function closedStreamsDropped(request) {
|
||||
await expect.poll(async () => {
|
||||
await expect(await request.post('/__test__/review/notify')).toBeOK();
|
||||
return (await streams(request)).streams;
|
||||
}).toBe(0);
|
||||
}
|
||||
const apply = page => panel(page).getByRole('button', { name: 'Apply association', exact: true });
|
||||
const warning = page => panel(page).locator('.warning-note');
|
||||
/** The page with its first item's sampler paths loaded. */
|
||||
async function open(page) {
|
||||
await page.goto(review);
|
||||
await expect(context(page)).toBeEnabled();
|
||||
}
|
||||
|
||||
test.beforeEach(async ({ page, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
page.reviewErrors = [];
|
||||
page.on('pageerror', error => page.reviewErrors.push(error.message));
|
||||
});
|
||||
test.afterEach(async ({ page, browserName, request }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.reviewErrors).toEqual([]);
|
||||
await request.post('/__test__/jobs/restart');
|
||||
});
|
||||
|
||||
test('queue shows persisted items and the nav count', async ({ page, request }) => {
|
||||
await seed(request, 3);
|
||||
await open(page);
|
||||
// SAVED VIEWS: Favorites, then Needs review with the exact pending count (ruling R5).
|
||||
await expect(savedNav(page).locator('a').nth(1)).toHaveAttribute('data-route', 'review');
|
||||
await expect(reviewRow(page)).toHaveText('Needs review3');
|
||||
await expect(reviewRow(page)).toHaveAttribute('aria-current', 'page');
|
||||
await expect(page).toHaveTitle('Review · Prompt Manager');
|
||||
await expect(page.locator('#workspace-breadcrumb')).toHaveText('Review');
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Metadata review');
|
||||
await expect(main(page).locator('.heading p.muted')).toHaveText('Uncertain evidence stays visible until you decide.');
|
||||
await expect(main(page).locator('.notice h2')).toHaveText('3 images need a decision');
|
||||
await expect(main(page).locator('.notice p')).toHaveText(NOTICE);
|
||||
await expect(cards(page)).toHaveCount(3);
|
||||
await expect(cards(page).first()).toHaveClass(/selected/);
|
||||
await expect(panel(page).locator('p.small')).toHaveText('review_0000.png');
|
||||
await expect(context(page).locator('option')).toHaveText(OPTIONS);
|
||||
await expect(context(page)).toHaveValue(/^[0-9a-f]{64}$/);
|
||||
await expect(panel(page).locator('.prompt-section.positive .prompt-text')).toHaveText('Golden dunes at dusk');
|
||||
await expect(panel(page).locator('.prompt-section.negative .prompt-text')).toHaveText('Soft negative, blur');
|
||||
await context(page).selectOption({ label: OPTIONS[1] });
|
||||
await expect(panel(page).locator('.prompt-section.positive .prompt-text')).toHaveText('Harbor lights at night');
|
||||
await context(page).selectOption({ label: 'Leave unresolved' });
|
||||
await expect(panel(page).locator('.prompt-section.positive .prompt-text')).toHaveText('Multiple possible prompts. Source could not be determined.');
|
||||
// Info selects another item.
|
||||
await cards(page).nth(1).getByRole('button', { name: 'Inspect review_0001.png', exact: true }).click();
|
||||
await expect(panel(page).locator('p.small')).toHaveText('review_0001.png');
|
||||
await expect(cards(page).nth(1)).toHaveClass(/selected/);
|
||||
await expect(context(page)).toBeEnabled();
|
||||
// The card's image opens the shared viewer.
|
||||
await cards(page).nth(1).getByRole('button', { name: 'Open review_0001.png', exact: true }).click();
|
||||
await expect(page.locator('#viewer-dialog')).toBeVisible();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(page.locator('#viewer-dialog')).toHaveCount(0);
|
||||
// Open activity.
|
||||
await main(page).getByRole('button', { name: 'Open activity', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity$/);
|
||||
});
|
||||
|
||||
test('apply candidate links the pair and the item leaves the queue and the count drops', async ({ page, request }) => {
|
||||
await seed(request, 2);
|
||||
await open(page);
|
||||
await expect(reviewRow(page)).toHaveText('Needs review2');
|
||||
const decided = page.waitForResponse(item => /\/review\/items\/[^/]+\/decision$/.test(item.url()));
|
||||
await panel(page).getByRole('button', { name: 'Apply association', exact: true }).click();
|
||||
expect((await decided).status()).toBe(200);
|
||||
await expect(toast(page)).toHaveText('Association accepted.');
|
||||
await expect(page).toHaveURL(/#\/images$/);
|
||||
await expect(reviewRow(page)).toHaveText('Needs review1');
|
||||
expect(await pending(request)).toBe(1);
|
||||
// The first sampler path's pair is linked to that file, and only to it.
|
||||
expect(await links(request)).toEqual([{ filename: 'review_0000.png', prompt: 'Golden dunes at dusk',
|
||||
pair_id: expect.any(Number), positive: 'Golden dunes at dusk', negative: 'Soft negative, blur' }]);
|
||||
});
|
||||
|
||||
test('conflict leaves links unchanged and explains', async ({ page, request }) => {
|
||||
// Queued first, then linked to another prompt (before a scan or monitor pass could settle it).
|
||||
await seed(request, 0, { conflict: true });
|
||||
const before = await links(request);
|
||||
expect(before).toEqual([{ filename: 'conflict.png', prompt: 'Misty forest path', pair_id: null, positive: null,
|
||||
negative: null }]);
|
||||
await open(page);
|
||||
await expect(panel(page).locator('p.small')).toHaveText('conflict.png');
|
||||
await expect(context(page).locator('option')).toHaveText(OPTIONS);
|
||||
const decided = page.waitForResponse(item => /\/decision$/.test(item.url()));
|
||||
await apply(page).click();
|
||||
const answer = await decided;
|
||||
expect(answer.status()).toBe(409);
|
||||
expect((await answer.json()).code).toBe('association-conflict');
|
||||
await expect(warning(page)).toHaveText('This image is already linked to another prompt or negative, so this sampler path cannot be attached safely. The existing links were preserved.');
|
||||
await expect(page).toHaveURL(/#\/review$/);
|
||||
await expect(context(page)).toBeEnabled();
|
||||
await expect(context(page)).toHaveValue(/^[0-9a-f]{64}$/);
|
||||
expect(await links(request)).toEqual(before);
|
||||
expect(await pending(request)).toBe(1);
|
||||
await expect(reviewRow(page)).toHaveText('Needs review1');
|
||||
// The file can still be left unresolved, which links nothing either.
|
||||
await context(page).selectOption({ label: 'Leave unresolved' });
|
||||
await apply(page).click();
|
||||
await expect(toast(page)).toHaveText('Left unresolved.');
|
||||
expect(await links(request)).toEqual(before);
|
||||
expect(await pending(request)).toBe(0);
|
||||
});
|
||||
|
||||
test('a limited item and an unavailable file can only be left unresolved', async ({ page, request }) => {
|
||||
await seed(request, 0, { limited: true, unavailable: true });
|
||||
await open(page);
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
await expect(panel(page).locator('p.small')).toHaveText('limited.png');
|
||||
await expect(context(page).locator('option')).toHaveText(['Leave unresolved']);
|
||||
await expect(warning(page)).toHaveText('This file has too many sampler paths to list. You can leave it unresolved.');
|
||||
// The unavailable file: no current root lists it, so its media and Tag are inert.
|
||||
const unavailable = cards(page).nth(1);
|
||||
await expect(unavailable.locator('.image-open')).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(unavailable.locator('.image-card-actions button').last()).toHaveAttribute('aria-disabled', 'true');
|
||||
// F3: its empty media button keeps the concept's 1.33 media area, so the card lines up with its neighbours.
|
||||
await expect(cards(page).first().locator('.image-open img')).toBeVisible();
|
||||
const sizes = await cards(page).evaluateAll(nodes => nodes.map(node => {
|
||||
const art = node.querySelector('.image-open').getBoundingClientRect();
|
||||
return [Math.round(art.width), Math.round(art.height), Math.round(node.getBoundingClientRect().height)];
|
||||
}));
|
||||
expect(sizes[1]).toEqual(sizes[0]);
|
||||
await unavailable.getByRole('button', { name: 'Inspect unavailable.png', exact: true }).click();
|
||||
await expect(panel(page).locator('p.small')).toHaveText('unavailable.png');
|
||||
await expect(warning(page)).toHaveText('This file is not under a current image source, so it cannot be opened. You can leave it unresolved.');
|
||||
await expect(panel(page).getByRole('button', { name: 'Tag image', exact: true })).toHaveAttribute('aria-disabled', 'true');
|
||||
const observes = [];
|
||||
page.on('request', item => { if (item.url().endsWith('/image-annotations/observe')) observes.push(item.url()); });
|
||||
await apply(page).click();
|
||||
await expect(toast(page)).toHaveText('Left unresolved.');
|
||||
expect(observes).toEqual([]);
|
||||
expect(await pending(request)).toBe(1);
|
||||
});
|
||||
|
||||
test('leave unresolved removes the item from pending', async ({ page, request }) => {
|
||||
await seed(request, 2);
|
||||
await open(page);
|
||||
await context(page).selectOption({ label: 'Leave unresolved' });
|
||||
await panel(page).getByRole('button', { name: 'Apply association', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Left unresolved.');
|
||||
await expect(page).toHaveURL(/#\/images$/);
|
||||
expect(await pending(request)).toBe(1);
|
||||
});
|
||||
|
||||
test('leave for later writes nothing and goes to Images', async ({ page, request }) => {
|
||||
await seed(request, 2);
|
||||
await open(page);
|
||||
const writes = [];
|
||||
page.on('request', item => { if (item.method() !== 'GET') writes.push(item.url()); });
|
||||
await panel(page).getByRole('button', { name: 'Leave for later', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/images$/);
|
||||
expect(writes).toEqual([]);
|
||||
expect(await pending(request)).toBe(2);
|
||||
});
|
||||
|
||||
test('tag image from review saves tags and the item remains pending', async ({ page, request }) => {
|
||||
await seed(request, 1);
|
||||
await open(page);
|
||||
const before = await links(request);
|
||||
await panel(page).getByRole('button', { name: 'Tag image', exact: true }).click();
|
||||
const dialog = page.getByRole('dialog', { name: 'Tag image', exact: true });
|
||||
await expect(dialog).toBeVisible();
|
||||
await expect(dialog.locator('p.tag-scope-note')).toHaveText('Tags belong to this image. Resolve its prompt association separately in Needs review.');
|
||||
await dialog.getByRole('textbox', { name: 'Image tags · edit manually', exact: true }).fill('dunes');
|
||||
await dialog.getByRole('button', { name: 'Save manual tags', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('1 tags saved for review_0000.png. Prompt association still awaits review.');
|
||||
await expect(dialog).toHaveCount(0);
|
||||
expect(await pending(request)).toBe(1);
|
||||
await expect(page).toHaveURL(/#\/review$/);
|
||||
// The item is untouched: still listed, still open, nothing linked.
|
||||
await expect(cards(page)).toHaveCount(1);
|
||||
await expect(context(page)).toBeEnabled();
|
||||
expect(await links(request)).toEqual(before);
|
||||
const [item] = (await (await request.get('/prompt_manager/review/items')).json()).items;
|
||||
expect([item.state, item.revision]).toEqual(['pending', 0]);
|
||||
// The card's Tag opens the same dialog.
|
||||
await cards(page).first().getByRole('button', { name: 'Tag or auto-tag review_0000.png', exact: true }).click();
|
||||
await expect(dialog).toBeVisible();
|
||||
await expect(dialog.locator('p.tag-scope-note')).toBeVisible();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(dialog).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('an empty queue shows the empty notice only', async ({ page, request }) => {
|
||||
await seed(request, 0);
|
||||
await page.goto(review);
|
||||
await expect(main(page).locator('.notice h2')).toHaveText('No images need a decision');
|
||||
await expect(main(page).locator('.notice p')).toHaveText(EMPTY);
|
||||
await expect(main(page).locator('.gallery-layout')).toHaveCount(0);
|
||||
await expect(reviewRow(page)).toHaveText('Needs review0');
|
||||
});
|
||||
|
||||
test('load more appends the next page', async ({ page, request }) => {
|
||||
await seed(request, 45);
|
||||
await open(page);
|
||||
await expect(cards(page)).toHaveCount(40);
|
||||
const continuation = main(page).locator('.infinite-continuation');
|
||||
await expect(continuation.locator('p')).toHaveText('40 of 45 images');
|
||||
await continuation.getByRole('button', { name: 'Load more images', exact: true }).click();
|
||||
await expect(cards(page)).toHaveCount(45);
|
||||
await expect(continuation).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('a scan job with review items offers Review metadata in Activity', async ({ page, request }) => {
|
||||
await seed(request, 1);
|
||||
const started = await (await request.post('/__test__/jobs/start', { data: { total: 10, review_items: 2 } })).json();
|
||||
await expect(await request.post(`/__test__/jobs/${started.job.id}/finish`, { data: {} })).toBeOK();
|
||||
const plain = await (await request.post('/__test__/jobs/start', { data: { total: 10 } })).json();
|
||||
await expect(await request.post(`/__test__/jobs/${plain.job.id}/finish`, { data: {} })).toBeOK();
|
||||
await page.goto('/prompt_manager/workspace#/activity');
|
||||
const item = main(page).locator(`.activity-item[data-job-id="${started.job.id}"]`);
|
||||
await expect(item.getByRole('button', { name: 'Review metadata', exact: true })).toBeVisible();
|
||||
await expect(main(page).locator(`.activity-item[data-job-id="${plain.job.id}"] button`)).toHaveCount(0);
|
||||
await page.goto(`/prompt_manager/workspace#/activity/${started.job.id}`);
|
||||
const actions = main(page).locator('.heading-actions button');
|
||||
await expect(actions).toHaveText(['Download report', 'Review metadata']);
|
||||
await actions.nth(1).click();
|
||||
await expect(page).toHaveURL(/#\/review$/);
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Metadata review');
|
||||
});
|
||||
|
||||
test('a failed item read says so and Retry reads it again', async ({ page, request }) => {
|
||||
await seed(request, 1);
|
||||
await page.route(url => /\/prompt_manager\/review\/items\/[0-9a-f-]{36}$/.test(url.pathname),
|
||||
route => route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ success: false, code: 'review-failed', error: 'x' }) }),
|
||||
{ times: 1 });
|
||||
await page.goto(review);
|
||||
await expect(panel(page).locator('.warning-note')).toHaveText('This item could not be read. Select it again to retry.');
|
||||
await expect(panel(page).getByText('Could not read prompts from this file.', { exact: true })).toBeVisible();
|
||||
await expect(panel(page).getByRole('button', { name: 'Apply association', exact: true })).toHaveAttribute('aria-disabled', 'true');
|
||||
await panel(page).getByRole('button', { name: 'Retry prompt details', exact: true }).click();
|
||||
await expect(context(page)).toBeEnabled();
|
||||
await expect(context(page).locator('option')).toHaveText(OPTIONS);
|
||||
await expect(panel(page).locator('.warning-note')).toBeHidden();
|
||||
});
|
||||
|
||||
test('a failed queue read offers Retry', async ({ page, request }) => {
|
||||
await seed(request, 1);
|
||||
await page.route(url => url.pathname.endsWith('/prompt_manager/review/items') && url.searchParams.get('limit') === '40',
|
||||
route => route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ success: false, code: 'review-failed', error: 'x' }) }),
|
||||
{ times: 1 });
|
||||
await page.goto(review);
|
||||
await expect(main(page).getByRole('alert')).toHaveText('The review queue could not be read.');
|
||||
await main(page).getByRole('button', { name: 'Retry', exact: true }).click();
|
||||
await expect(context(page)).toBeEnabled();
|
||||
await expect(main(page).locator('.notice h2')).toHaveText('1 image needs a decision');
|
||||
});
|
||||
|
||||
test('keyboard focus labels and touch targets', async ({ page, request }) => {
|
||||
await seed(request, 2);
|
||||
await open(page);
|
||||
const first = cards(page).first();
|
||||
for (const [name, role] of [['Open review_0000.png', 'button'], ['Inspect review_0000.png', 'button'],
|
||||
['Tag or auto-tag review_0000.png', 'button']]) await expect(first.getByRole(role, { name, exact: true })).toBeVisible();
|
||||
await expect(main(page).getByRole('list', { name: 'Review queue', exact: true })).toBeVisible();
|
||||
await expect(panel(page)).toBeVisible();
|
||||
// Keyboard: Info on the second card opens it, and the panel's controls take focus in their order.
|
||||
const info = cards(page).nth(1).getByRole('button', { name: 'Inspect review_0001.png', exact: true });
|
||||
await info.focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(panel(page).locator('p.small')).toHaveText('review_0001.png');
|
||||
await expect(context(page)).toBeEnabled();
|
||||
await context(page).focus();
|
||||
await expect(context(page)).toBeFocused();
|
||||
const order = await panel(page).evaluate(node => [...node.querySelectorAll('button, select')]
|
||||
.filter(control => control.offsetParent !== null).map(control => control.getAttribute('aria-label') || control.textContent.trim()));
|
||||
expect(order[0]).toBe('Tag image');
|
||||
expect(order.slice(-2)).toEqual(['Leave for later', 'Apply association']);
|
||||
// Targets: the panel's controls are 40px on a coarse pointer (the concept's coarse buttons) and 24px otherwise (the
|
||||
// WCAG 2.2 minimum); the card's buttons keep the concept's compact size, never under 24px (its ≤700px rule).
|
||||
const coarse = await page.evaluate(() => matchMedia('(pointer: coarse)').matches);
|
||||
const sized = [
|
||||
[panel(page).getByRole('button', { name: 'Tag image', exact: true }), coarse ? 40 : 24],
|
||||
[context(page), coarse ? 40 : 24],
|
||||
[panel(page).getByRole('button', { name: 'Leave for later', exact: true }), coarse ? 40 : 24],
|
||||
[apply(page), coarse ? 40 : 24],
|
||||
[first.getByRole('button', { name: 'Inspect review_0000.png', exact: true }), 24],
|
||||
[first.getByRole('button', { name: 'Tag or auto-tag review_0000.png', exact: true }), 24],
|
||||
];
|
||||
for (const [control, minimum] of sized) {
|
||||
const box = await control.boundingBox();
|
||||
const label = await control.evaluate(node => node.getAttribute('aria-label') || node.textContent.trim().slice(0, 40));
|
||||
expect(box.height, label).toBeGreaterThanOrEqual(minimum);
|
||||
expect(box.width, label).toBeGreaterThanOrEqual(24);
|
||||
}
|
||||
// Enter on Leave for later returns to Images.
|
||||
await panel(page).getByRole('button', { name: 'Leave for later', exact: true }).focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(page).toHaveURL(/#\/images$/);
|
||||
});
|
||||
|
||||
test('an open page shows items a scan adds and keeps the open choice', async ({ page, request }) => {
|
||||
await seed(request, 1);
|
||||
await open(page);
|
||||
await expect.poll(async () => (await streams(request)).streams).toBe(1);
|
||||
await context(page).selectOption({ label: OPTIONS[1] });
|
||||
await expect(reviewRow(page)).toHaveText('Needs review1');
|
||||
await expect(await request.post('/__test__/review/add', { data: { kind: 'scan' } })).toBeOK();
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
await expect(main(page).locator('.notice h2')).toHaveText('2 images need a decision');
|
||||
await expect(reviewRow(page)).toHaveText('Needs review2');
|
||||
await expect(cards(page).first()).toHaveClass(/selected/);
|
||||
await expect(panel(page).locator('p.small')).toHaveText('review_0000.png');
|
||||
await expect(context(page).locator('option:checked')).toHaveText(OPTIONS[1]);
|
||||
await expect(warning(page)).toBeHidden();
|
||||
});
|
||||
|
||||
test('an open page shows items a metadata scan adds', async ({ page, request }) => {
|
||||
await seed(request, 1);
|
||||
await expect(await request.post('/__test__/review/add', { data: { kind: 'file' } })).toBeOK();
|
||||
await open(page);
|
||||
await expect.poll(async () => (await streams(request)).streams).toBe(1);
|
||||
await expect(cards(page)).toHaveCount(1);
|
||||
const started = await request.post('/prompt_manager/jobs/scan', { data: { scope: 'new' } });
|
||||
expect(started.status()).toBe(202);
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
await expect(cards(page).nth(1).getByRole('button', { name: 'Inspect added_0000.png', exact: true })).toBeVisible();
|
||||
await expect(reviewRow(page)).toHaveText('Needs review2');
|
||||
});
|
||||
|
||||
test('a decision in another tab leaves the open page and opens the next image', async ({ page, request, context: browserContext }) => {
|
||||
await seed(request, 2);
|
||||
await open(page);
|
||||
await expect.poll(async () => (await streams(request)).streams).toBe(1);
|
||||
const other = await browserContext.newPage();
|
||||
await open(other);
|
||||
await context(other).selectOption({ label: 'Leave unresolved' });
|
||||
await apply(other).click();
|
||||
await expect(other).toHaveURL(/#\/images$/);
|
||||
await other.close();
|
||||
await expect(cards(page)).toHaveCount(1);
|
||||
await expect(panel(page).locator('p.small')).toHaveText('review_0001.png');
|
||||
await expect(warning(page)).toHaveText('review_0000.png was decided elsewhere, so the next image is open.');
|
||||
await expect(main(page).locator('.notice h2')).toHaveText('1 image needs a decision');
|
||||
await expect(reviewRow(page)).toHaveText('Needs review1');
|
||||
await expect(context(page)).toBeEnabled();
|
||||
});
|
||||
|
||||
/** The queue reads (`?limit=40`, cursors) and count reads (`?limit=1`) the page sends from now on. */
|
||||
function reviewReads(page) {
|
||||
const reads = [];
|
||||
page.on('request', item => {
|
||||
const url = new URL(item.url());
|
||||
if (url.pathname === '/prompt_manager/review/items') reads.push(url.searchParams.get('cursor') ? 'more' : url.search);
|
||||
});
|
||||
return reads;
|
||||
}
|
||||
|
||||
test('a visit reads its queue once, and a change reads it once more', async ({ page, request }) => {
|
||||
// F3 ruling: the stream's ready must not read a queue the first read already holds.
|
||||
await seed(request, 1);
|
||||
const reads = reviewReads(page);
|
||||
await open(page);
|
||||
await expect.poll(async () => (await streams(request)).streams).toBe(1);
|
||||
await expect(await request.post('/__test__/review/add', { data: { kind: 'scan' } })).toBeOK();
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
await expect(reviewRow(page)).toHaveText('Needs review2');
|
||||
// The page's first read and the change's re-read; the shell's count read at start, and the row follows the page.
|
||||
expect(reads.filter(search => search === '?limit=40')).toHaveLength(2);
|
||||
expect(reads.filter(search => search === '?limit=1')).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('a ready whose version cannot be read re-reads the queue once', async ({ page, request }) => {
|
||||
// The stream's ready names no readable version (an older server, a proxy that rewrote it): the page reads again.
|
||||
await seed(request, 1);
|
||||
const reads = reviewReads(page);
|
||||
// M1: `retry` keeps the browser from reconnecting the ended stream during the test, so exactly one re-read follows.
|
||||
await page.route(url => url.pathname === '/prompt_manager/review/events', route => route.fulfill({
|
||||
status: 200, headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-store' },
|
||||
body: 'retry: 3600000\nevent: ready\ndata: not json\n\n' }));
|
||||
let reread = 0;
|
||||
const second = page.waitForResponse(item => item.url().endsWith('/prompt_manager/review/items?limit=40') && ++reread === 2);
|
||||
await open(page);
|
||||
await second;
|
||||
await expect(cards(page)).toHaveCount(1);
|
||||
expect(reads.filter(search => search === '?limit=40')).toHaveLength(2);
|
||||
});
|
||||
|
||||
const live = page => main(page).locator('.heading .review-live');
|
||||
const NOT_LIVE = 'Live updates are paused: changes made elsewhere appear once the connection returns.';
|
||||
|
||||
test('a stream the browser gives up on is opened again, and the page says it is not live meanwhile', async ({ page, request }) => {
|
||||
// I3: a 503 (shutdown, or a proxy while the backend restarts) fails an EventSource for good.
|
||||
await seed(request, 1);
|
||||
let attempts = 0;
|
||||
await page.route(url => url.pathname === '/prompt_manager/review/events', route => {
|
||||
attempts += 1;
|
||||
if (attempts === 1) return route.fulfill({ status: 503, contentType: 'application/json', body: '{"success":false}' });
|
||||
return route.fallback();
|
||||
});
|
||||
await open(page);
|
||||
await expect(live(page)).toHaveText(NOT_LIVE);
|
||||
await expect(live(page)).toBeVisible();
|
||||
// The page opens the stream again (after 3 s) and is live once it sends ready.
|
||||
await expect.poll(async () => (await streams(request)).streams, { timeout: 15000 }).toBe(1);
|
||||
await expect(live(page)).toBeHidden();
|
||||
await expect(await request.post('/__test__/review/add', { data: { kind: 'scan' } })).toBeOK();
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
expect(attempts).toBe(2);
|
||||
});
|
||||
|
||||
/** Make the page hidden or visible as a tab switch does. */
|
||||
async function setVisible(page, visible) {
|
||||
await page.evaluate(shown => {
|
||||
Object.defineProperty(document, 'hidden', { configurable: true, get: () => !shown });
|
||||
Object.defineProperty(document, 'visibilityState', { configurable: true, get: () => (shown ? 'visible' : 'hidden') });
|
||||
document.dispatchEvent(new Event('visibilitychange'));
|
||||
}, visible);
|
||||
}
|
||||
|
||||
test('a hidden tab closes its stream and a visible one reads only what changed meanwhile', async ({ page, request }) => {
|
||||
// I4: background tabs must not hold one of the origin's six connections.
|
||||
await seed(request, 1);
|
||||
const reads = reviewReads(page);
|
||||
await open(page);
|
||||
await expect.poll(async () => (await streams(request)).streams).toBe(1);
|
||||
await setVisible(page, false);
|
||||
await expect(await request.post('/__test__/review/add', { data: { kind: 'scan' } })).toBeOK();
|
||||
await closedStreamsDropped(request);
|
||||
await expect(cards(page)).toHaveCount(1);
|
||||
expect(reads.filter(search => search === '?limit=40')).toHaveLength(1);
|
||||
// Visible again: the reopened stream's ready names a newer version, so the page reads once.
|
||||
const reread = page.waitForResponse(item => item.url().endsWith('/prompt_manager/review/items?limit=40'));
|
||||
await setVisible(page, true);
|
||||
await expect.poll(async () => (await streams(request)).streams).toBe(1);
|
||||
await reread;
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
await expect(live(page)).toBeHidden();
|
||||
expect(reads.filter(search => search === '?limit=40')).toHaveLength(2);
|
||||
});
|
||||
|
||||
test('a stream whose close reaches the server after a change is dropped at a later write', async ({ page, request }) => {
|
||||
// The CI ordering made certain: the hidden page's close arrives only after the scan addition's change was
|
||||
// written to its stream, so that write succeeds and a later one finds the stream closed.
|
||||
await seed(request, 1);
|
||||
await open(page);
|
||||
await expect.poll(async () => (await streams(request)).streams).toBe(1);
|
||||
await page.evaluate(() => {
|
||||
const close = EventSource.prototype.close;
|
||||
window.lateChanges = 0;
|
||||
window.heldCloses = [];
|
||||
EventSource.prototype.close = function () {
|
||||
this.addEventListener('change', () => { window.lateChanges += 1; });
|
||||
window.heldCloses.push(() => close.call(this));
|
||||
};
|
||||
});
|
||||
await setVisible(page, false);
|
||||
await expect(await request.post('/__test__/review/add', { data: { kind: 'scan' } })).toBeOK();
|
||||
await expect.poll(() => page.evaluate(() => window.lateChanges)).toBe(1);
|
||||
await page.evaluate(() => window.heldCloses.splice(0).forEach(close => close()));
|
||||
expect((await streams(request)).streams).toBe(1);
|
||||
await closedStreamsDropped(request);
|
||||
await expect(cards(page)).toHaveCount(1);
|
||||
});
|
||||
|
||||
test('a re-read refreshes a card whose media or target changed', async ({ page, request }) => {
|
||||
// M6 and M10: a root came or went between reads; the card follows, and a known thumbnail is its media.
|
||||
await seed(request, 2);
|
||||
let rewrite = false;
|
||||
await page.route(url => url.pathname === '/prompt_manager/review/items' && url.search === '?limit=40', async route => {
|
||||
if (!rewrite) return route.fallback();
|
||||
const response = await route.fetch();
|
||||
const body = await response.json();
|
||||
body.items[0] = { ...body.items[0], thumbnail_url: `${body.items[0].url}&thumb=1` };
|
||||
body.items[1] = { ...body.items[1], url: null, thumbnail_url: null, target: null, unavailable: true };
|
||||
return route.fulfill({ response, json: body });
|
||||
});
|
||||
await open(page);
|
||||
await expect.poll(async () => (await streams(request)).streams).toBe(1);
|
||||
const second = cards(page).nth(1);
|
||||
await expect(second.getByRole('button', { name: 'Open review_0001.png', exact: true })).not.toHaveAttribute('aria-disabled', 'true');
|
||||
rewrite = true;
|
||||
await expect(await request.post('/__test__/review/add', { data: { kind: 'scan' } })).toBeOK();
|
||||
await expect(cards(page)).toHaveCount(3);
|
||||
await expect(cards(page).first().locator('img')).toHaveAttribute('src', /&thumb=1$/);
|
||||
await expect(second.getByRole('button', { name: 'Open review_0001.png', exact: true })).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(second.locator('img')).toHaveCount(0);
|
||||
await expect(second.locator('.image-card-actions button').last()).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(cards(page).first()).toHaveClass(/selected/);
|
||||
});
|
||||
|
||||
test('leaving during the observe releases the capture it answers', async ({ page, request }) => {
|
||||
// M4: the server made the capture; the page left before the answer came.
|
||||
await seed(request, 1);
|
||||
await open(page);
|
||||
let held;
|
||||
const sent = new Promise(resolve => { held = resolve; });
|
||||
await page.route(url => url.pathname === '/prompt_manager/image-annotations/observe', async route => {
|
||||
const response = await route.fetch();
|
||||
held(() => route.fulfill({ response }));
|
||||
});
|
||||
await apply(page).click();
|
||||
const answer = await sent;
|
||||
expect((await streams(request)).captures).toBe(1);
|
||||
await page.evaluate(() => { window.location.hash = '#/tags'; });
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Tags & categories');
|
||||
await answer();
|
||||
await expect.poll(async () => (await streams(request)).captures).toBe(0);
|
||||
});
|
||||
|
||||
test('a live re-read keeps the loaded depth and the scroll position', async ({ page, request }) => {
|
||||
// F3 ruling: a queue change re-reads as many items as Load more had shown, and the page does not jump.
|
||||
await seed(request, 45);
|
||||
await open(page);
|
||||
await expect.poll(async () => (await streams(request)).streams).toBe(1);
|
||||
await main(page).getByRole('button', { name: 'Load more images', exact: true }).click();
|
||||
await expect(cards(page)).toHaveCount(45);
|
||||
await cards(page).last().scrollIntoViewIfNeeded();
|
||||
const scrollTop = () => cards(page).first().evaluate(node => {
|
||||
for (let el = node.parentElement; el; el = el.parentElement) {
|
||||
const style = getComputedStyle(el);
|
||||
if (/(auto|scroll)/.test(style.overflowY) && el.scrollHeight > el.clientHeight) return el.scrollTop;
|
||||
}
|
||||
return document.scrollingElement.scrollTop;
|
||||
});
|
||||
const before = await scrollTop();
|
||||
expect(before).toBeGreaterThan(0);
|
||||
const reads = reviewReads(page);
|
||||
await expect(await request.post('/__test__/review/add', { data: { kind: 'scan' } })).toBeOK();
|
||||
await expect(main(page).locator('.notice h2')).toHaveText('46 images need a decision');
|
||||
await expect(cards(page)).toHaveCount(46);
|
||||
expect(await scrollTop()).toBe(before);
|
||||
expect(reads).toEqual(['?limit=40', 'more']);
|
||||
await expect(main(page).locator('.infinite-continuation')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('teardown releases captures listeners and requests', async ({ page, request }) => {
|
||||
await seed(request, 1);
|
||||
await open(page);
|
||||
await expect.poll(async () => (await streams(request)).streams).toBe(1);
|
||||
const inFlight = new Set();
|
||||
page.on('request', item => { if (item.url().includes('/prompt_manager/review/')) inFlight.add(item); });
|
||||
page.on('requestfinished', item => inFlight.delete(item));
|
||||
page.on('requestfailed', item => inFlight.delete(item));
|
||||
// A decision whose answer never comes: leaving the page must still release its capture.
|
||||
let held;
|
||||
const sent = new Promise(resolve => { held = resolve; });
|
||||
await page.route(url => /\/decision$/.test(url.pathname), route => held(route));
|
||||
await apply(page).click();
|
||||
const route = await sent;
|
||||
expect((await streams(request)).captures).toBe(1);
|
||||
await page.evaluate(() => { window.location.hash = '#/tags'; });
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Tags & categories');
|
||||
await expect.poll(async () => (await streams(request)).captures).toBe(0);
|
||||
await route.abort().catch(() => {});
|
||||
await closedStreamsDropped(request);
|
||||
await expect.poll(() => inFlight.size).toBe(0);
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from '../../tools/vendor/node_modules/@playwright/test/index.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
// Saved views (#249 D16, maintainer ruling 2026-10-21): the toolbar's Save view (concept app.js a359, a1330–1346),
|
||||
// the sidebar SAVED VIEWS rows (a331–335) and `#/saved/<id>` (a1158–1162), on Images and the Prompt library.
|
||||
@@ -372,7 +372,7 @@ test('Favorites is the first SAVED VIEWS row, with a real count that follows eve
|
||||
expect(await favoritesRow(page).evaluate(link => [...link.childNodes].map(node => node.nodeType === Node.TEXT_NODE
|
||||
? `text:${node.textContent}` : `${node.localName}.${node.getAttribute('class')}`)))
|
||||
.toEqual(['svg.icon', 'text:Favorites', 'span.nav-count']);
|
||||
expect(await savedNav(page).locator('a').evaluateAll(links => links.map(link => link.dataset.route))).toEqual(['favorites']);
|
||||
expect(await savedNav(page).locator('a').evaluateAll(links => links.map(link => link.dataset.route))).toEqual(['favorites', 'review']);
|
||||
|
||||
await star(page, '1:shared/same.png');
|
||||
await expect(toast(page)).toHaveText('Added to favorites.');
|
||||
|
||||
@@ -0,0 +1,535 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
|
||||
// System › Sources & scans (#249 System S4 Task 11) on the concept-shaped real fixture, whose two sources are this
|
||||
// server's `output` and `input` folders: the source list, monitoring and worker rows read GET /settings (S4 Task 9),
|
||||
// Add/Remove edit the host draft behind the one Save row, and every scan starts the session job through the shared
|
||||
// "Review metadata scan" (D55), scoped to one source from its row (S4 Task 10). The hooks are workspace_system_fixture.py's.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const SETTINGS_URL = '/prompt_manager/settings';
|
||||
const NOTICE_KEY = 'pm_update_notice_dismissed';
|
||||
const OUTSIDE_REASON = 'Gallery root must be inside a ComfyUI directory (output, input, temp, user) or one listed in PROMPT_MANAGER_EXTRA_GALLERY_ROOTS';
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
page.systemErrors = [];
|
||||
page.on('pageerror', error => page.systemErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/release');
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.systemErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
const layoutOf = page => page.locator('#workspace-main .settings-layout');
|
||||
const sourceList = page => layoutOf(page).locator('#source-list');
|
||||
const statusOf = page => layoutOf(page).locator('#settings-status');
|
||||
const saveButton = page => layoutOf(page).getByRole('button', { name: 'Save preferences', exact: true });
|
||||
const dialog = page => page.locator('dialog#modal[open]');
|
||||
|
||||
async function serverJson(request, url) {
|
||||
const response = await request.get(url);
|
||||
await expect(response).toBeOK();
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/** Open Sources with the update notice dismissed (or shown, `notice: true`) once the source list has loaded. */
|
||||
async function openSources(page, { notice = false } = {}) {
|
||||
if (!notice) await page.addInitScript(key => { if (location.protocol.startsWith('http')) localStorage.setItem(key, '3.2.0'); }, NOTICE_KEY);
|
||||
await page.goto(`${workspace}#/system/sources`);
|
||||
await expect(sourceList(page).locator('.operation').first()).toBeVisible();
|
||||
await expect(layoutOf(page).locator('#workers')).toBeEnabled();
|
||||
return layoutOf(page);
|
||||
}
|
||||
|
||||
/** The source list's rows: `[h3, p, [buttons]]`, then its note line when there is one. */
|
||||
function rowsOf(page) {
|
||||
return sourceList(page).evaluate(list => [...list.children].map(row => row.matches('.operation')
|
||||
? [row.querySelector('h3').textContent, row.querySelector('p').textContent,
|
||||
[...row.querySelectorAll('.row-actions > button')].map(button => button.textContent)]
|
||||
: [row.localName, row.className, row.textContent]));
|
||||
}
|
||||
|
||||
const sourceRow = (page, title) => sourceList(page).locator('.operation').filter({ has: page.getByRole('heading', { name: title, exact: true }) });
|
||||
|
||||
test('notice Later persists the 3.2.0 dismissal (pm_update_notice_dismissed) without scanning', async ({ page, request }) => {
|
||||
const writes = [];
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) writes.push(item.url()); });
|
||||
const layout = await openSources(page, { notice: true });
|
||||
const notice = layout.locator(':scope > .notice');
|
||||
await expect(notice.locator('h2')).toHaveText('Bring older images into your prompt history');
|
||||
await expect(notice.locator('p')).toHaveText('The schema upgrades automatically. An optional metadata scan links existing positives and negatives without regenerating all thumbnails.');
|
||||
await expect(notice.locator('.row-actions > button')).toHaveText(['Later', 'Review scan']);
|
||||
await notice.getByRole('button', { name: 'Later', exact: true }).click();
|
||||
await expect(layout.locator(':scope > .notice')).toHaveCount(0);
|
||||
expect(await page.evaluate(key => localStorage.getItem(key), NOTICE_KEY)).toBe('3.2.0');
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
// The dismissal is the legacy admin page's own, so neither page shows the notice again.
|
||||
await page.reload();
|
||||
await expect(sourceList(page).locator('.operation').first()).toBeVisible();
|
||||
await expect(layout.locator(':scope > .notice')).toHaveCount(0);
|
||||
expect(writes).toEqual([]);
|
||||
expect((await serverJson(request, '/prompt_manager/jobs')).jobs).toEqual([]);
|
||||
});
|
||||
|
||||
test('Review scan opens the shared Review metadata scan with all sources', async ({ page, request }) => {
|
||||
const layout = await openSources(page, { notice: true });
|
||||
// The notice's Review scan dismisses it, as the legacy page's does, and reviews every source.
|
||||
await layout.locator(':scope > .notice').getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review metadata scan');
|
||||
await expect(dialog(page).locator('dl.meta-grid > div > dd')).toHaveText(['2 output folders', 'New and unresolved files']);
|
||||
expect(await page.evaluate(key => localStorage.getItem(key), NOTICE_KEY)).toBe('3.2.0');
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(layout.locator(':scope > .notice')).toHaveCount(0);
|
||||
// Enrich existing metadata is the same review.
|
||||
const enrich = layout.locator('.operation').filter({ hasText: 'Enrich existing metadata' });
|
||||
await expect(enrich.locator('p')).toHaveText('Resolve both prompts, deduplicate recipes, preserve uncertain links.');
|
||||
await enrich.getByRole('button', { name: 'Review scan', exact: true }).click();
|
||||
await expect(dialog(page).locator('dl.meta-grid > div > dd')).toHaveText(['2 output folders', 'New and unresolved files']);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
expect((await serverJson(request, '/prompt_manager/jobs')).jobs).toEqual([]);
|
||||
});
|
||||
|
||||
test('Scan changes opens it with that one source ("1 output folder") and starts a scoped job that opens in Activity', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/monitor', { data: { watching: ['input'] } })).toBeOK();
|
||||
const bodies = [];
|
||||
page.on('request', item => { if (item.method() === 'POST' && item.url().endsWith('/prompt_manager/jobs/scan')) bodies.push(item.postDataJSON()); });
|
||||
await openSources(page);
|
||||
const input = sourceRow(page, 'ComfyUI input');
|
||||
await expect(input.locator('p')).toHaveText('input · watching');
|
||||
await input.getByRole('button', { name: 'Scan changes', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review metadata scan');
|
||||
await expect(dialog(page).locator('dl.meta-grid > div > dd')).toHaveText(['1 output folder', 'New and unresolved files']);
|
||||
await dialog(page).getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
// The toast says it started (or, for a scan this quick, that it already finished); Activity shows its progress.
|
||||
await expect(toast(page)).toHaveText(/^Scan (started\.|finished · )/);
|
||||
expect(bodies).toEqual([{ scope: 'new', roots: [1], paths: ['input'] }]);
|
||||
const id = page.url().split('/').pop();
|
||||
// The job walks only that source (the count is recorded once the walk has listed it).
|
||||
await expect.poll(async () => (await serverJson(request, `/prompt_manager/jobs/${id}`)).job.scope)
|
||||
.toMatchObject({ files: 'new', roots: [1], output_folders: 1 });
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Scan for metadata');
|
||||
});
|
||||
|
||||
test('Add source validates on the server, then marks the host draft dirty; Save persists', async ({ page, request }) => {
|
||||
// A folder inside ComfyUI's output folder that is not a source yet.
|
||||
await expect(await request.post('/__test__/system/unthumbnailed', { data: { count: 1 } })).toBeOK();
|
||||
const layout = await openSources(page);
|
||||
expect(await rowsOf(page)).toEqual([
|
||||
['ComfyUI output', 'output · manual scan', ['Scan folder', 'Remove']],
|
||||
['ComfyUI input', 'input · manual scan', ['Scan folder', 'Remove']],
|
||||
]);
|
||||
await expect(layout.locator('.panel-heading > h2')).toHaveText('Image sources');
|
||||
const add = layout.locator('.panel-heading').getByRole('button', { name: '+ Add source', exact: true });
|
||||
await add.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Add image source');
|
||||
await expect(dialog(page).locator('label[for="action-input"]')).toHaveText('Folder path');
|
||||
await expect(dialog(page).locator('#action-input')).toHaveValue('');
|
||||
await expect(dialog(page).locator('.dialog-actions > button')).toHaveText(['Cancel', 'Add']);
|
||||
await dialog(page).locator('#action-input').fill('output/unthumbnailed');
|
||||
const validated = page.waitForResponse(response => response.url().endsWith('/prompt_manager/settings/validate-source'));
|
||||
await dialog(page).getByRole('button', { name: 'Add', exact: true }).click();
|
||||
expect(await (await validated).json()).toEqual({ success: true, valid: true, label: 'unthumbnailed', path: 'output/unthumbnailed', code: null, reason: null });
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(add).toBeFocused();
|
||||
// Validation wrote nothing: the draft is dirty and the server still has two sources.
|
||||
expect(await rowsOf(page)).toEqual([
|
||||
['ComfyUI output', 'output · manual scan', ['Scan folder', 'Remove']],
|
||||
['ComfyUI input', 'input · manual scan', ['Scan folder', 'Remove']],
|
||||
['unthumbnailed', 'output/unthumbnailed · not saved yet', ['Remove']],
|
||||
]);
|
||||
await expect(statusOf(page)).toHaveText('Unsaved changes.');
|
||||
expect((await serverJson(request, SETTINGS_URL)).sources.effective).toHaveLength(2);
|
||||
const saved = page.waitForRequest(item => item.method() === 'POST' && item.url().endsWith(SETTINGS_URL));
|
||||
await saveButton(page).click();
|
||||
const body = (await saved).postDataJSON();
|
||||
expect(body.gallery_root_paths).toEqual(['output', 'input', 'output/unthumbnailed']);
|
||||
expect(typeof body.expected_revision).toBe('string');
|
||||
await expect(statusOf(page)).toHaveText('Saved to the server configuration.');
|
||||
// The list is the server's again: three effective sources, each scannable.
|
||||
await expect(sourceRow(page, 'unthumbnailed').locator('p')).toHaveText('output/unthumbnailed · manual scan');
|
||||
await expect(sourceRow(page, 'unthumbnailed').getByRole('button', { name: 'Scan folder', exact: true })).toBeVisible();
|
||||
expect((await serverJson(request, SETTINGS_URL)).sources.effective.map(source => source.path)).toEqual(['output', 'input', 'output/unthumbnailed']);
|
||||
});
|
||||
|
||||
test('invalid source shows the server reason', async ({ page, request }) => {
|
||||
const layout = await openSources(page);
|
||||
await layout.getByRole('button', { name: '+ Add source', exact: true }).click();
|
||||
const field = dialog(page).locator('#action-input');
|
||||
for (const [path, reason, submit] of [
|
||||
['output/not-there', 'Gallery root does not exist', 'click'],
|
||||
// Enter in the field adds too (M6).
|
||||
['/etc', OUTSIDE_REASON, 'Enter'],
|
||||
]) {
|
||||
await field.fill(path);
|
||||
if (submit === 'Enter') await field.press('Enter');
|
||||
else await dialog(page).getByRole('button', { name: 'Add', exact: true }).click();
|
||||
await expect(dialog(page).locator('p.small[role="alert"]')).toHaveText(reason);
|
||||
await expect(field).toHaveAttribute('aria-invalid', 'true');
|
||||
await expect(field).toHaveValue(path);
|
||||
}
|
||||
// A folder already listed (the server's answer names it) is not added twice.
|
||||
await field.fill('input');
|
||||
await dialog(page).getByRole('button', { name: 'Add', exact: true }).click();
|
||||
await expect(dialog(page).locator('p.small[role="alert"]')).toHaveText('This folder is already a source.');
|
||||
// A check that cannot finish says so and adds nothing.
|
||||
await page.route('**/prompt_manager/settings/validate-source', route => route.fulfill({ status: 500,
|
||||
json: { success: false, code: null, error: 'Failed to check the source' } }));
|
||||
await field.fill('output/not-checked');
|
||||
await dialog(page).getByRole('button', { name: 'Add', exact: true }).click();
|
||||
await expect(dialog(page).locator('p.small[role="alert"]')).toHaveText('The folder could not be checked. Try again.');
|
||||
await page.unroute('**/prompt_manager/settings/validate-source');
|
||||
// Typing again clears the reason.
|
||||
await field.fill('output/unthumbnailed');
|
||||
await expect(dialog(page).locator('p.small[role="alert"]')).toHaveCount(0);
|
||||
await expect(field).not.toHaveAttribute('aria-invalid', 'true');
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(statusOf(page)).toHaveText('No unsaved changes.');
|
||||
expect((await serverJson(request, SETTINGS_URL)).sources.configured).toHaveLength(2);
|
||||
});
|
||||
|
||||
test('Remove marks the draft; removing the last entry states auto-detect', async ({ page, request }) => {
|
||||
await openSources(page);
|
||||
await sourceRow(page, 'ComfyUI input').getByRole('button', { name: 'Remove', exact: true }).click();
|
||||
expect(await rowsOf(page)).toEqual([['ComfyUI output', 'output · manual scan', ['Scan folder', 'Remove']]]);
|
||||
await expect(statusOf(page)).toHaveText('Unsaved changes.');
|
||||
await sourceRow(page, 'ComfyUI output').getByRole('button', { name: 'Remove', exact: true }).click();
|
||||
expect(await rowsOf(page)).toEqual([['p', 'small', 'No source is configured. After saving, ComfyUI’s output folder is used.']]);
|
||||
await saveButton(page).click();
|
||||
await expect(statusOf(page)).toHaveText('Saved to the server configuration.');
|
||||
// Saved: ComfyUI's output folder is detected and listed without Remove, its row saying it is the default.
|
||||
await expect.poll(rowsOf.bind(null, page)).toEqual([
|
||||
['ComfyUI output', 'ComfyUI/output · manual scan · default', ['Scan folder']],
|
||||
]);
|
||||
const { sources } = await serverJson(request, SETTINGS_URL);
|
||||
expect([sources.configured, sources.auto_detected]).toEqual([[], true]);
|
||||
});
|
||||
|
||||
test('a configured source that does not exist says why and can be removed', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/missing-source')).toBeOK();
|
||||
await openSources(page);
|
||||
// The fixture's unmounted drive is not inside a ComfyUI folder, so the server says that (it never probes outside).
|
||||
expect((await rowsOf(page)).at(-1)).toEqual(['unmounted-drive', `unmounted-drive · ${OUTSIDE_REASON}`, ['Remove']]);
|
||||
await sourceRow(page, 'unmounted-drive').getByRole('button', { name: 'Remove', exact: true }).click();
|
||||
await expect(sourceList(page).locator('.operation')).toHaveCount(2);
|
||||
await saveButton(page).click();
|
||||
await expect(statusOf(page)).toHaveText('Saved to the server configuration.');
|
||||
expect((await serverJson(request, SETTINGS_URL)).sources.configured.map(source => source.path)).toEqual(['output', 'input']);
|
||||
});
|
||||
|
||||
test('watching only with a live observer; restart pending otherwise', async ({ page, request }) => {
|
||||
// No image monitor in this process: every source is scanned by hand.
|
||||
await openSources(page);
|
||||
await expect(sourceList(page).locator('.operation p')).toHaveText(['output · manual scan', 'input · manual scan']);
|
||||
// A live observer that scheduled output: output is watched; input was added since it started.
|
||||
await expect(await request.post('/__test__/system/monitor', { data: { watching: ['output'] } })).toBeOK();
|
||||
await page.reload();
|
||||
await expect(sourceList(page).locator('.operation p')).toHaveText(['output · watching', 'input · restart pending']);
|
||||
await expect(sourceList(page).locator('.operation .row-actions > button:first-child')).toHaveText(['Scan changes', 'Scan folder']);
|
||||
// An observer that died watches nothing.
|
||||
await expect(await request.post('/__test__/system/monitor', { data: { watching: ['output', 'input'], alive: false } })).toBeOK();
|
||||
await page.reload();
|
||||
await expect(sourceList(page).locator('.operation p')).toHaveText(['output · restart pending', 'input · restart pending']);
|
||||
});
|
||||
|
||||
test('Watch for new outputs saves monitoring_enabled; a restart applies it', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/monitor', { data: { watching: ['output', 'input'] } })).toBeOK();
|
||||
const layout = await openSources(page);
|
||||
const monitoring = layout.getByRole('switch', { name: 'Watch for new outputs', exact: true });
|
||||
await expect(monitoring).toBeChecked();
|
||||
await expect(layout.locator('.setting-row').filter({ has: page.locator('#monitoring') }).locator('p'))
|
||||
.toHaveText('New files enter the same library. Existing scans do not count as new runs.');
|
||||
await monitoring.uncheck();
|
||||
await expect(statusOf(page)).toHaveText('Unsaved changes.');
|
||||
const saved = page.waitForResponse(response => response.request().method() === 'POST' && response.url().endsWith(SETTINGS_URL));
|
||||
await saveButton(page).click();
|
||||
const answer = await (await saved).json();
|
||||
expect(answer.restart_required_fields).toEqual(['monitoring_enabled']);
|
||||
// D128: saved for the next start, and the status says a restart applies it.
|
||||
await expect(statusOf(page)).toHaveText('Saved to the server configuration. Restart ComfyUI to apply: Watch for new outputs.');
|
||||
// The running observer keeps watching until ComfyUI restarts, and the list says so.
|
||||
const { settings, monitor } = await serverJson(request, SETTINGS_URL);
|
||||
expect([settings.monitoring_enabled, monitor.observer_alive]).toEqual([false, true]);
|
||||
await expect(sourceList(page).locator('.operation p')).toHaveText(['output · watching until restart', 'input · watching until restart']);
|
||||
await expect(sourceList(page).locator('.operation .row-actions > button:first-child')).toHaveText(['Scan changes', 'Scan changes']);
|
||||
// Switched back on before the restart: the observer still runs, so nothing waits for one.
|
||||
await monitoring.check();
|
||||
await saveButton(page).click();
|
||||
await expect(statusOf(page)).toHaveText('Saved to the server configuration.');
|
||||
await expect(sourceList(page).locator('.operation p')).toHaveText(['output · watching', 'input · watching']);
|
||||
});
|
||||
|
||||
test('worker range max is cpu_count', async ({ page, request }) => {
|
||||
const { settings } = await serverJson(request, SETTINGS_URL);
|
||||
const layout = await openSources(page);
|
||||
const workers = layout.locator('#workers');
|
||||
await expect(layout.locator('label[for="workers"]')).toHaveText('Worker threads');
|
||||
await expect(workers).toHaveAttribute('type', 'range');
|
||||
await expect(workers).toHaveAttribute('min', '1');
|
||||
await expect(workers).toHaveAttribute('max', String(settings.cpu_count));
|
||||
await expect(workers).toHaveValue(String(settings.worker_threads));
|
||||
const next = settings.worker_threads === 1 ? settings.cpu_count : 1;
|
||||
await workers.fill(String(next));
|
||||
await expect(statusOf(page)).toHaveText(next === settings.worker_threads ? 'No unsaved changes.' : 'Unsaved changes.');
|
||||
test.skip(next === settings.worker_threads, 'One core: the range has a single value');
|
||||
const saved = page.waitForRequest(item => item.method() === 'POST' && item.url().endsWith(SETTINGS_URL));
|
||||
await saveButton(page).click();
|
||||
expect((await saved).postDataJSON().worker_threads).toBe(next);
|
||||
await expect(statusOf(page)).toHaveText('Saved to the server configuration.');
|
||||
expect((await serverJson(request, SETTINGS_URL)).settings.worker_threads).toBe(next);
|
||||
});
|
||||
|
||||
test('Refresh files calls /gallery/rescan and toasts its counts', async ({ page }) => {
|
||||
const layout = await openSources(page);
|
||||
const refresh = layout.locator('.operation').filter({ has: page.getByRole('heading', { name: 'Refresh files', exact: true }) });
|
||||
await expect(refresh.locator('p')).toHaveText('Read every source again to find files added or removed outside ComfyUI.');
|
||||
const answered = page.waitForResponse(response => response.url().endsWith('/prompt_manager/gallery/rescan'));
|
||||
await refresh.getByRole('button', { name: 'Refresh', exact: true }).click();
|
||||
const { total, roots } = await (await answered).json();
|
||||
expect(roots).toBe(2);
|
||||
await expect(toast(page)).toHaveText(`Files refreshed. ${total.toLocaleString('en-US')} files in 2 output folders.`);
|
||||
// A failed refresh says so.
|
||||
await page.route('**/prompt_manager/gallery/rescan', route => route.fulfill({ status: 500, json: { success: false, error: 'busy disk' } }));
|
||||
await refresh.getByRole('button', { name: 'Refresh', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Files could not be refreshed. Try again.');
|
||||
});
|
||||
|
||||
const THUMBNAIL_PLAN_URL = '/prompt_manager/images/thumbnails/plan';
|
||||
const THUMBNAIL_JOB_URL = '/prompt_manager/jobs/thumbnails';
|
||||
const planCount = value => value.toLocaleString('en-US');
|
||||
|
||||
/** Open "Thumbnail maintenance" from Sources' Thumbnail generation → Review options. */
|
||||
async function openThumbnailReview(page) {
|
||||
const layout = await openSources(page);
|
||||
const thumbnails = layout.locator('.operation').filter({ has: page.getByRole('heading', { name: 'Thumbnail generation', exact: true }) });
|
||||
await expect(thumbnails.locator('p')).toHaveText('Generate missing previews; rebuilding all is a separate explicit option.');
|
||||
await thumbnails.getByRole('button', { name: 'Review options', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Thumbnail maintenance');
|
||||
return dialog(page);
|
||||
}
|
||||
|
||||
/** The thumbnail requests this page sends: plan reads (their query) and job starts (their body). */
|
||||
function thumbnailRequests(page) {
|
||||
const seen = { plans: [], starts: [] };
|
||||
page.on('request', item => {
|
||||
const url = new URL(item.url());
|
||||
if (url.pathname === THUMBNAIL_PLAN_URL) seen.plans.push(Object.fromEntries(url.searchParams));
|
||||
if (item.method() === 'POST' && url.pathname === THUMBNAIL_JOB_URL) seen.starts.push(item.postDataJSON());
|
||||
});
|
||||
return seen;
|
||||
}
|
||||
|
||||
const missingLine = plan => `${planCount(plan.targets)} ${plan.targets === 1 ? 'preview' : 'previews'} to generate · ` +
|
||||
`${planCount(plan.fresh)} already up to date.`;
|
||||
|
||||
test('Review options shows the plan counts for the chosen scope/quality/size before Start', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/unthumbnailed', { data: { count: 3 } })).toBeOK();
|
||||
// Image loading quality (Appearance, Q3) preselects the dialog's quality.
|
||||
await page.addInitScript(() => { if (location.protocol.startsWith('http')) localStorage.setItem('gallerySettings', JSON.stringify({ imageQuality: 'high' })); });
|
||||
const seen = thumbnailRequests(page);
|
||||
const plan = await serverJson(request, `${THUMBNAIL_PLAN_URL}?mode=missing&size=512&quality=high`);
|
||||
expect(plan.targets).toBe(3);
|
||||
const review = await openThumbnailReview(page);
|
||||
await expect(review.locator('#modal-body > *')).toHaveCount(5);
|
||||
await expect(review.locator('.field > label')).toHaveText(['Scope', 'Quality', 'Preview size limit']);
|
||||
await expect(review.locator('#thumb-mode option')).toHaveText(['Missing thumbnails only', 'Rebuild all (explicit opt-in)']);
|
||||
await expect(review.locator('#thumb-quality option')).toHaveText(['Medium · optimized', 'High · original', 'Low · fastest']);
|
||||
await expect(review.locator('#thumb-mode')).toHaveValue('missing');
|
||||
await expect(review.locator('#thumb-quality')).toHaveValue('high');
|
||||
await expect(review.locator('#thumb-limit')).toHaveValue('512');
|
||||
await expect(review.locator('#thumb-limit')).toHaveAttribute('min', '128');
|
||||
await expect(review.locator('#thumb-limit')).toHaveAttribute('max', '2048');
|
||||
const line = review.locator('#modal-body > p.small');
|
||||
await expect(line).toHaveAttribute('role', 'status');
|
||||
await expect(line).toHaveText(missingLine(plan));
|
||||
expect(seen.plans).toEqual([{ mode: 'missing', size: '512', quality: 'high' }]);
|
||||
await expect(review.locator('.dialog-actions > button')).toHaveText(['Cancel', 'Start']);
|
||||
// Reviewing starts nothing; Cancel closes.
|
||||
await review.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(seen.starts).toEqual([]);
|
||||
expect((await serverJson(request, '/prompt_manager/jobs')).jobs).toEqual([]);
|
||||
});
|
||||
|
||||
test('size outside 128–2048 reports validity and requests nothing', async ({ page, request }) => {
|
||||
const seen = thumbnailRequests(page);
|
||||
const review = await openThumbnailReview(page);
|
||||
const line = review.locator('#modal-body > p.small');
|
||||
await expect(line).toHaveText(/ already up to date\.$/);
|
||||
const size = review.locator('#thumb-limit');
|
||||
for (const value of ['4096', '64', '']) {
|
||||
await size.fill(value);
|
||||
await expect(size).toHaveAttribute('aria-invalid', 'true');
|
||||
await expect(line).toHaveText('Enter a size from 128 to 2048 pixels.');
|
||||
expect(await size.evaluate(input => input.validity.valid)).toBe(false);
|
||||
// Start keeps the dialog open and sends nothing; neither does a scope change.
|
||||
await review.getByRole('button', { name: 'Start', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Thumbnail maintenance');
|
||||
await review.locator('#thumb-mode').selectOption('rebuild');
|
||||
await expect(line).toHaveText('Enter a size from 128 to 2048 pixels.');
|
||||
await review.locator('#thumb-mode').selectOption('missing');
|
||||
}
|
||||
expect(seen.starts).toEqual([]);
|
||||
expect(seen.plans).toEqual([{ mode: 'missing', size: '512', quality: 'medium' }]);
|
||||
await size.fill('1024');
|
||||
await expect(size).not.toHaveAttribute('aria-invalid');
|
||||
await expect(line).toHaveText(/ already up to date\.$/);
|
||||
expect((await serverJson(request, '/prompt_manager/jobs')).jobs).toEqual([]);
|
||||
});
|
||||
|
||||
test('Start closes the dialog, toasts, and opens the job in Activity', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/unthumbnailed', { data: { count: 2 } })).toBeOK();
|
||||
const seen = thumbnailRequests(page);
|
||||
const review = await openThumbnailReview(page);
|
||||
await review.locator('#thumb-quality').selectOption('low');
|
||||
await review.locator('#thumb-limit').fill('256');
|
||||
await review.getByRole('button', { name: 'Start', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
// The toast says it started (or, for a run this quick, that it already finished); Activity shows its progress.
|
||||
await expect(toast(page)).toHaveText(/^(Thumbnail generation started\.|Thumbnails finished · 2 generated · 0 errors)$/);
|
||||
expect(seen.starts).toEqual([{ mode: 'missing', size: 256, quality: 'low' }]);
|
||||
const id = page.url().split('/').pop();
|
||||
const { job } = await serverJson(request, `/prompt_manager/jobs/${id}`);
|
||||
expect(job).toMatchObject({ kind: 'thumbnails', title: 'Thumbnail generation', scope: { mode: 'missing', size: 256, quality: 'low' } });
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Thumbnail generation');
|
||||
await expect.poll(async () => (await serverJson(request, `/prompt_manager/jobs/${id}`)).job.state).toBe('finished');
|
||||
});
|
||||
|
||||
test('Rebuild all is explicit and the plan line says every preview is regenerated', async ({ page, request }) => {
|
||||
const seen = thumbnailRequests(page);
|
||||
const rebuild = await serverJson(request, `${THUMBNAIL_PLAN_URL}?mode=rebuild&size=512&quality=medium`);
|
||||
expect(rebuild.targets).toBeGreaterThan(0);
|
||||
const review = await openThumbnailReview(page);
|
||||
// Missing thumbnails only is the default; rebuilding is the explicit second option.
|
||||
await expect(review.locator('#thumb-mode')).toHaveValue('missing');
|
||||
const line = review.locator('#modal-body > p.small');
|
||||
await expect(line).toHaveText(/ already up to date\.$/);
|
||||
await review.locator('#thumb-mode').selectOption('rebuild');
|
||||
await expect(line).toHaveText(`Every preview is regenerated: ${planCount(rebuild.targets)} files.`);
|
||||
expect(seen.plans).toEqual([{ mode: 'missing', size: '512', quality: 'medium' }, { mode: 'rebuild', size: '512', quality: 'medium' }]);
|
||||
// Back and forth reads each scope once.
|
||||
await review.locator('#thumb-mode').selectOption('missing');
|
||||
await expect(line).toHaveText(/ already up to date\.$/);
|
||||
await review.locator('#thumb-mode').selectOption('rebuild');
|
||||
await expect(line).toHaveText(`Every preview is regenerated: ${planCount(rebuild.targets)} files.`);
|
||||
expect(seen.plans).toHaveLength(2);
|
||||
await review.getByRole('button', { name: 'Start', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
expect(seen.starts).toEqual([{ mode: 'rebuild', size: 512, quality: 'medium' }]);
|
||||
const id = page.url().split('/').pop();
|
||||
await expect.poll(async () => (await serverJson(request, `/prompt_manager/jobs/${id}`)).job.state).toBe('finished');
|
||||
});
|
||||
|
||||
test('a plan that cannot be read says so; Start still starts', async ({ page }) => {
|
||||
await page.route(`**${THUMBNAIL_PLAN_URL}?*`, route => route.fulfill({ status: 500, json: { success: false, error: 'disk' } }));
|
||||
const review = await openThumbnailReview(page);
|
||||
await expect(review.locator('#modal-body > p.small')).toHaveText('The previews could not be counted.');
|
||||
await expect(review.getByRole('button', { name: 'Start', exact: true })).not.toHaveAttribute('aria-disabled');
|
||||
await review.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
});
|
||||
|
||||
test('a refused start shows the busy link toast', async ({ page, request }) => {
|
||||
const running = await (await request.post('/__test__/jobs/start', { data: { title: 'Scan for metadata' } })).json();
|
||||
const review = await openThumbnailReview(page);
|
||||
await review.getByRole('button', { name: 'Start', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(toast(page)).toHaveText('Another job is running: Scan for metadata.');
|
||||
await expect(toast(page).getByRole('link', { name: 'Scan for metadata', exact: true })).toHaveAttribute('href', `#/activity/${running.job.id}`);
|
||||
await expect(page).toHaveURL(`${workspace}#/system/sources`);
|
||||
expect((await serverJson(request, '/prompt_manager/jobs')).jobs.map(job => job.kind)).toEqual(['scan']);
|
||||
await request.post(`/__test__/jobs/${running.job.id}/finish`, { data: { state: 'finished' } });
|
||||
});
|
||||
|
||||
test('a save refused by a running scan shows the busy link toast', async ({ page, request }) => {
|
||||
const running = await (await request.post('/__test__/jobs/start', { data: { title: 'Scan for metadata' } })).json();
|
||||
await openSources(page);
|
||||
await sourceRow(page, 'ComfyUI input').getByRole('button', { name: 'Remove', exact: true }).click();
|
||||
await saveButton(page).click();
|
||||
await expect(toast(page)).toHaveText('Another job is running: Scan for metadata.');
|
||||
await expect(toast(page).getByRole('link', { name: 'Scan for metadata', exact: true })).toHaveAttribute('href', `#/activity/${running.job.id}`);
|
||||
await expect(statusOf(page)).toHaveText('Some changes were not saved. Review the highlighted settings.');
|
||||
// The draft is kept for the next Save, and the server still has both sources.
|
||||
await expect(sourceList(page).locator('.operation')).toHaveCount(1);
|
||||
expect((await serverJson(request, SETTINGS_URL)).sources.configured).toHaveLength(2);
|
||||
// An answer that does not name its job (an older server): the toast names the running one from the job list.
|
||||
await page.route(`**${SETTINGS_URL}`, route => (route.request().method() === 'POST'
|
||||
? route.fulfill({ status: 409, json: { success: false, code: 'busy', error: 'Scan for metadata already running' } })
|
||||
: route.continue()));
|
||||
await page.locator('#workspace-toast').evaluate(node => { node.textContent = ''; });
|
||||
await saveButton(page).click();
|
||||
await expect(toast(page)).toHaveText('Another job is running: Scan for metadata.');
|
||||
await expect(toast(page).getByRole('link', { name: 'Scan for metadata', exact: true })).toHaveAttribute('href', `#/activity/${running.job.id}`);
|
||||
await page.unroute(`**${SETTINGS_URL}`);
|
||||
await request.post(`/__test__/jobs/${running.job.id}/finish`, { data: { state: 'finished' } });
|
||||
});
|
||||
|
||||
test('a source list that cannot be read says so and retries', async ({ page }) => {
|
||||
await page.addInitScript(key => { if (location.protocol.startsWith('http')) localStorage.setItem(key, '3.2.0'); }, NOTICE_KEY);
|
||||
let failing = true;
|
||||
await page.route(`**${SETTINGS_URL}`, route => (failing && route.request().method() === 'GET'
|
||||
? route.fulfill({ status: 500, json: { success: false, error: 'Injected failure' } }) : route.continue()));
|
||||
await page.goto(`${workspace}#/system/sources`);
|
||||
await expect(sourceList(page).getByRole('alert')).toHaveText('Sources could not be loaded. Your saved data has not changed.');
|
||||
await expect(sourceList(page).locator('.operation')).toHaveCount(0);
|
||||
// The server settings rows say so too, and stay disabled.
|
||||
await expect(layoutOf(page).locator('#workers')).toBeDisabled();
|
||||
failing = false;
|
||||
await sourceList(page).getByRole('button', { name: 'Retry', exact: true }).click();
|
||||
await expect(sourceList(page).locator('.operation')).toHaveCount(2);
|
||||
});
|
||||
|
||||
test('a scan from a list that changed since it was read starts nothing, says why and reads the list again', async ({ page, request }) => {
|
||||
await openSources(page);
|
||||
// The legacy page (or another tab) removes ComfyUI output: the page's "input" is index 1, now past the end.
|
||||
await expect(await request.post(SETTINGS_URL, { data: { gallery_root_paths: ['input'] } })).toBeOK();
|
||||
const refused = page.waitForResponse(response => response.url().endsWith('/prompt_manager/jobs/scan'));
|
||||
await sourceRow(page, 'ComfyUI input').getByRole('button', { name: 'Scan folder', exact: true }).click();
|
||||
await dialog(page).getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
const answer = await refused;
|
||||
expect([answer.status(), (await answer.json()).code]).toEqual([409, 'sources-changed']);
|
||||
await expect(toast(page)).toHaveText('The sources changed since they were loaded. Reload them.');
|
||||
// No scan started (the other save ran as its own source-change hold).
|
||||
expect((await serverJson(request, '/prompt_manager/jobs')).jobs.map(job => job.kind)).toEqual(['source_change']);
|
||||
// Read again: the one source is index 0 now, and its scan starts.
|
||||
await expect.poll(rowsOf.bind(null, page)).toEqual([['ComfyUI input', 'input · manual scan', ['Scan folder', 'Remove']]]);
|
||||
await expect(statusOf(page)).toHaveText('No unsaved changes.');
|
||||
const started = page.waitForRequest(item => item.method() === 'POST' && item.url().endsWith('/prompt_manager/jobs/scan'));
|
||||
await sourceRow(page, 'ComfyUI input').getByRole('button', { name: 'Scan folder', exact: true }).click();
|
||||
await dialog(page).getByRole('button', { name: 'Start scan', exact: true }).click();
|
||||
expect((await started).postDataJSON()).toEqual({ scope: 'new', roots: [0], paths: ['input'] });
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
});
|
||||
|
||||
test('a save refused for one unusable source names it and marks its row', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/missing-source', { data: { inside: true } })).toBeOK();
|
||||
await openSources(page);
|
||||
await expect(sourceRow(page, 'unplugged').locator('p')).toHaveText('output/unplugged · manual scan');
|
||||
// The drive is unplugged after the page read the list; removing another source resends the whole list.
|
||||
await expect(await request.post('/__test__/system/unplug')).toBeOK();
|
||||
await sourceRow(page, 'ComfyUI input').getByRole('button', { name: 'Remove', exact: true }).click();
|
||||
const saved = page.waitForResponse(response => response.request().method() === 'POST' && response.url().endsWith(SETTINGS_URL));
|
||||
await saveButton(page).click();
|
||||
const answer = await saved;
|
||||
expect(answer.status()).toBe(400);
|
||||
expect(await answer.json()).toMatchObject({ code: 'invalid-source', source: 'output/unplugged', index: 1 });
|
||||
await expect(statusOf(page)).toHaveText("output/unplugged can't be used: Gallery root does not exist. Remove it or reconnect it, then save.");
|
||||
await expect(sourceRow(page, 'unplugged')).toHaveAttribute('aria-invalid', 'true');
|
||||
await expect(sourceList(page).locator('.operation[aria-invalid="true"]')).toHaveCount(1);
|
||||
expect((await serverJson(request, SETTINGS_URL)).sources.configured).toHaveLength(3);
|
||||
// Removed, the list saves.
|
||||
await sourceRow(page, 'unplugged').getByRole('button', { name: 'Remove', exact: true }).click();
|
||||
await expect(sourceList(page).locator('.operation[aria-invalid="true"]')).toHaveCount(0);
|
||||
await saveButton(page).click();
|
||||
await expect(statusOf(page)).toHaveText('Saved to the server configuration.');
|
||||
expect((await serverJson(request, SETTINGS_URL)).sources.configured.map(source => source.path)).toEqual(['output']);
|
||||
});
|
||||
@@ -0,0 +1,737 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
|
||||
// System › Overview (#249 System S2 Task 6) on the concept-shaped real fixture: the metrics, rows, Statistics dialog and
|
||||
// Health report read the server, never sample values (D93, D95, D96). The hooks are workspace_system_fixture.py's.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const OVERVIEW_URL = '/prompt_manager/system/overview';
|
||||
const THUMBNAILS_URL = '/prompt_manager/images/thumbnails/status';
|
||||
const number = value => value.toLocaleString('en-US');
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
page.manifest = await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
page.systemErrors = [];
|
||||
page.on('pageerror', error => page.systemErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/release');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.systemErrors).toEqual([]);
|
||||
});
|
||||
|
||||
async function serverJson(request, url) {
|
||||
const response = await request.get(url);
|
||||
await expect(response).toBeOK();
|
||||
return response.json();
|
||||
}
|
||||
|
||||
const layoutOf = page => page.locator('#workspace-main .settings-layout');
|
||||
|
||||
async function openOverview(page) {
|
||||
await page.goto(`${workspace}#/system/overview`);
|
||||
await expect(layoutOf(page).locator(':scope > .summary-metrics')).toBeVisible();
|
||||
return layoutOf(page);
|
||||
}
|
||||
|
||||
/** The rows of one Overview surface: `[h3, p, button]`. */
|
||||
function rowsOf(layout, heading) {
|
||||
return layout.locator('.two-col > section.surface').filter({ has: layout.page().getByRole('heading', { name: heading, exact: true }) })
|
||||
.locator('.operation').evaluateAll(rows => rows.map(row => [row.querySelector('h3').textContent,
|
||||
row.querySelector('p').textContent, row.querySelector('.row-actions button').textContent]));
|
||||
}
|
||||
|
||||
test('overview shows real counts and Not checked', async ({ page, request }) => {
|
||||
const writes = [];
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) writes.push(item.url()); });
|
||||
const overview = await serverJson(request, OVERVIEW_URL);
|
||||
const thumbnails = await serverJson(request, THUMBNAILS_URL);
|
||||
// The concept fixture: six positives, their saved negatives and 24 linked images, all with thumbnails.
|
||||
expect([overview.library.positive_prompts, overview.images.indexed, thumbnails.pending]).toEqual([6, 24, 0]);
|
||||
// I2: the Overview never walks the sources for thumbnails; the row reads the status route itself.
|
||||
expect(overview).not.toHaveProperty('thumbnails');
|
||||
// The fixture records no ambiguous file: the Needs review queue is empty.
|
||||
expect(overview.needs_review).toEqual({ pending: 0 });
|
||||
const layout = await openOverview(page);
|
||||
expect(await layout.locator('.summary-metrics > .metric').evaluateAll(items => items.map(item =>
|
||||
[item.querySelector('strong').textContent, item.querySelector('span').textContent]))).toEqual([
|
||||
[number(overview.library.positive_prompts), 'Positive prompts'],
|
||||
[number(overview.library.negative_variants), 'Negative variants'],
|
||||
[number(overview.images.indexed), 'Images'],
|
||||
['Not checked', 'Database integrity'],
|
||||
]);
|
||||
expect(await rowsOf(layout, 'Library health')).toEqual([
|
||||
['0 ambiguous image links', 'Review metadata before assigning a positive / negative pair.', 'Review metadata'],
|
||||
['Backup status', 'No retained backup record.', 'View backups'],
|
||||
// D129: the fixture's database is configured explicitly, so it reads as config.json's choice and does not move.
|
||||
['Library location', 'browser.db · The database path set in config.json.', 'Details'],
|
||||
['No missing thumbnails', 'Generate only missing previews when necessary.', 'Review thumbnails'],
|
||||
]);
|
||||
expect(await rowsOf(layout, 'Workspace preferences')).toEqual([
|
||||
['One display configuration', 'Grid and density settings are shared across views.', 'Appearance'],
|
||||
['2 output sources', 'Monitoring and enrichment share one source list.', 'Manage sources'],
|
||||
['Statistics', 'Runs and ratings by category, with library totals.', 'Explore'],
|
||||
]);
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
|
||||
// D129 (#249 S6b): where the library lives. The hook makes the fixture's database read as a node-folder library; its
|
||||
// Move library is a stand-in that touches no file (workspace_system_fixture.py).
|
||||
const banner = page => page.locator('#library-banner');
|
||||
const LIBRARY_URL = '/prompt_manager/system/library';
|
||||
|
||||
test('a node-folder library is offered a move above every page; Keep it here stops the offer', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/library', { data: { state: 'node_folder' } })).toBeOK();
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(banner(page).locator('.notice h2')).toHaveText('Your library is inside the node folder');
|
||||
await expect(banner(page).locator('.notice p')).toHaveText(/^Your library is stored inside the node folder, where an update or reinstall can delete it\./);
|
||||
await expect(banner(page).locator('.notice .row-actions button')).toHaveText(['Keep it here', 'Move library']);
|
||||
await expect(banner(page).getByRole('button', { name: 'Move library', exact: true })).toHaveClass(/ca-primary/);
|
||||
await page.goto(`${workspace}#/library`);
|
||||
await expect(banner(page).locator('.notice')).toBeVisible();
|
||||
await banner(page).getByRole('button', { name: 'Keep it here', exact: true }).click();
|
||||
await expect(banner(page)).toBeHidden();
|
||||
const { location } = await serverJson(request, LIBRARY_URL);
|
||||
expect([location.keep, location.offer, location.movable]).toEqual([true, false, true]);
|
||||
await page.reload();
|
||||
const layout = await openOverview(page);
|
||||
await expect(banner(page)).toBeHidden();
|
||||
const row = layout.locator('.operation').filter({ hasText: 'Library location' });
|
||||
await expect(row.locator('p')).toHaveText('browser.db · The node folder, where an update or reinstall can delete it. '
|
||||
+ 'You chose to keep it here.');
|
||||
// The row stays: the move is still one press away.
|
||||
await expect(row.locator('.row-actions button')).toHaveText('Move library');
|
||||
});
|
||||
|
||||
test('Move library is reviewed, runs as an Activity job and ends the offer', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/library', { data: { state: 'node_folder' } })).toBeOK();
|
||||
const layout = await openOverview(page);
|
||||
const row = layout.locator('.operation').filter({ hasText: 'Library location' });
|
||||
await expect(row.locator('p')).toHaveText('browser.db · The node folder, where an update or reinstall can delete it.');
|
||||
const move = banner(page).getByRole('button', { name: 'Move library', exact: true });
|
||||
await move.click();
|
||||
const dialog = page.locator('dialog#modal[open]');
|
||||
await expect(dialog.locator('#modal-title')).toHaveText('Move library');
|
||||
expect(await dialog.locator('dl.meta-grid > div').evaluateAll(items => items.map(item =>
|
||||
[item.querySelector('dt').textContent, item.querySelector('dd').textContent]))).toEqual([
|
||||
['From', 'browser.db'], ['To', 'user/__promptmanager/prompts.db'], ['Size', expect.stringMatching(/ (KB|MB)$/)],
|
||||
['Moves with it', 'config.json and the database backups'], ['Old files', 'Renamed *.moved-to-…, never deleted'],
|
||||
]);
|
||||
await expect(dialog.locator('.dialog-actions button')).toHaveText(['Cancel', 'Move library']);
|
||||
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open]')).toHaveCount(0);
|
||||
await expect(move).toBeFocused();
|
||||
await row.getByRole('button', { name: 'Move library', exact: true }).click();
|
||||
await dialog.getByRole('button', { name: 'Move library', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[^/]+$/);
|
||||
await expect(page.locator('#workspace-main')).toContainText(
|
||||
/Moved the library to user\/__promptmanager: \d+ prompts? and 0 images verified\. The old files were renamed browser\.db\.moved-to-__promptmanager; nothing was deleted\./);
|
||||
// The offer ends with the job; the Overview says where the library lives now.
|
||||
await expect(banner(page)).toBeHidden();
|
||||
const after = await openOverview(page);
|
||||
await expect(after.locator('.operation').filter({ hasText: 'Library location' }).locator('p'))
|
||||
.toHaveText('browser.db · ComfyUI\'s user folder, which updates and reinstalls of the node leave alone.');
|
||||
});
|
||||
|
||||
test('a move that fails says so and keeps the offer; a refused start says why', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/library', { data: { state: 'failing' } })).toBeOK();
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await banner(page).getByRole('button', { name: 'Move library', exact: true }).click();
|
||||
await page.locator('dialog#modal[open]').getByRole('button', { name: 'Move library', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[^/]+$/);
|
||||
await expect(page.locator('#workspace-main')).toContainText(
|
||||
'The copy did not pass verification. Nothing was moved. The library is still in the node folder, unchanged.');
|
||||
await expect(banner(page).locator('.notice')).toBeVisible();
|
||||
await expect(await request.post('/__test__/system/fail', { data: { path: `${LIBRARY_URL}/move` } })).toBeOK();
|
||||
await banner(page).getByRole('button', { name: 'Move library', exact: true }).click();
|
||||
await page.locator('dialog#modal[open]').getByRole('button', { name: 'Move library', exact: true }).click();
|
||||
await expect(page.locator('#workspace-toast')).toHaveText('Move library could not start. Try again.');
|
||||
});
|
||||
|
||||
test('a library that does not move shows its details', async ({ page }) => {
|
||||
const layout = await openOverview(page);
|
||||
await expect(banner(page)).toBeHidden();
|
||||
const details = layout.locator('.operation').filter({ hasText: 'Library location' }).getByRole('button', { name: 'Details', exact: true });
|
||||
await details.click();
|
||||
const dialog = page.locator('dialog#modal[open]');
|
||||
await expect(dialog.locator('#modal-title')).toHaveText('Library location');
|
||||
expect(await dialog.locator('dl.meta-grid > div').evaluateAll(items => items.map(item =>
|
||||
[item.querySelector('dt').textContent, item.querySelector('dd').textContent.replace(/[\d.,]+ (KB|MB)$/, 'N')]))).toEqual([
|
||||
['Database', 'browser.db'], ['Why here', 'The database path set in config.json'],
|
||||
['Beside it', 'config.json, the database backups and the logs'], ['Size', 'N'],
|
||||
]);
|
||||
await dialog.getByRole('button', { name: 'Close', exact: true }).click();
|
||||
await expect(details).toBeFocused();
|
||||
});
|
||||
|
||||
test('Run health checks opens Health report and updates metric 4', async ({ page, request }) => {
|
||||
await page.goto(`${workspace}#/system/maintenance`);
|
||||
const layout = layoutOf(page);
|
||||
await expect(layout.locator(':scope > .notice h2')).toHaveText('Check first. Review changes. Then apply.');
|
||||
// The rest of Maintenance is still the existing tools' (D94).
|
||||
await expect(layout.locator(':scope > section.handoff h2')).toHaveText('Your current tools are ready');
|
||||
const run = layout.getByRole('button', { name: 'Run health checks', exact: true });
|
||||
await run.click();
|
||||
const dialog = page.locator('dialog#modal[open]');
|
||||
await expect(dialog.locator('#modal-title')).toHaveText('Health report');
|
||||
const { report } = await serverJson(request, '/prompt_manager/system/health');
|
||||
expect(report.status).toBe('incomplete');
|
||||
expect(await dialog.locator('.operation-list > .operation').evaluateAll(rows => rows.map(row =>
|
||||
[row.querySelector('h3').textContent, row.querySelector('span.chip').textContent]))).toEqual([
|
||||
['Integrity check', 'Passed'], ['Foreign keys & pair references', 'Passed'], ['Canonical hashes', 'Passed'],
|
||||
['Pair uniqueness', 'Passed'], ['Missing files · output', 'Passed'], ['Missing files · input', 'Passed'],
|
||||
['Ambiguous metadata', 'Not available yet'], ['Consistency', 'Passed'],
|
||||
]);
|
||||
await expect(dialog.locator('#modal-body > p.small')).toHaveText(/^Checked .+\. Some checks could not run, so this report is incomplete\.$/);
|
||||
await expect(dialog.locator('.dialog-actions button')).toHaveText(['Close', 'Review missing links']);
|
||||
await dialog.getByRole('button', { name: 'Close', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open]')).toHaveCount(0);
|
||||
await expect(run).toBeFocused();
|
||||
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect(layout.locator('.metric').nth(3).locator('strong')).toHaveText('Incomplete');
|
||||
|
||||
// A missing file and an unmounted source: the report counts the readable sources and names the other one.
|
||||
await expect(await request.post('/__test__/system/missing-file')).toBeOK();
|
||||
await expect(await request.post('/__test__/system/missing-source')).toBeOK();
|
||||
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Maintenance', exact: true }).click();
|
||||
await layout.getByRole('button', { name: 'Run health checks', exact: true }).click();
|
||||
await expect(dialog.locator('.operation').filter({ hasText: 'Missing files' }).locator('.chip'))
|
||||
.toHaveText(['1 link needs review', 'Passed', 'Not available']);
|
||||
await expect(dialog.locator('.operation h3').filter({ hasText: 'Missing files' }))
|
||||
.toHaveText(['Missing files · output', 'Missing files · input', 'Missing files · unmounted-drive']);
|
||||
// Review missing links goes to Maintenance, where the cleanup's existing tool is linked.
|
||||
await dialog.getByRole('button', { name: 'Review missing links', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open]')).toHaveCount(0);
|
||||
await expect(page).toHaveURL(`${workspace}#/system/maintenance`);
|
||||
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect(layout.locator('.metric').nth(3).locator('strong')).toHaveText('1 issue');
|
||||
});
|
||||
|
||||
test('a health check that fails says so and keeps the last result', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/fail', { data: { path: '/prompt_manager/system/health' } })).toBeOK();
|
||||
await page.goto(`${workspace}#/system/maintenance`);
|
||||
const run = layoutOf(page).getByRole('button', { name: 'Run health checks', exact: true });
|
||||
await run.click();
|
||||
await expect(page.locator('#workspace-toast')).toHaveText('The health check could not finish. Nothing was changed.');
|
||||
await expect(page.locator('dialog#modal[open]')).toHaveCount(0);
|
||||
await expect(run).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await openOverview(page);
|
||||
await expect(layoutOf(page).locator('.metric').nth(3).locator('strong')).toHaveText('Not checked');
|
||||
});
|
||||
|
||||
test('thumbnail row reflects the status endpoint', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/unthumbnailed', { data: { count: 3 } })).toBeOK();
|
||||
const status = await serverJson(request, '/prompt_manager/images/thumbnails/status');
|
||||
expect(status.pending).toBe(3);
|
||||
const layout = await openOverview(page);
|
||||
const row = layout.locator('.operation').filter({ has: page.getByRole('button', { name: 'Review thumbnails', exact: true }) });
|
||||
await expect(row.locator('h3')).toHaveText('3 missing thumbnails');
|
||||
});
|
||||
|
||||
test('Overview Review thumbnails opens the same dialog', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/unthumbnailed', { data: { count: 2 } })).toBeOK();
|
||||
const plan = await serverJson(request, '/prompt_manager/images/thumbnails/plan?mode=missing&size=512&quality=medium');
|
||||
const starts = [];
|
||||
page.on('request', item => { if (item.method() === 'POST' && item.url().endsWith('/prompt_manager/jobs/thumbnails')) starts.push(item.postDataJSON()); });
|
||||
const layout = await openOverview(page);
|
||||
const row = layout.locator('.operation').filter({ has: page.getByRole('button', { name: 'Review thumbnails', exact: true }) });
|
||||
await expect(row.locator('h3')).toHaveText('2 missing thumbnails');
|
||||
await row.getByRole('button', { name: 'Review thumbnails', exact: true }).click();
|
||||
const review = page.locator('dialog#modal[open]');
|
||||
await expect(review.locator('#modal-title')).toHaveText('Thumbnail maintenance');
|
||||
await expect(page).toHaveURL(`${workspace}#/system/overview`);
|
||||
await expect(review.locator('.field > label')).toHaveText(['Scope', 'Quality', 'Preview size limit']);
|
||||
await expect(review.locator('#modal-body > p.small')).toHaveText(`2 previews to generate · ${number(plan.fresh)} already up to date.`);
|
||||
// Escape closes it and focus returns to the row's button.
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(review).toHaveCount(0);
|
||||
await expect(row.getByRole('button', { name: 'Review thumbnails', exact: true })).toBeFocused();
|
||||
await row.getByRole('button', { name: 'Review thumbnails', exact: true }).click();
|
||||
await review.getByRole('button', { name: 'Start', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
expect(starts).toEqual([{ mode: 'missing', size: 512, quality: 'medium' }]);
|
||||
const id = page.url().split('/').pop();
|
||||
await expect.poll(async () => (await serverJson(request, `/prompt_manager/jobs/${id}`)).job.state).toBe('finished');
|
||||
});
|
||||
|
||||
test('the thumbnail row paints when its own status read answers', async ({ page, request }) => {
|
||||
const held = async () => (await serverJson(request, '/__test__/hold')).held[THUMBNAILS_URL] ?? 0;
|
||||
await expect(await request.post('/__test__/hold', { data: { path: THUMBNAILS_URL } })).toBeOK();
|
||||
const layout = await openOverview(page);
|
||||
const row = layout.locator('.operation').filter({ has: page.getByRole('button', { name: 'Review thumbnails', exact: true }) });
|
||||
// The other values do not wait for the walk of every source.
|
||||
await expect(row.locator('h3')).toHaveText('Checking thumbnails…');
|
||||
await expect(layout.locator('.summary-metrics > .metric')).toHaveCount(4);
|
||||
await expect.poll(held).toBe(1);
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await expect(row.locator('h3')).toHaveText('No missing thumbnails');
|
||||
});
|
||||
|
||||
test('a failed thumbnail read is unavailable, never zero', async ({ page }) => {
|
||||
await page.route(`**${THUMBNAILS_URL}`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'Injected failure' }) }));
|
||||
const layout = await openOverview(page);
|
||||
const row = layout.locator('.operation').filter({ has: page.getByRole('button', { name: 'Review thumbnails', exact: true }) });
|
||||
await expect(row.locator('h3')).toHaveText('Thumbnail status unavailable');
|
||||
await expect(layout.getByRole('alert')).toHaveCount(0);
|
||||
await expect(layout.locator('.metric').nth(2).locator('strong')).toHaveText('24');
|
||||
});
|
||||
|
||||
test('ambiguous row reads the pending count and Review metadata opens Needs review', async ({ page }) => {
|
||||
const layout = await openOverview(page);
|
||||
const row = layout.locator('.operation').filter({ hasText: 'ambiguous image links' });
|
||||
await expect(row.locator('h3')).toHaveText('0 ambiguous image links');
|
||||
await expect(row.locator('p')).toHaveText('Review metadata before assigning a positive / negative pair.');
|
||||
const review = row.getByRole('button', { name: 'Review metadata', exact: true });
|
||||
await expect(review).not.toHaveAttribute('aria-disabled');
|
||||
await review.click();
|
||||
// The concept's go-review: the Needs review page.
|
||||
await expect(page).toHaveURL(`${workspace}#/review`);
|
||||
await expect(page.getByRole('heading', { name: 'Metadata review', exact: true })).toBeVisible();
|
||||
});
|
||||
|
||||
test('View backups / Appearance / Manage sources switch sections', async ({ page }) => {
|
||||
for (const [label, section, pressed] of [['View backups', 'backups', 'Backup & export'], ['Appearance', 'appearance', 'Appearance'],
|
||||
['Manage sources', 'sources', 'Sources & scans']]) {
|
||||
const layout = await openOverview(page);
|
||||
await layout.getByRole('button', { name: label, exact: true }).click();
|
||||
await expect(page).toHaveURL(`${workspace}#/system/${section}`);
|
||||
await expect(page.locator('#workspace-main .section-nav [aria-pressed="true"]')).toHaveText(pressed);
|
||||
await expect(layout.locator(':scope > .summary-metrics')).toHaveCount(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('statistics dialog lists runs by category and library totals', async ({ page, request }) => {
|
||||
const statistics = await serverJson(request, '/prompt_manager/system/statistics');
|
||||
expect(statistics.by_category.length).toBeGreaterThan(1);
|
||||
const layout = await openOverview(page);
|
||||
const explore = layout.getByRole('button', { name: 'Explore', exact: true });
|
||||
await explore.click();
|
||||
const dialog = page.locator('dialog#modal[open]');
|
||||
await expect(dialog.locator('#modal-title')).toHaveText('Usage statistics');
|
||||
const tables = dialog.locator('#modal-body > table');
|
||||
await expect(tables.locator('caption')).toHaveText(['Runs by category', 'Library totals']);
|
||||
await expect(tables.first().locator('thead th')).toHaveText(['Category', 'Runs', 'Avg. rating']);
|
||||
const cells = table => table.locator('tbody tr').evaluateAll(rows => rows.map(row => [...row.cells].map(cell => cell.textContent)));
|
||||
expect(await cells(tables.first())).toEqual(statistics.by_category.map(row => [row.category ?? 'Uncategorized',
|
||||
number(row.runs), row.average_rating === null ? 'Not rated' : `${number(row.average_rating)} / 5`]));
|
||||
expect(await cells(tables.nth(1))).toEqual([['Prompts', number(statistics.totals.total_prompts)],
|
||||
['Tags', number(statistics.totals.total_tags)], ['Images', number(statistics.totals.total_images)],
|
||||
['Rated images', number(statistics.rated_images)]]);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(explore).toBeFocused();
|
||||
// A statistics read that fails says so; the Overview stays.
|
||||
await expect(await request.post('/__test__/system/fail', { data: { path: '/prompt_manager/system/statistics' } })).toBeOK();
|
||||
await explore.click();
|
||||
await expect(page.locator('#workspace-toast')).toHaveText('Statistics could not be loaded. Your saved data has not changed.');
|
||||
await expect(page.locator('dialog#modal[open]')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('load failure shows Retry and no zeros', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/fail', { data: { path: OVERVIEW_URL } })).toBeOK();
|
||||
await page.goto(`${workspace}#/system/overview`);
|
||||
const layout = layoutOf(page);
|
||||
await expect(layout.getByRole('alert')).toHaveText('The overview could not be loaded. Your saved data has not changed.');
|
||||
await expect(layout.locator('.metric')).toHaveCount(0);
|
||||
await expect(layout).not.toContainText('0');
|
||||
await layout.getByRole('button', { name: 'Retry', exact: true }).click();
|
||||
await expect(layout.locator('.summary-metrics > .metric')).toHaveCount(4);
|
||||
await expect(layout.getByRole('alert')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('out-of-order overview responses never repaint a newer section', async ({ page, request }) => {
|
||||
const held = async () => (await serverJson(request, '/__test__/hold')).held[OVERVIEW_URL] ?? 0;
|
||||
await expect(await request.post('/__test__/hold', { data: { path: OVERVIEW_URL } })).toBeOK();
|
||||
await page.goto(`${workspace}#/system/overview`);
|
||||
const layout = layoutOf(page);
|
||||
await expect(layout.getByRole('status')).toHaveText('Loading the overview…');
|
||||
await expect.poll(held).toBe(1);
|
||||
// Leave for Appearance, come back (a second read), then leave again before either answers.
|
||||
const nav = page.locator('#workspace-main .section-nav');
|
||||
await nav.getByRole('button', { name: 'Appearance', exact: true }).click();
|
||||
await nav.getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect.poll(held).toBe(2);
|
||||
// Integrations still hands off (S10), so its body is one known panel.
|
||||
await nav.getByRole('button', { name: 'Integrations', exact: true }).click();
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await expect.poll(held).toBe(0);
|
||||
// Both late answers reached a section that had left: Integrations keeps its own body.
|
||||
await expect(layout.locator(':scope > section.handoff')).toBeVisible();
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
await expect(layout.locator('.summary-metrics')).toHaveCount(0);
|
||||
expect(await layout.evaluate(node => [...node.children].map(child => child.className))).toEqual(['surface handoff', 'save-row']);
|
||||
// Back on Overview, one fresh read paints one set of metrics.
|
||||
await nav.getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect(layout.locator('.summary-metrics')).toHaveCount(1);
|
||||
await expect(layout.locator('.two-col')).toHaveCount(1);
|
||||
});
|
||||
|
||||
// System › Appearance (#249 System S3 Task 8): the browser display defaults (`gallerySettings`, preferences.mjs) and the
|
||||
// two host fields behind the one Save row, and the consumers that read them on their next visit.
|
||||
const SETTINGS_URL = '/prompt_manager/settings';
|
||||
const INSPECT_PATH = '/prompt_manager/images/inspect/';
|
||||
const statusOf = page => page.locator('#workspace-main #settings-status');
|
||||
const saveOf = page => page.locator('#workspace-main .save-row > button');
|
||||
const storedSettings = page => page.evaluate(() => JSON.parse(localStorage.getItem('gallerySettings') || 'null'));
|
||||
const imageCards = page => page.locator('#workspace-main article.media-card');
|
||||
const panel = page => page.getByRole('complementary', { name: 'Selected image details', exact: true });
|
||||
|
||||
async function openAppearance(page) {
|
||||
await page.goto(`${workspace}#/system/appearance`);
|
||||
await expect(page.locator('#workspace-main .settings-layout > section.surface > h2')).toHaveText('Browsing & display');
|
||||
// The host rows are enabled once the server settings answered.
|
||||
await expect(page.locator('#timeout')).toBeEnabled();
|
||||
}
|
||||
|
||||
async function saveAppearance(page, text) {
|
||||
await expect(saveOf(page)).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await saveOf(page).click();
|
||||
await expect(statusOf(page)).toHaveText(text);
|
||||
}
|
||||
|
||||
/** Set toggles (by row id) through their Toolkit switches. */
|
||||
async function toggle(page, values) {
|
||||
for (const [id, checked] of Object.entries(values)) await page.locator(`#${id}`).setChecked(checked, { force: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* A request barrier: one request issued and answered after everything the page has asked for so far, so a read the
|
||||
* last action started has been seen (Playwright reports a page's requests in order).
|
||||
*/
|
||||
async function requestBarrier(page) {
|
||||
expect(await page.evaluate(url => fetch(url, { cache: 'no-store' }).then(response => response.ok), SETTINGS_URL)).toBe(true);
|
||||
}
|
||||
|
||||
async function openImages(page) {
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await expect(imageCards(page).first()).toBeVisible();
|
||||
}
|
||||
|
||||
test('defaults apply to a new visit but not to the current view or a saved view', async ({ page, request }) => {
|
||||
const created = await request.post('/prompt_manager/saved-views', { data: { page: 'images', name: 'Everything',
|
||||
state: { q: '', category: '', sort: 'newest', source: 'all', folder: '', media: 'all', tags: [], favorites: false } } });
|
||||
expect(created.status()).toBe(201);
|
||||
const view = (await created.json()).view;
|
||||
try {
|
||||
await openImages(page);
|
||||
// The current view: the toolbar sets List.
|
||||
await page.getByRole('group', { name: 'Image layout' }).getByRole('button', { name: 'List', exact: true }).click();
|
||||
await expect(page.locator('.image-grid')).toHaveAttribute('data-layout', 'list');
|
||||
await openAppearance(page);
|
||||
await expect(page.locator('#default-view')).toHaveValue('grid');
|
||||
await page.locator('#default-view').selectOption('masonry');
|
||||
await page.locator('#density').selectOption('compact');
|
||||
await expect(statusOf(page)).toHaveText('Unsaved changes.');
|
||||
await saveAppearance(page, 'Saved in this browser.');
|
||||
const stored = await storedSettings(page);
|
||||
expect([stored.workspaceLayout, stored.workspaceDensity]).toEqual(['masonry', 'compact']);
|
||||
// Back in Images (the page session's last visit): the current view keeps its own layout and size.
|
||||
await page.locator('#primary-nav a[data-route="images"]').click();
|
||||
await expect(page.locator('.image-grid')).toHaveAttribute('data-layout', 'list');
|
||||
await expect(page.locator('.image-grid')).toHaveAttribute('data-density', 'comfortable');
|
||||
// The saved view opens as before.
|
||||
await page.goto(`${workspace}#/saved/${view.id}`);
|
||||
await expect(imageCards(page).first()).toBeVisible();
|
||||
await expect(page.locator('.image-grid')).toHaveAttribute('data-layout', 'grid');
|
||||
await expect(page.locator('.image-grid')).toHaveAttribute('data-density', 'comfortable');
|
||||
// A new visit starts from the defaults.
|
||||
await openImages(page);
|
||||
await page.reload();
|
||||
await expect(imageCards(page).first()).toBeVisible();
|
||||
await expect(page.locator('.image-grid')).toHaveAttribute('data-layout', 'masonry');
|
||||
await expect(page.locator('.image-grid')).toHaveAttribute('data-density', 'compact');
|
||||
await expect(page.getByRole('group', { name: 'Image layout' }).getByRole('button', { name: 'Masonry', exact: true }))
|
||||
.toHaveAttribute('aria-pressed', 'true');
|
||||
} finally {
|
||||
await request.delete(`/prompt_manager/saved-views/${view.id}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('the defaults reach the Images page only; embedded galleries keep their own presentation', async ({ page }) => {
|
||||
await openAppearance(page);
|
||||
await page.locator('#default-view').selectOption('list');
|
||||
await page.locator('#density').selectOption('compact');
|
||||
await saveAppearance(page, 'Saved in this browser.');
|
||||
// The prompt history's shared gallery has no toolbar to change them, so it opens as it always did.
|
||||
await page.goto(`${workspace}#/library/${page.manifest.prompts[0].positive_id}`);
|
||||
await expect(imageCards(page).first()).toBeVisible();
|
||||
await expect(page.locator('#workspace-main .image-grid')).toHaveAttribute('data-layout', 'grid');
|
||||
await expect(page.locator('#workspace-main .image-grid')).toHaveAttribute('data-density', 'comfortable');
|
||||
// The Images page starts from them.
|
||||
await page.locator('#primary-nav a[data-route="images"]').click();
|
||||
await expect(imageCards(page).first()).toBeVisible();
|
||||
await expect(page.locator('.image-grid')).toHaveAttribute('data-layout', 'list');
|
||||
await expect(page.locator('.image-grid')).toHaveAttribute('data-density', 'compact');
|
||||
});
|
||||
|
||||
test('metadata off shows an explicit load and loads once on request', async ({ page }) => {
|
||||
await openAppearance(page);
|
||||
await toggle(page, { 'metadata-auto': false });
|
||||
await saveAppearance(page, 'Saved in this browser.');
|
||||
const reads = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname.startsWith(INSPECT_PATH)) reads.push(item.url()); });
|
||||
await openImages(page);
|
||||
const first = imageCards(page).first();
|
||||
await first.locator('.image-card-actions button').first().click();
|
||||
await expect(panel(page).getByRole('status')).toHaveText('Metadata loads when you ask for it.');
|
||||
const load = panel(page).getByRole('button', { name: 'Load metadata', exact: true });
|
||||
await expect(load).toBeVisible();
|
||||
expect(reads).toEqual([]);
|
||||
await load.click();
|
||||
await expect(panel(page).locator('.panel-heading h2')).toBeVisible();
|
||||
expect(reads).toHaveLength(1);
|
||||
// The same file again reads nothing more (the cache is on).
|
||||
await first.locator('.image-card-actions button').first().click();
|
||||
await expect(panel(page).locator('.panel-heading h2')).toBeVisible();
|
||||
await requestBarrier(page);
|
||||
expect(reads).toHaveLength(1);
|
||||
// The viewer's details use the same inspector: another file waits there too, and loads on request.
|
||||
await imageCards(page).nth(1).locator('.image-open').click();
|
||||
const viewerDetails = page.locator('#viewer-details');
|
||||
await expect(viewerDetails.getByRole('status')).toHaveText('Metadata loads when you ask for it.');
|
||||
expect(reads).toHaveLength(1);
|
||||
await viewerDetails.getByRole('button', { name: 'Load metadata', exact: true }).click();
|
||||
await expect(viewerDetails.locator('.panel-heading h2')).toBeVisible();
|
||||
expect(reads).toHaveLength(2);
|
||||
});
|
||||
|
||||
test('cache off rereads the current selection; on reuses the one current record', async ({ page }) => {
|
||||
const reads = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname.startsWith(INSPECT_PATH)) reads.push(item.url()); });
|
||||
for (const cache of [true, false]) {
|
||||
if (!cache) {
|
||||
await openAppearance(page);
|
||||
await toggle(page, { 'metadata-cache': false });
|
||||
await saveAppearance(page, 'Saved in this browser.');
|
||||
await page.reload();
|
||||
}
|
||||
await openImages(page);
|
||||
const info = imageCards(page).first().locator('.image-card-actions button').first();
|
||||
await info.click();
|
||||
await expect(panel(page).locator('.panel-heading h2')).toBeVisible();
|
||||
await expect.poll(() => reads.length).toBeGreaterThan(0);
|
||||
await requestBarrier(page);
|
||||
reads.length = 0;
|
||||
// Inspect the selected file twice more.
|
||||
for (const _ of [1, 2]) {
|
||||
const answered = cache ? null : page.waitForResponse(response => new URL(response.url()).pathname.startsWith(INSPECT_PATH));
|
||||
await info.click();
|
||||
if (answered) await answered;
|
||||
await expect(panel(page).locator('.panel-heading h2')).toBeVisible();
|
||||
}
|
||||
await requestBarrier(page);
|
||||
expect(reads, `cache ${cache}`).toHaveLength(cache ? 0 : 2);
|
||||
}
|
||||
});
|
||||
|
||||
test('paths hidden only changes display', async ({ page }) => {
|
||||
const rows = async () => page.locator('#viewer-details details').filter({ hasText: 'File & provenance' }).locator('dt')
|
||||
.evaluateAll(terms => terms.map(term => term.textContent));
|
||||
const openViewer = async () => {
|
||||
await openImages(page);
|
||||
await imageCards(page).first().locator('.image-open').click();
|
||||
const provenance = page.locator('#viewer-details details').filter({ hasText: 'File & provenance' });
|
||||
await provenance.locator('summary').click();
|
||||
await expect(provenance.locator('dl')).toBeVisible();
|
||||
return provenance;
|
||||
};
|
||||
let provenance = await openViewer();
|
||||
expect(await rows()).toContain('Path');
|
||||
const before = { show: await provenance.getByRole('button', { name: /^Show .* in folder$/ }).count(),
|
||||
download: await provenance.getByRole('link', { name: 'Download original', exact: true }).getAttribute('href') };
|
||||
await page.keyboard.press('Escape');
|
||||
await openAppearance(page);
|
||||
await toggle(page, { paths: false });
|
||||
await saveAppearance(page, 'Saved in this browser.');
|
||||
provenance = await openViewer();
|
||||
const shown = await rows();
|
||||
expect(shown).not.toContain('Path');
|
||||
expect(shown).toEqual(expect.arrayContaining(['Source', 'Size']));
|
||||
// Everything else is as it was: the folder action and the original download stay.
|
||||
expect(await provenance.getByRole('button', { name: /^Show .* in folder$/ }).count()).toBe(before.show);
|
||||
await expect(provenance.getByRole('link', { name: 'Download original', exact: true })).toHaveAttribute('href', before.download);
|
||||
});
|
||||
|
||||
test('video defaults reach the viewer', async ({ page }) => {
|
||||
await openAppearance(page);
|
||||
await toggle(page, { 'video-auto': true, 'video-muted': false, 'video-loop': false });
|
||||
await saveAppearance(page, 'Saved in this browser.');
|
||||
expect(await page.evaluate(async () => (await import('/prompt_manager/js/workspace/media-viewer.mjs')).readViewerPreferences()))
|
||||
.toEqual({ videoAutoplay: true, videoMute: false, videoLoop: false });
|
||||
const stored = await storedSettings(page);
|
||||
expect([stored.videoAutoplay, stored.videoMute, stored.videoLoop]).toEqual([true, false, false]);
|
||||
});
|
||||
|
||||
test('host fields round-trip through /settings with revision', async ({ page, request }) => {
|
||||
const posts = [];
|
||||
page.on('request', item => { if (item.method() === 'POST' && new URL(item.url()).pathname === SETTINGS_URL) posts.push(item.postDataJSON()); });
|
||||
const before = await serverJson(request, SETTINGS_URL);
|
||||
await openAppearance(page);
|
||||
await expect(page.locator('#open-mode')).toHaveValue(before.settings.webui_display_mode);
|
||||
await expect(page.locator('#timeout')).toHaveValue(String(before.settings.result_timeout));
|
||||
// Only whole seconds from 0 to 300: anything else is highlighted and never sent.
|
||||
for (const bad of ['301', '-1', '2.5']) {
|
||||
await page.locator('#timeout').fill(bad);
|
||||
await page.locator('#timeout').press('Tab');
|
||||
await expect(page.locator('#timeout')).toHaveAttribute('aria-invalid', 'true');
|
||||
await expect(statusOf(page)).toHaveText('Some changes were not saved. Review the highlighted settings.');
|
||||
await expect(saveOf(page)).toHaveAttribute('aria-disabled', 'true');
|
||||
}
|
||||
await page.locator('#timeout').fill('0');
|
||||
await page.locator('#timeout').press('Tab');
|
||||
await expect(page.locator('#timeout')).not.toHaveAttribute('aria-invalid');
|
||||
await page.locator('#open-mode').selectOption('popup');
|
||||
await saveAppearance(page, 'Saved to the server configuration.');
|
||||
expect(posts).toEqual([{ webui_display_mode: 'popup', result_timeout: 0, expected_revision: before.revision }]);
|
||||
const after = await serverJson(request, SETTINGS_URL);
|
||||
expect([after.settings.webui_display_mode, after.settings.result_timeout]).toEqual(['popup', 0]);
|
||||
expect(after.revision).not.toBe(before.revision);
|
||||
// No browser key was written for a host field.
|
||||
expect(await storedSettings(page)).toBeNull();
|
||||
});
|
||||
|
||||
test('server settings that cannot be read say so in their rows', async ({ page }) => {
|
||||
await page.route(url => url.pathname === SETTINGS_URL, route => (route.request().method() === 'GET'
|
||||
? route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ success: false, error: 'Injected failure' }) })
|
||||
: route.continue()));
|
||||
await page.goto(`${workspace}#/system/appearance`);
|
||||
for (const id of ['open-mode', 'timeout']) {
|
||||
const row = page.locator('#workspace-main .setting-row').filter({ has: page.locator(`#${id}`) });
|
||||
await expect(row.locator('p.small')).toHaveText('Server settings could not be read.');
|
||||
await expect(page.locator(`#${id}`)).toBeDisabled();
|
||||
}
|
||||
// The browser rows are this browser's, and work as before.
|
||||
await expect(page.locator('#default-view')).toBeEnabled();
|
||||
await expect(page.locator('#workspace-main .setting-row p.small')).toHaveCount(2);
|
||||
});
|
||||
|
||||
test('stale revision shows the conflict status', async ({ page, request }) => {
|
||||
await openAppearance(page);
|
||||
// Another page saves after this one read the settings.
|
||||
const current = await serverJson(request, SETTINGS_URL);
|
||||
await expect(await request.post(SETTINGS_URL, { data: { result_timeout: 42, expected_revision: current.revision } })).toBeOK();
|
||||
await page.locator('#open-mode').selectOption('popup');
|
||||
await saveAppearance(page, 'Settings changed elsewhere. Reload this section to see them.');
|
||||
await expect(page.locator('#open-mode')).toHaveAttribute('aria-invalid', 'true');
|
||||
// The draft is kept and the section now shows the other page's value.
|
||||
await expect(page.locator('#open-mode')).toHaveValue('popup');
|
||||
await expect(page.locator('#timeout')).toHaveValue('42');
|
||||
expect((await serverJson(request, SETTINGS_URL)).settings.webui_display_mode).toBe('newtab');
|
||||
// With the reloaded revision, Save succeeds.
|
||||
await saveAppearance(page, 'Saved to the server configuration.');
|
||||
expect((await serverJson(request, SETTINGS_URL)).settings.webui_display_mode).toBe('popup');
|
||||
});
|
||||
|
||||
test('reload keeps saved values; a new tab reads them', async ({ page, context }) => {
|
||||
await openAppearance(page);
|
||||
await page.locator('#image-quality').selectOption('high');
|
||||
await toggle(page, { 'card-details': false });
|
||||
await page.locator('#open-mode').selectOption('popup');
|
||||
await saveAppearance(page, 'Saved in this browser. Saved to the server configuration.');
|
||||
await page.reload();
|
||||
await expect(page.locator('#timeout')).toBeEnabled();
|
||||
await expect(page.locator('#image-quality')).toHaveValue('high');
|
||||
await expect(page.locator('#card-details')).not.toBeChecked();
|
||||
await expect(page.locator('#open-mode')).toHaveValue('popup');
|
||||
const other = await context.newPage();
|
||||
await openAppearance(other);
|
||||
await expect(other.locator('#image-quality')).toHaveValue('high');
|
||||
await expect(other.locator('#card-details')).not.toBeChecked();
|
||||
// A change saved in one tab reaches the other's clean controls; a draft it overtakes is flagged and kept.
|
||||
await other.locator('#density').selectOption('large');
|
||||
await page.locator('#density').selectOption('compact');
|
||||
await saveAppearance(page, 'Saved in this browser.');
|
||||
await expect(other.locator('#density')).toHaveValue('large');
|
||||
await expect(other.locator('#settings-status')).toHaveText('Settings changed elsewhere. Reload this section to see them.');
|
||||
await expect(other.locator('#density')).toHaveAttribute('aria-invalid', 'true');
|
||||
await page.locator('#image-quality').selectOption('low');
|
||||
await saveAppearance(page, 'Saved in this browser.');
|
||||
await expect(other.locator('#image-quality')).toHaveValue('low');
|
||||
await other.close();
|
||||
});
|
||||
|
||||
test('a change another tab saved while System was closed is a conflict on return, never saved over silently', async ({ page, context }) => {
|
||||
await openAppearance(page);
|
||||
await page.locator('#density').selectOption('compact');
|
||||
await expect(statusOf(page)).toHaveText('Unsaved changes.');
|
||||
// Leave System for Images: the draft stays for the page session, but System no longer listens.
|
||||
await page.locator('#primary-nav a[data-route="images"]').click();
|
||||
await expect(imageCards(page).first()).toBeVisible();
|
||||
const other = await context.newPage();
|
||||
await openAppearance(other);
|
||||
await other.locator('#density').selectOption('large');
|
||||
await saveAppearance(other, 'Saved in this browser.');
|
||||
await other.close();
|
||||
// Back in System: the draft is kept and flagged, and nothing was written over the other tab's value.
|
||||
await page.evaluate(() => { location.hash = '#/system/appearance'; });
|
||||
await expect(page.locator('#workspace-main .settings-layout > section.surface > h2')).toHaveText('Browsing & display');
|
||||
await expect(statusOf(page)).toHaveText('Settings changed elsewhere. Reload this section to see them.');
|
||||
await expect(page.locator('#density')).toHaveAttribute('aria-invalid', 'true');
|
||||
await expect(page.locator('#density')).toHaveValue('compact');
|
||||
expect((await storedSettings(page)).workspaceDensity).toBe('large');
|
||||
// Saving after the conflict was shown is an explicit choice: it wins.
|
||||
await saveAppearance(page, 'Saved in this browser.');
|
||||
expect((await storedSettings(page)).workspaceDensity).toBe('compact');
|
||||
await expect(page.locator('#density')).not.toHaveAttribute('aria-invalid');
|
||||
});
|
||||
|
||||
test('a browser that refuses storage is saved for this tab only, never "Saved"', async ({ page }) => {
|
||||
await openAppearance(page);
|
||||
await page.evaluate(() => {
|
||||
Storage.prototype.setItem = function setItem() { throw new DOMException('Quota exceeded', 'QuotaExceededError'); };
|
||||
});
|
||||
await page.locator('#default-view').selectOption('list');
|
||||
await saveAppearance(page, 'Saved for this tab only; browser storage is unavailable.');
|
||||
expect(await storedSettings(page)).toBeNull();
|
||||
// This tab's next Images visit still uses it.
|
||||
await page.locator('#primary-nav a[data-route="images"]').click();
|
||||
await expect(imageCards(page).first()).toBeVisible();
|
||||
await expect(page.locator('.image-grid')).toHaveAttribute('data-layout', 'list');
|
||||
});
|
||||
|
||||
test('added rows reach their consumers', async ({ page }) => {
|
||||
await openImages(page);
|
||||
await expect(imageCards(page).first().locator('.media-info h3')).toBeVisible();
|
||||
await openAppearance(page);
|
||||
await toggle(page, { 'card-details': false, 'viewer-details': true });
|
||||
await saveAppearance(page, 'Saved in this browser.');
|
||||
await page.reload();
|
||||
await openImages(page);
|
||||
// Show image details on cards off: no title or file line; Info and Tag stay.
|
||||
const card = imageCards(page).first();
|
||||
await expect(card.locator('.media-info h3')).toBeHidden();
|
||||
await expect(card.locator('.image-card-actions button')).toHaveCount(2);
|
||||
// Collapse viewer details by default: the viewer opens with its details hidden behind one toggle.
|
||||
await card.locator('.image-open').click();
|
||||
const details = page.locator('#viewer-details');
|
||||
await expect(page.locator('#viewer-dialog')).toBeVisible();
|
||||
await expect(details).toBeHidden();
|
||||
const show = page.locator('#viewer-toolbar').getByRole('button', { name: 'Show details', exact: true });
|
||||
await expect(show).toHaveAttribute('aria-expanded', 'false');
|
||||
await show.click();
|
||||
await expect(details).toBeVisible();
|
||||
await expect(page.locator('#viewer-toolbar').getByRole('button', { name: 'Hide details', exact: true })).toHaveAttribute('aria-expanded', 'true');
|
||||
});
|
||||
|
||||
test('reset restores defaults without touching other keys', async ({ page }) => {
|
||||
await page.goto(`${workspace}#/system/appearance`);
|
||||
await page.evaluate(() => localStorage.setItem('gallerySettings', JSON.stringify({ gridColumns: 5, defaultLimit: 200,
|
||||
thumbnailPromptDismissed: 'kept', workspaceLayout: 'list', videoMute: false, showImageInfo: false })));
|
||||
await page.reload();
|
||||
await expect(page.locator('#timeout')).toBeEnabled();
|
||||
await expect(page.locator('#default-view')).toHaveValue('list');
|
||||
const reset = page.locator('#workspace-main .operation').filter({ hasText: 'Reset display defaults' }).getByRole('button', { name: 'Reset', exact: true });
|
||||
await reset.click();
|
||||
await expect(page.locator('#default-view')).toHaveValue('grid');
|
||||
await expect(page.locator('#video-muted')).toBeChecked();
|
||||
await expect(statusOf(page)).toHaveText('Unsaved changes.');
|
||||
await saveAppearance(page, 'Saved in this browser.');
|
||||
const stored = await storedSettings(page);
|
||||
expect(stored).toEqual({ gridColumns: 5, defaultLimit: 200, thumbnailPromptDismissed: 'kept', workspaceLayout: 'grid',
|
||||
videoMute: true, showImageInfo: true });
|
||||
});
|
||||
@@ -0,0 +1,323 @@
|
||||
// The Images selection bar's Add tags and Auto-tag selected (#249 PR D, plan Task D2, Mock inventory M3; D85, D90,
|
||||
// D91) against the real `image-tags` job route and the injected fake model (tests/browser/workspace_tagging_fixture.py).
|
||||
// No model is ever downloaded or loaded for real: the fake answers, and every test checks no download request was sent.
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
const images = '/prompt_manager/workspace#/images';
|
||||
const FIRST = 'tagging/dunes_00001.png', SECOND = 'tagging/dunes_00002.png', UNLINKED = 'loose/unlinked.png';
|
||||
const card = (page, path) => page.locator(`article[data-image-key="0:${path}"]`);
|
||||
const bar = page => page.locator('#workspace-main > #bulkbar > .bulkbar');
|
||||
const barButton = (page, name) => bar(page).getByRole('button', { name, exact: true });
|
||||
const modal = page => page.locator('dialog#modal[open]');
|
||||
const title = page => modal(page).locator('#modal-title');
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
const toasts = page => page.evaluate(() => window.__toasts);
|
||||
const chips = page => modal(page).locator('#tag-suggestions .chip.tag-suggestion');
|
||||
const actionButtons = page => modal(page).locator('.dialog-actions button');
|
||||
const panel = page => page.getByRole('complementary', { name: 'Selected image details', exact: true });
|
||||
const tagRow = page => panel(page).locator('.detail-inner > .tags').last();
|
||||
const NOT_DOWNLOADED = 'WD14 SwinV2 is not downloaded. Download it from the legacy Auto-tag page.';
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
// Every toast text in order: a fast job's finish toast may replace its start toast before an assertion reads it.
|
||||
await page.addInitScript(() => {
|
||||
window.__toasts = [];
|
||||
new MutationObserver(() => {
|
||||
const text = document.getElementById('workspace-toast')?.textContent ?? '';
|
||||
if (text && window.__toasts.at(-1) !== text) window.__toasts.push(text);
|
||||
}).observe(document, { subtree: true, childList: true, characterData: true });
|
||||
});
|
||||
await expect(await request.post('/__test__/tagging/reset', { data: {} })).toBeOK();
|
||||
page.taggingErrors = [];
|
||||
page.on('pageerror', error => page.taggingErrors.push(error.message));
|
||||
page.downloads = [];
|
||||
page.on('request', item => { if (/\/autotag\/download\//.test(item.url())) page.downloads.push(item.url()); });
|
||||
});
|
||||
test.afterEach(async ({ page, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.taggingErrors).toEqual([]);
|
||||
expect(page.downloads).toEqual([]);
|
||||
});
|
||||
|
||||
/** Images with `paths` checked, in that order. */
|
||||
async function select(page, ...paths) {
|
||||
await page.goto(images);
|
||||
for (const path of paths) await card(page, path).getByRole('checkbox').check();
|
||||
await expect(bar(page).locator('strong')).toHaveText(`${paths.length} selected`);
|
||||
}
|
||||
async function scopeOf(request) {
|
||||
const listing = await (await request.get('/prompt_manager/images/output?mode=workspace&limit=40')).json();
|
||||
return listing.images[0].root_scope;
|
||||
}
|
||||
/** The saved tags of `path` through the real observe route, then that capture's release. */
|
||||
async function savedTags(request, path) {
|
||||
const target = { root_index: 0, root_scope: await scopeOf(request), path };
|
||||
const observed = await (await request.post('/prompt_manager/image-annotations/observe', { data: { target } })).json();
|
||||
expect(observed.success).toBe(true);
|
||||
await request.delete(`/prompt_manager/image-annotations/capture/${observed.capture.token}`);
|
||||
return observed.tags;
|
||||
}
|
||||
/** Wait until the session's job `id` has ended; returns its summary. */
|
||||
async function ended(request, id) {
|
||||
let job;
|
||||
await expect.poll(async () => {
|
||||
const { jobs } = await (await request.get('/prompt_manager/jobs')).json();
|
||||
job = jobs.find(item => item.id === id);
|
||||
return job?.state;
|
||||
}).toBe('finished');
|
||||
return job;
|
||||
}
|
||||
const jobStart = page => page.waitForRequest(item => item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/jobs/image-tags');
|
||||
|
||||
test('selection bar shows Add tags then Auto-tag selected before Set category', async ({ page }) => {
|
||||
await select(page, FIRST);
|
||||
await expect(bar(page).locator(':scope > *')).toHaveText(['1 selected', 'Add tags', 'Auto-tag selected', 'Set category', 'Export metadata', 'Clear']);
|
||||
});
|
||||
|
||||
test('Add tags runs a job and the cards show the tags after it finishes', async ({ page, request }) => {
|
||||
await select(page, SECOND, FIRST);
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
await barButton(page, 'Add tags').click();
|
||||
await expect(title(page)).toHaveText('Tag selected images');
|
||||
const field = modal(page).getByRole('textbox', { name: 'Tags, separated by commas', exact: true });
|
||||
await expect(field).toHaveValue('');
|
||||
await expect(actionButtons(page)).toHaveText(['Cancel', 'Apply']);
|
||||
let posted = 0;
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === '/prompt_manager/jobs/image-tags') posted += 1; });
|
||||
// A field that parses to no names keeps the dialog and its text, and reports why on the field (D118).
|
||||
await field.fill(' , ,');
|
||||
await actionButtons(page).last().click();
|
||||
await expect.poll(() => field.evaluate(node => node.validationMessage)).toBe('Enter at least one tag.');
|
||||
await expect(title(page)).toHaveText('Tag selected images');
|
||||
await expect(field).toHaveValue(' , ,');
|
||||
await field.fill('sunset, Sunset , blue_hour,,');
|
||||
expect(await field.evaluate(node => node.validity.valid)).toBe(true);
|
||||
const started = jobStart(page);
|
||||
const answer = page.waitForResponse(item => new URL(item.url()).pathname === '/prompt_manager/jobs/image-tags');
|
||||
await actionButtons(page).last().click();
|
||||
const scope = await scopeOf(request);
|
||||
// Listing order (the newer FIRST before SECOND), the reveal identity, the dialog's dedupe.
|
||||
expect((await started).postDataJSON()).toEqual({ mode: 'add', tags: ['sunset', 'blue_hour'], targets: [
|
||||
{ root_index: 0, root_scope: scope, path: FIRST }, { root_index: 0, root_scope: scope, path: SECOND }] });
|
||||
const { job } = await (await answer).json();
|
||||
// The refused Apply sent nothing: only the valid one was posted.
|
||||
expect(posted).toBe(1);
|
||||
await expect.poll(() => toasts(page)).toContain('Tagging started.');
|
||||
expect(await toasts(page)).not.toContain('Enter at least one tag.');
|
||||
await expect(modal(page)).toHaveCount(0);
|
||||
await expect(bar(page).locator('strong')).toHaveText('2 selected');
|
||||
expect((await ended(request, job.id)).counters).toEqual({ tagged: 2, unchanged: 0, skipped: 0, errors: 0 });
|
||||
// The finished job's files are read again: the selected card's panel shows its new image tags.
|
||||
await expect(tagRow(page).locator('button.chip')).toHaveText(['sunset', 'blue_hour']);
|
||||
expect(await savedTags(request, SECOND)).toEqual(['sunset', 'blue_hour']);
|
||||
expect(await savedTags(request, UNLINKED)).toEqual([]);
|
||||
});
|
||||
|
||||
test('Auto-tag selected with Apply automatically runs a job with the chosen model, thresholds and suggestion count', async ({ page, request }) => {
|
||||
await select(page, FIRST, UNLINKED);
|
||||
await barButton(page, 'Auto-tag selected').click();
|
||||
await expect(title(page)).toHaveText('Auto-tag selected images');
|
||||
await expect(modal(page)).toHaveClass(/(^| )tagging-modal( |$)/);
|
||||
const body = modal(page).locator('#modal-body');
|
||||
expect(await body.evaluate(node => [...node.children].map(child => child.className))).toEqual(['tag-model-panel', 'field', 'small', 'dialog-actions']);
|
||||
await expect(body.locator('.tag-model-panel h3')).toHaveText('Auto-tag 2 images');
|
||||
// No preview, manual field, suggestions or Generate: only the model panel's fields.
|
||||
await expect(body.getByRole('button', { name: 'Generate suggestions', exact: true })).toHaveCount(0);
|
||||
await expect(body.locator('#manual-image-tags, .tag-suggestion-panel, .tag-image-preview')).toHaveCount(0);
|
||||
const group = body.getByRole('radiogroup', { name: 'How to apply', exact: true });
|
||||
await expect(group.getByRole('radio', { name: 'Apply automatically', exact: true })).toBeChecked();
|
||||
await expect(group.getByRole('radio', { name: 'Review each image', exact: true })).not.toBeChecked();
|
||||
await expect(actionButtons(page)).toHaveText(['Cancel', 'Start auto-tagging']);
|
||||
await body.getByRole('combobox', { name: 'Tagging model', exact: true }).selectOption('wd14-vit');
|
||||
await body.getByRole('spinbutton', { name: 'General confidence', exact: true }).fill('0.5');
|
||||
await body.getByRole('spinbutton', { name: 'Character confidence', exact: true }).fill('0.9');
|
||||
const limit = body.getByRole('spinbutton', { name: 'Suggestions', exact: true });
|
||||
// An invalid count reports its validity and sends nothing.
|
||||
await limit.fill('0');
|
||||
let posted = 0;
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === '/prompt_manager/jobs/image-tags') posted += 1; });
|
||||
await actionButtons(page).last().click();
|
||||
expect(await limit.evaluate(node => node.validity.valid)).toBe(false);
|
||||
await expect(title(page)).toHaveText('Auto-tag selected images');
|
||||
await limit.fill('2');
|
||||
const started = jobStart(page);
|
||||
const answer = page.waitForResponse(item => new URL(item.url()).pathname === '/prompt_manager/jobs/image-tags');
|
||||
await actionButtons(page).last().click();
|
||||
const scope = await scopeOf(request);
|
||||
expect((await started).postDataJSON()).toEqual({ mode: 'auto', model_type: 'wd14-vit',
|
||||
options: { general_threshold: 0.5, character_threshold: 0.9, suggestion_limit: 2 },
|
||||
targets: [{ root_index: 0, root_scope: scope, path: FIRST }, { root_index: 0, root_scope: scope, path: UNLINKED }] });
|
||||
const { job } = await (await answer).json();
|
||||
// The invalid press sent nothing: only this Start was posted.
|
||||
expect(posted).toBe(1);
|
||||
await expect.poll(() => toasts(page)).toContain('Auto-tagging started.');
|
||||
await expect(modal(page)).toHaveCount(0);
|
||||
await expect(bar(page).locator('strong')).toHaveText('2 selected');
|
||||
expect((await ended(request, job.id)).counters).toEqual({ tagged: 2, unchanged: 0, skipped: 0, errors: 0 });
|
||||
expect(await savedTags(request, FIRST)).toEqual(['sandstone', 'desert']);
|
||||
expect(await savedTags(request, UNLINKED)).toEqual(['sandstone', 'desert']);
|
||||
expect(await savedTags(request, SECOND)).toEqual([]);
|
||||
});
|
||||
|
||||
test('a model that is not downloaded keeps Start disabled and says why', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/tagging/models', { data: { unavailable: ['wd14-swinv2'] } })).toBeOK();
|
||||
await select(page, FIRST);
|
||||
await barButton(page, 'Auto-tag selected').click();
|
||||
const status = modal(page).locator('#tag-generation-status');
|
||||
await expect(status).toHaveAttribute('role', 'status');
|
||||
await expect(status).toHaveText(NOT_DOWNLOADED);
|
||||
const start = actionButtons(page).last();
|
||||
await expect(start).toHaveAttribute('aria-disabled', 'true');
|
||||
let posted = 0;
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === '/prompt_manager/jobs/image-tags') posted += 1; });
|
||||
// aria-disabled: a press does nothing.
|
||||
await start.dispatchEvent('click');
|
||||
await expect(title(page)).toHaveText('Auto-tag selected images');
|
||||
await modal(page).getByRole('combobox', { name: 'Tagging model', exact: true }).selectOption('wd14-vit');
|
||||
await expect(status).toHaveText('');
|
||||
await expect(start).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await modal(page).getByRole('combobox', { name: 'Tagging model', exact: true }).selectOption('wd14-swinv2');
|
||||
await expect(start).toHaveAttribute('aria-disabled', 'true');
|
||||
await actionButtons(page).first().click();
|
||||
await expect(modal(page)).toHaveCount(0);
|
||||
await expect(barButton(page, 'Auto-tag selected')).toBeFocused();
|
||||
expect(posted).toBe(0);
|
||||
});
|
||||
|
||||
test('Suggestions field prefills from the setting and caps chips at the entered count', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/tagging/defaults', { data: { general: 0.5, character: 0.9, limit: 7 } })).toBeOK();
|
||||
await select(page, FIRST);
|
||||
await barButton(page, 'Auto-tag selected').click();
|
||||
const limit = modal(page).getByRole('spinbutton', { name: 'Suggestions', exact: true });
|
||||
await expect(limit).toHaveValue('7');
|
||||
await expect(modal(page).getByRole('spinbutton', { name: 'General confidence', exact: true })).toHaveValue('0.5');
|
||||
await expect(modal(page).getByRole('spinbutton', { name: 'Character confidence', exact: true })).toHaveValue('0.9');
|
||||
await modal(page).getByRole('radio', { name: 'Review each image', exact: true }).check();
|
||||
await expect(actionButtons(page)).toHaveText(['Cancel', 'Start review']);
|
||||
await limit.fill('3');
|
||||
const suggest = page.waitForRequest(item => item.url().endsWith('/image-annotations/suggest'));
|
||||
await actionButtons(page).last().click();
|
||||
await expect(title(page)).toHaveText('Tag image · Image 1 of 1');
|
||||
expect((await suggest).postDataJSON().options).toEqual({ general_threshold: 0.5, character_threshold: 0.9, suggestion_limit: 3 });
|
||||
await expect(chips(page)).toHaveCount(3);
|
||||
await expect(modal(page).getByRole('spinbutton', { name: 'Suggestions', exact: true })).toHaveValue('3');
|
||||
await expect(actionButtons(page)).toHaveText(['Stop reviewing', 'Skip', 'Approve and finish']);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(modal(page)).toHaveCount(0);
|
||||
await expect(toast(page)).toHaveText('0 of 1 images tagged.');
|
||||
expect(await savedTags(request, FIRST)).toEqual([]);
|
||||
});
|
||||
|
||||
test('Review each image walks the selection one dialog at a time and only approved images gain tags', async ({ page, request }) => {
|
||||
await select(page, FIRST, SECOND, UNLINKED);
|
||||
// The second image's first Generate finds the model busy (a legacy batch): the dialog waits for Retry or Skip.
|
||||
let suggests = 0;
|
||||
await page.route('**/prompt_manager/image-annotations/suggest', async route => {
|
||||
suggests += 1;
|
||||
if (suggests === 2) {
|
||||
await route.fulfill({ status: 409, json: { success: false, code: 'model-busy', error: 'The tagging model is busy. Try again when it finishes.' } });
|
||||
return;
|
||||
}
|
||||
await route.continue();
|
||||
});
|
||||
const puts = [];
|
||||
page.on('request', item => { if (item.method() === 'PUT' && item.url().endsWith('/image-annotations/tags')) puts.push(item.postDataJSON()); });
|
||||
await barButton(page, 'Auto-tag selected').click();
|
||||
await modal(page).getByRole('radio', { name: 'Review each image', exact: true }).check();
|
||||
await modal(page).getByRole('spinbutton', { name: 'Suggestions', exact: true }).fill('2');
|
||||
await actionButtons(page).last().click();
|
||||
await expect(title(page)).toHaveText('Tag image · Image 1 of 3');
|
||||
await expect(modal(page).getByRole('combobox', { name: 'Tagging model', exact: true })).toHaveValue('wd14-swinv2');
|
||||
await expect(chips(page)).toHaveCount(2);
|
||||
await expect(actionButtons(page)).toHaveText(['Stop reviewing', 'Skip', 'Approve and next']);
|
||||
await expect(actionButtons(page).last()).toHaveClass(/ca-primary/);
|
||||
// Approve saves the dialog's manual field plus its remaining chips.
|
||||
await modal(page).getByRole('textbox', { name: 'Image tags · edit manually', exact: true }).fill('mine');
|
||||
await modal(page).getByRole('button', { name: 'Remove desert', exact: true }).click();
|
||||
await actionButtons(page).last().click();
|
||||
await expect(title(page)).toHaveText('Tag image · Image 2 of 3');
|
||||
await expect(modal(page).locator('#tag-generation-status')).toHaveText('The tagging model is busy. Try again when it finishes.');
|
||||
// No suggestions on screen: the primary would save the manual field only (D90).
|
||||
await expect(actionButtons(page)).toHaveText(['Retry', 'Stop reviewing', 'Skip', 'Save and next']);
|
||||
await actionButtons(page).nth(2).click();
|
||||
await expect(title(page)).toHaveText('Tag image · Image 3 of 3');
|
||||
await expect(chips(page)).toHaveCount(2);
|
||||
await expect(actionButtons(page)).toHaveText(['Stop reviewing', 'Skip', 'Approve and finish']);
|
||||
await actionButtons(page).last().click();
|
||||
await expect(modal(page)).toHaveCount(0);
|
||||
await expect(toast(page)).toHaveText('2 of 3 images tagged.');
|
||||
await expect(barButton(page, 'Auto-tag selected')).toBeFocused();
|
||||
await expect(bar(page).locator('strong')).toHaveText('3 selected');
|
||||
expect(puts.map(body => [body.target.path, body.tags])).toEqual([[FIRST, ['mine', 'sandstone']], [UNLINKED, ['sandstone', 'desert']]]);
|
||||
expect(await savedTags(request, FIRST)).toEqual(['mine', 'sandstone']);
|
||||
expect(await savedTags(request, SECOND)).toEqual([]);
|
||||
expect(await savedTags(request, UNLINKED)).toEqual(['sandstone', 'desert']);
|
||||
// The reviewed file's panel shows its saved tags without reselecting.
|
||||
await expect(tagRow(page).locator('button.chip')).toHaveText(['mine', 'sandstone']);
|
||||
await expect.poll(async () => (await (await request.get('/__test__/tagging/captures')).json()).captures).toBe(0);
|
||||
});
|
||||
|
||||
test('Review holds Stop reviewing, Escape and × while a save is in flight, and names a manual-only save', async ({ page, request }) => {
|
||||
await select(page, FIRST, SECOND);
|
||||
let release;
|
||||
const held = new Promise(resolve => { release = resolve; });
|
||||
await page.route('**/prompt_manager/image-annotations/tags', async route => {
|
||||
if (route.request().method() === 'PUT') await held;
|
||||
await route.continue();
|
||||
});
|
||||
await barButton(page, 'Auto-tag selected').click();
|
||||
await modal(page).getByRole('radio', { name: 'Review each image', exact: true }).check();
|
||||
await actionButtons(page).last().click();
|
||||
await expect(title(page)).toHaveText('Tag image · Image 1 of 2');
|
||||
await expect(chips(page)).not.toHaveCount(0);
|
||||
await expect(actionButtons(page)).toHaveText(['Stop reviewing', 'Skip', 'Approve and next']);
|
||||
// D90: a model change clears the suggestions, so the primary saves the manual field only and says so.
|
||||
await modal(page).getByRole('combobox', { name: 'Tagging model', exact: true }).selectOption('wd14-vit');
|
||||
await expect(chips(page)).toHaveCount(0);
|
||||
await expect(actionButtons(page)).toHaveText(['Stop reviewing', 'Skip', 'Save and next']);
|
||||
await modal(page).getByRole('textbox', { name: 'Image tags · edit manually', exact: true }).fill('mine');
|
||||
const put = page.waitForRequest(item => item.method() === 'PUT' && item.url().endsWith('/image-annotations/tags'));
|
||||
await actionButtons(page).last().click();
|
||||
await put;
|
||||
const stop = actionButtons(page).first();
|
||||
await expect(stop).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(actionButtons(page).nth(1)).toHaveAttribute('aria-disabled', 'true');
|
||||
// None of these ends the session while the save is in flight; the count then includes it.
|
||||
await page.keyboard.press('Escape');
|
||||
await modal(page).locator('#modal-close').click();
|
||||
await stop.dispatchEvent('click'); // aria-disabled: Playwright's click would wait for it
|
||||
release();
|
||||
await expect(title(page)).toHaveText('Tag image · Image 2 of 2');
|
||||
await expect(chips(page)).not.toHaveCount(0);
|
||||
await expect(actionButtons(page)).toHaveText(['Stop reviewing', 'Skip', 'Approve and finish']);
|
||||
await expect(actionButtons(page).first()).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await actionButtons(page).first().click();
|
||||
await expect(modal(page)).toHaveCount(0);
|
||||
await expect(toast(page)).toHaveText('1 of 2 images tagged.');
|
||||
expect(await savedTags(request, FIRST)).toEqual(['mine']);
|
||||
expect(await savedTags(request, SECOND)).toEqual([]);
|
||||
});
|
||||
|
||||
test('a running scan makes the start report busy', async ({ page, request }) => {
|
||||
const scan = await (await request.post('/__test__/jobs/start', { data: { kind: 'scan', title: 'Scan for metadata' } })).json();
|
||||
try {
|
||||
await select(page, FIRST, SECOND);
|
||||
await barButton(page, 'Add tags').click();
|
||||
await modal(page).getByRole('textbox', { name: 'Tags, separated by commas', exact: true }).fill('sunset');
|
||||
await actionButtons(page).last().click();
|
||||
await expect(toast(page)).toHaveText('Another job is running: Scan for metadata.');
|
||||
await expect(toast(page).getByRole('link', { name: 'Scan for metadata', exact: true })).toHaveAttribute('href', `#/activity/${scan.job.id}`);
|
||||
await expect(bar(page).locator('strong')).toHaveText('2 selected');
|
||||
// Auto-tag is refused the same way.
|
||||
await barButton(page, 'Auto-tag selected').click();
|
||||
await expect(actionButtons(page).last()).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await actionButtons(page).last().click();
|
||||
await expect(toast(page)).toHaveText('Another job is running: Scan for metadata.');
|
||||
await expect(bar(page).locator('strong')).toHaveText('2 selected');
|
||||
expect(await savedTags(request, FIRST)).toEqual([]);
|
||||
} finally {
|
||||
await request.post(`/__test__/jobs/${scan.job.id}/finish`, { data: { state: 'finished' } });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,71 @@
|
||||
// The Tag image dialog's capture lifetime when the page or the dialog goes away mid-flight (#249 PR C final review I4,
|
||||
// I5), run on all four projects: leaving the page with the dialog open releases the capture (a `keepalive` DELETE sent
|
||||
// on `pagehide`, which also closes the dialog), and closing the dialog while its save is in flight keeps the capture
|
||||
// until the save settles, then still reports the save. The fixture is tests/browser/workspace_tagging_fixture.py.
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
const images = '/prompt_manager/workspace#/images';
|
||||
const FIRST = 'tagging/dunes_00001.png';
|
||||
const TAGS_URL = '**/prompt_manager/image-annotations/tags';
|
||||
const dialog = page => page.getByRole('dialog', { name: 'Tag image', exact: true });
|
||||
const manual = page => dialog(page).getByRole('textbox', { name: 'Image tags · edit manually', exact: true });
|
||||
const tagButton = page => page.locator(`article[data-image-key="0:${FIRST}"]`).getByRole('button', { name: 'Tag or auto-tag dunes_00001.png', exact: true });
|
||||
const openCaptures = async request => (await (await request.get('/__test__/tagging/captures')).json()).captures;
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await expect(await request.post('/__test__/tagging/reset', { data: {} })).toBeOK();
|
||||
page.problems = [];
|
||||
page.on('pageerror', error => page.problems.push(`pageerror: ${error.message}`));
|
||||
page.on('console', message => { if (message.type() === 'error') page.problems.push(`console: ${message.text()}`); });
|
||||
});
|
||||
test.afterEach(async ({ page, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.problems).toEqual([]);
|
||||
});
|
||||
|
||||
async function openTag(page) {
|
||||
await page.goto(images);
|
||||
const observed = page.waitForResponse(item => item.url().endsWith('/image-annotations/observe'));
|
||||
await tagButton(page).click();
|
||||
await expect(dialog(page)).toBeVisible();
|
||||
expect((await observed).status()).toBe(200);
|
||||
await expect(manual(page)).toBeEditable();
|
||||
}
|
||||
|
||||
test('leaving the page with the dialog open releases its capture without a console error (I4)', async ({ page, request }) => {
|
||||
await openTag(page);
|
||||
expect(await openCaptures(request)).toBe(1);
|
||||
// Another document: the old one's pagehide closes the dialog and sends the release with keepalive.
|
||||
await page.goto('/prompt_manager/workspace?left=1#/tags');
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Tags & categories');
|
||||
await expect.poll(() => openCaptures(request)).toBe(0);
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('closing the dialog during a save releases the capture after the save and still reports it (I5)', async ({ page, request }) => {
|
||||
await openTag(page);
|
||||
await manual(page).fill('saved_after_close');
|
||||
const order = [];
|
||||
page.on('response', item => { if (item.url().endsWith('/image-annotations/tags')) order.push(`PUT ${item.status()}`); });
|
||||
page.on('request', item => { if (item.url().includes('/image-annotations/capture/') && item.method() === 'DELETE') order.push('DELETE'); });
|
||||
let release;
|
||||
const held = new Promise(resolve => { release = resolve; });
|
||||
const sent = new Promise(resolve => {
|
||||
page.route(TAGS_URL, async route => { resolve(); await held; await route.continue(); });
|
||||
});
|
||||
await dialog(page).getByRole('button', { name: 'Save manual tags', exact: true }).click();
|
||||
await sent;
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(await openCaptures(request)).toBe(1);
|
||||
expect(order).toEqual([]);
|
||||
release();
|
||||
await expect(page.locator('#workspace-toast')).toHaveText('1 tags saved for dunes_00001.png.');
|
||||
await expect.poll(() => order).toEqual(['PUT 200', 'DELETE']);
|
||||
await expect.poll(() => openCaptures(request)).toBe(0);
|
||||
// The page followed the late save: the selected file's details show the new image tag.
|
||||
await expect(page.getByRole('complementary', { name: 'Selected image details', exact: true }).getByText('saved_after_close', { exact: true })).toBeVisible();
|
||||
await page.unroute(TAGS_URL);
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user