Compare commits

..
Author SHA1 Message Date
Vito Sansevero 8f889f92bd Merge remote-tracking branch 'origin/main' into fix/376-exit-cancel
# Conflicts:
#	utils/parallel.py
2026-10-28 02:24:48 -07:00
Vito Sansevero a8ab71eb1d fix(exit): abort Xet downloads and stop LoRA staging at exit (#376)
Review follow-ups: Xet transfers report progress from native code and ignore
the chunk checkpoint, so the exit hook calls huggingface_hub's own
abort_xet_session when it is loaded; an aborted download logs as stopped.
LoRA import staging reads the exit signal itself, so the legacy LoRA scan
stops too. The test's fake encoder outlives the ceiling by less, and
TESTING.md states the download limits of older huggingface_hub.
2026-10-27 19:32:24 -07:00
Vito Sansevero 2c1e401d86 fix(exit): stop running jobs before Python joins the executor threads (#376)
ComfyUI returns from main.py after Ctrl+C without closing its event loop,
and Python joins every executor thread before any atexit callback runs, so
a scan batch, thumbnail batch or model download in flight kept ComfyUI
alive until it ended (on Windows each ffmpeg up to its 30 s timeout).

The package now registers a threading._register_atexit hook (after
concurrent.futures', so it runs before that join) that sets an exit signal
and kills the running encoder processes. Job.cancelled follows the signal,
so every check a job already makes between files stops it at exit;
map_parallel starts no further item, the scan walk stops between folders and
a tagging model download stops before its next chunk. Without the internal
hook exit behaves as before.
2026-10-27 19:23:07 -07:00
153 changed files with 3317 additions and 14056 deletions
+8 -1
View File
@@ -3,10 +3,17 @@
"strict": false,
"contexts": [
"py3.12 on ubuntu-latest",
"py3.13 on ubuntu-latest",
"py3.14 on ubuntu-latest",
"py3.12 on windows-latest",
"py3.13 on windows-latest",
"py3.14 on windows-latest",
"py3.12 on macos-latest",
"py3.13 on macos-latest",
"py3.14 on macos-latest",
"coverage gate (85%)",
"black and flake8",
"frontend modules",
"all tests",
"viewerjs",
"real ComfyUI compatibility",
"bandit (high severity)"
-55
View File
@@ -1,55 +0,0 @@
name: Windows and macOS
permissions:
contents: read
# Non-blocking portability runs (#420): never on pull requests or in the merge queue, and no context
# here is a required check. A red run is fixed at its cause before the workspace cutover (#334).
on:
push:
branches: [main]
schedule:
- cron: "17 3 * * *"
workflow_dispatch:
# Every run gets its own group (run_id), so no main commit's run is ever cancelled or dropped as pending.
concurrency:
group: ${{ github.workflow }}-${{ github.run_id }}
cancel-in-progress: false
env:
PYTHONFAULTHANDLER: "1"
# Time limit rule as in test.yml: ceil(1.5 x the slowest of three recent green runs on main) + 3
# minutes, never under 5. Measured 2026-10-26 in test.yml on runs 40257217681 (3be0ada),
# 40240706568 (fb9134f), 40228167328 (f9835aa), before these legs moved here in #420:
# slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
# (macos 6m14s-8m30s, windows 11m42s-15m54s)
jobs:
test:
name: py${{ matrix.python-version }} on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
timeout-minutes: 27
strategy:
fail-fast: false
matrix:
os: [windows-latest, macos-latest]
python-version: ["3.12", "3.13", "3.14"]
steps:
- uses: actions/checkout@v7
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v7
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: requirements-dev.txt
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements-dev.txt
- name: Run tests
run: python -m tests.run_suite -v
+2 -42
View File
@@ -28,18 +28,10 @@ env:
# py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
# (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s)
# 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs.
# Windows and macOS moved to portability.yml (#420), which keeps this 27-minute limit; this
# Linux-only matrix keeps it too until it is re-measured.
# real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min
# coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min
# perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s,
# frontend modules 23s / 19s / 29s -> 5 min (the floor)
# all tests: a result check without setup, 5 min (the floor)
#
# Pull requests run a fast subset (#420): Python 3.12 on ubuntu-latest, coverage, lint and frontend
# modules. The merge queue (merge_group), pushes to main and manual runs add Python 3.13 and 3.14 on
# Linux and the real ComfyUI job. "all tests" is the required gate that holds the queue to that set.
# Windows and macOS run in portability.yml (push to main, nightly, on demand) and never gate a merge.
jobs:
test:
name: py${{ matrix.python-version }} on ${{ matrix.os }}
@@ -48,8 +40,8 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
python-version: ${{ github.event_name == 'pull_request' && fromJSON('["3.12"]') || fromJSON('["3.12", "3.13", "3.14"]') }}
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ["3.12", "3.13", "3.14"]
steps:
- uses: actions/checkout@v7
@@ -71,9 +63,6 @@ jobs:
comfyui:
name: real ComfyUI compatibility
# Merge queue, main and manual runs only. A job skipped by its condition reports Success, so this
# required context never blocks a pull request; "all tests" fails if it is skipped anywhere else.
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
# Chromium and its system libraries come from the pinned Playwright image (tag = @playwright/test in
# tools/vendor/package-lock.json; tests/vendor/browser-image.test.mjs checks it). The runner install
@@ -264,32 +253,3 @@ jobs:
- name: Run JS tests
run: node --test tests/js/*.test.js
# The required context "all tests". A dropped matrix leg never reports a check at all, so the eight
# legs that pull requests skip cannot be required by name; this gate requires them instead. Every
# job must succeed; real ComfyUI must be skipped on a pull request and succeed on every other event.
all-tests:
name: all tests
needs: [test, comfyui, coverage, lint, js-test]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Require every test job for this event
env:
EVENT_NAME: ${{ github.event_name }}
TEST_RESULT: ${{ needs.test.result }}
COMFYUI_RESULT: ${{ needs.comfyui.result }}
COVERAGE_RESULT: ${{ needs.coverage.result }}
LINT_RESULT: ${{ needs.lint.result }}
JS_RESULT: ${{ needs.js-test.result }}
run: |
queue_only=success
if [ "$EVENT_NAME" = pull_request ]; then queue_only=skipped; fi
echo "event: $EVENT_NAME (real ComfyUI must be $queue_only)"
echo "test matrix: $TEST_RESULT"
echo "comfyui: $COMFYUI_RESULT"
echo "coverage: $COVERAGE_RESULT"
echo "lint: $LINT_RESULT"
echo "js-test: $JS_RESULT"
test "$TEST_RESULT" = success && test "$COVERAGE_RESULT" = success && test "$LINT_RESULT" = success && test "$JS_RESULT" = success && test "$COMFYUI_RESULT" = "$queue_only"
+3 -23
View File
@@ -27,15 +27,6 @@ concurrency:
# coverage shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox 7m57s/5m21s (tests
# 7m08s/4m34s), webkit 8m16s/5m03s (tests 7m26s/3m58s); a cache miss adds up to 5m to npm ci, which
# the 20-minute limits absorb.
# Hangs (#425): a job killed at its limit keeps no log and skips its upload steps, so each browser
# test step has its own timeout-minutes (18) below the job's 20. At the step limit, SIGTERM makes
# tools/vendor/running-tests-reporter.mjs print the tests still running and hand over to Playwright's
# interrupt, which writes the blob report; the always() uploads then run.
# Measured on main (runs 40725289094, 40719375442, 40700439399): setup before the test step <= 52s,
# uploads <= 35s; test steps up to 14m42s (webkit shard 1/2) and 13m05s for coverage (shard 1/3:
# 12m53s, node units and the viewer suite before 11.8m of workflows), so coverage is the tight one
# in its 18 minutes. This holds only when the npm cache hits: a cold cache adds up to 5m to npm ci,
# and then the job limit can still end the job before the step limit does.
jobs:
frontend-coverage:
name: frontend coverage (shard ${{ matrix.shard }}/3)
@@ -88,7 +79,6 @@ jobs:
- if: matrix.shard == 3
run: npm run --prefix tools/vendor test
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
timeout-minutes: 18
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
- name: Upload raw coverage
uses: actions/upload-artifact@v7
@@ -97,7 +87,6 @@ jobs:
path: coverage/js/raw/
if-no-files-found: error
retention-days: 3
# Traces and the blob report (test-results/blob/) of a failed, timed-out or cancelled step.
- name: Upload Chromium and touch traces
if: always()
uses: actions/upload-artifact@v7
@@ -176,8 +165,6 @@ jobs:
browser-workflows:
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
# Merge queue, main and manual runs only (#420); pull requests run the Chromium/touch coverage shards.
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
@@ -222,9 +209,7 @@ jobs:
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
- run: npm ci --prefix tools/vendor --ignore-scripts
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
timeout-minutes: 18
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
# Traces and the blob report (test-results/blob/) of a failed, timed-out or cancelled step.
- name: Upload browser traces
if: always()
uses: actions/upload-artifact@v7
@@ -234,9 +219,8 @@ jobs:
retention-days: 14
# Branch protection requires the exact context "viewerjs". This job keeps that
# name and fails unless every coverage shard and the coverage merge succeeded and
# the browser shards succeeded, or, on a pull request only, were skipped (#420).
# Any failure, cancellation or other skip fails it.
# name and fails unless every frontend job above succeeded (every shard, the
# coverage merge and every browser shard), including when one was cancelled or skipped.
viewerjs:
needs: [frontend-coverage, frontend-coverage-merge, browser-workflows]
if: always()
@@ -245,15 +229,11 @@ jobs:
steps:
- name: Require every frontend job to succeed
env:
EVENT_NAME: ${{ github.event_name }}
COVERAGE_RESULT: ${{ needs.frontend-coverage.result }}
MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }}
WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }}
run: |
queue_only=success
if [ "$EVENT_NAME" = pull_request ]; then queue_only=skipped; fi
echo "event: $EVENT_NAME (browser-workflows must be $queue_only)"
echo "frontend-coverage: $COVERAGE_RESULT"
echo "frontend-coverage-merge: $MERGE_RESULT"
echo "browser-workflows: $WORKFLOWS_RESULT"
test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = "$queue_only"
test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
+38 -35
View File
@@ -56,7 +56,7 @@ Both nodes include the complete PromptManager feature set:
- **📊 Analytics**: Track prompt usage patterns and effectiveness over time
- **🔬 Workflow Analysis**: Extract and analyze ComfyUI workflow data from PNG images
- **📋 Metadata Viewer**: Standalone tool for analyzing ComfyUI-generated images
- **🛠️ System Management**: Built-in diagnostics, backup/restore, and maintenance tools in System
- **🛠️ System Management**: Built-in diagnostics, backup/restore, and maintenance tools
- **🏷️ AI AutoTag**: Automatically tag your image collection using WD14 or JoyCaption vision models
- **🔗 LoRA Manager Integration**: Import LoRA metadata and preview images from [ComfyUI-Lora-Manager](https://github.com/willmiao/ComfyUI-Lora-Manager)
@@ -206,11 +206,14 @@ match Classic and Focused.
- **Responsive Design**: Works on desktop, tablet, and mobile devices
- **Real-time Search**: Instant search results as you type
- **Bulk Operations**: Edit multiple prompts simultaneously with batch tools
- **Settings and System tools live in System**: the dashboard's **Settings** opens System › Overview, **Maintenance** opens System › Maintenance, **Backup DB** and **Restore DB** open System › Backup & export, and **Diagnostics** and **View Logs** open System › Diagnostics & logs, each in a new tab. System reviews every destructive operation before it runs it as an Activity job.
- **Page options**: the list header's **Page options** keeps the prompt list's Infinite Scrolling default
- **Dashboard tools**:
- **Settings Panel**: Configure behavior and display options
- **System Management**: Complete toolset for database operations:
- **🔍 Metadata Viewer**: Launch standalone PNG analysis tool
- **📸 Scan Images**: Bulk scan output directories for ComfyUI images (with a quick backup first)
- **📸 Scan Images**: Bulk scan output directories for ComfyUI images
- **💾 Backup/Restore**: Database backup and recovery operations
- **🔧 Maintenance**: Database optimization and cleanup tools including orphaned prompt removal
- **📋 Logs**: Comprehensive logging system with download capability
- **🔍 Diagnostics**: System health checks and troubleshooting
- **Export Tools**: Download prompts and metadata in various formats
- **Progress Tracking**: Real-time progress indicators for long operations
- **Error Handling**: Graceful error display with recovery suggestions
@@ -312,12 +315,6 @@ Both nodes support the same metadata fields:
Both nodes will automatically save prompts to the database and link any generated images to the prompt.
- **Save to library** (`save_to_library`, on by default; "Save to library" in the Focused, Split and Pair editors):
turn it off while iterating and PromptManager ignores that node's runs. The prompt is not saved, the run is not
counted and its images are not linked (a scan skips them too), while the node's output is unchanged. Workflows
saved before this switch existed load with it on. If any PromptManager node feeding a sampler's positive input
has it off, none of that run's images are linked, because the image cannot show which sampler produced it.
### 🖼️ Using the Output-Wide Gallery
The comprehensive gallery system provides complete access to all your ComfyUI output:
@@ -336,11 +333,14 @@ The comprehensive gallery system provides complete access to all your ComfyUI ou
#### **⚙️ Gallery Settings**
The gallery's **Settings** opens System › Appearance in a new tab: image quality, details on cards, collapsed viewer details, video autoplay/mute/loop, metadata loading and file paths are set there and apply to this page too.
Configure your viewing experience in the settings panel:
The gallery's **Page options** keep the page's own settings: the thumbnail check at startup, lazy loading, items per page, grid columns, infinite scrolling, debug mode and API timeout. Reset puts back only these.
Thumbnail generation, the server thumbnail and browser cache clears, refreshing the file listing and finding duplicate images are in System › Sources & scans and System › Maintenance (Page options links both); each runs as an Activity job you can follow after closing the page.
- **Display Settings**: Default view, items per page, grid columns, media info, infinite scrolling
- **Performance Settings**: Image quality, lazy loading
- **Video Settings**: Autoplay, mute by default, loop videos
- **Thumbnail Management**: Generate optimized thumbnails for faster loading (runs on the configured worker threads; a run that stops at the per-run cap tells you how many remain)
- **Find Duplicate Images**: Hashes every file on the worker threads with a progress bar; close the window and it keeps running, results wait in a pill
- **Rescan Output Folder**: Drops the server-side listing cache and rewalks the folders
#### **🖼️ Image Viewing Experience**
@@ -398,7 +398,7 @@ Import existing ComfyUI images into your database:
- Any errors or issues encountered
5. **Access imported data** through the normal gallery interface
The scan reads PNG text chunks and A1111-style parameters, and, when `ffprobe` is installed, the prompt ComfyUI's video savers embed in MP4/MOV/WebM files. Files are read on the number of **worker threads** set in System › Sources & scans (default: half your cores), which makes a large library several times faster.
The scan reads PNG text chunks and A1111-style parameters, and, when `ffprobe` is installed, the prompt ComfyUI's video savers embed in MP4/MOV/WebM files. Files are read on the number of **worker threads** set in Settings → Performance (default: half your cores), which makes a large library several times faster.
### 🏷️ AI AutoTag
@@ -414,7 +414,7 @@ Automatically tag your entire image collection using AI vision models:
1. **AutoTag (Batch Mode)**: Tag your entire collection automatically
- Click **"🏷️ AutoTag"** in the admin dashboard
- Choose your model type (FP16 or GGUF). A model that is not downloaded yet shows **Manage in System**: download (or unload) it in System › Auto-tagging, then reopen the dialog
- Choose your model type (FP16 or GGUF)
- Select how to handle already-tagged images:
- **Skip images with existing tags**: Preserve your meticulous manual tagging work
- **Re-tag all images**: Overwrite existing tags with fresh AI analysis
@@ -445,7 +445,7 @@ Adjust the system prompt to match your tagging style:
#### **Requirements**
- Sufficient VRAM for the chosen model (FP16 requires more, GGUF is lighter)
- Models are downloaded in System › Auto-tagging
- Models are downloaded automatically on first use
- Feature requests and improvements are welcome!
### 📁 Folder Filter
@@ -470,18 +470,18 @@ If you use [ComfyUI-Lora-Manager](https://github.com/willmiao/ComfyUI-Lora-Manag
#### **Enabling the Integration**
1. Open System › Integrations (the admin dashboard's **Settings** opens System)
2. PromptManager auto-detects whether LoRA Manager is installed
1. Open the admin dashboard and click **Settings**
2. Scroll to the **Integrations** section — PromptManager will auto-detect if LoRA Manager is installed
![LoRA Enabled](images/pm-lora-enabled.png)
3. Toggle the **LoraManager** switch to enable
4. Optionally enable **Auto-inject trigger words** to automatically append LoRA trigger words when `<lora:name:weight>` is detected in your prompts
5. Click **Save preferences** (requires a ComfyUI restart to take effect)
5. Click **Save Settings** (requires a ComfyUI restart to take effect)
#### **Importing LoRA Data**
Click **Review import** in System › Integrations; Start runs the import as an Activity job:
Click **Import LoRA Data** in the settings panel to start the import. A progress popup shows real-time status as each LoRA is processed:
- Reads LoRA Manager's own settings the way LoRA Manager does: from the folder in `LORA_MANAGER_SETTINGS_DIR`, from the plugin's `settings.json` in portable mode, or from your user config folder, and only the keys that say where its files are
- Finds each LoRA's metadata in both storage modes: beside the model (the default) or in the central sidecar folder, and imports only LoRAs whose model file still exists in a LoRA folder (the review says how many metadata files it skipped)
@@ -502,7 +502,7 @@ A CivitAI API key is **required if your library contains NSFW LoRAs** — CivitA
To add your key:
1. Get your API key from [civitai.com/user/account](https://civitai.com/user/account)
2. Paste it into the **CivitAI API key** field in System › Integrations
2. Paste it into the **CivitAI API Key** field in the Integrations settings
3. Save and re-import to download any previously skipped images
The key is sent only over HTTPS to CivitAI's own hosts. CivitAI serves images through a redirect to its storage host; PromptManager follows such a redirect only over HTTPS to a host on CivitAI's own `civitai.com`, `civitai.red` or `civitai.green` domains, at most three hops, and never sends the key with it. When downloads fail, the import's status line says how many; when none could be downloaded, it starts with a warning. The server log has one warning per failed image with the reason (HTTP status, refused redirect, size cap, unreadable image, timeout or network error).
@@ -517,20 +517,23 @@ The comprehensive web interface provides:
- **Filter Options**: Filter by category, tags, rating, and date
- **Bulk Operations**: Select multiple prompts for batch editing
- **Export Tools**: Download your prompt collection
- **Settings**: System (new tab) for settings, maintenance, backups and diagnostics; **Page options** for the page's own display settings
- **Settings Panel**: Configure auto-save and display options
- **Diagnostics**: System health checks and troubleshooting
### 🔧 Database Maintenance
Keep your database optimized in System › Maintenance (the admin dashboard's **Maintenance** opens it in a new tab):
Keep your database optimized with built-in maintenance tools:
- **Duplicate prompt text**: merge duplicate prompts while preserving images
- **Optimize database**: reclaim unused SQLite space
- **Missing-file links**: remove image records for missing files
- **Unused prompt components**: remove prompts nothing uses
- **Run health checks**: verify database integrity and relationships (the Health report)
- **Database size and prompt counts**: System › Diagnostics & logs (Database) and the Overview's Statistics
Each operation opens a review that shows what it will change and the safety backup it takes, then runs as an Activity job.
1. **Open the admin dashboard** and click **"🔧 Maintenance"**
2. **Select maintenance operations**:
- **Remove Duplicates**: Find and merge duplicate prompts while preserving images
- **Vacuum Database**: Optimize database file size and performance
- **Clean Orphaned Images**: Remove image records for missing files
- **🚮 Prune Orphaned Prompts**: Remove prompts that have no linked images
- **Check Consistency**: Verify database integrity and relationships
- **Database Statistics**: Generate detailed database reports
3. **Run selected operations** and monitor progress in real-time
4. **Review results** showing detailed information about each operation
### Database Location
@@ -832,8 +835,8 @@ Before upgrading or changing this setting:
1. Identify the database your current installation actually uses and confirm its
library contents. Check both the configured path and the prior fallback
`<extension>/prompts.db`; keep track of which existing file you intend to use.
2. Preserve a consistent backup of the active database with
System › Backup & export's Back up or SQLite's online backup API. Do not copy only
2. Preserve a consistent backup of the active database with the dashboard's
Backup/Restore backup action or SQLite's online backup API. Do not copy only
the main file of a live SQLite database: committed data may still be in its
`-wal` file. For a filesystem copy, stop ComfyUI cleanly first and preserve any
remaining `-wal` file together with the main database.
+59 -473
View File
@@ -181,12 +181,10 @@ check, so run it before pushing a new or deleted frontend file instead of
waiting for the `frontend coverage` jobs.
The `viewerjs` check runs on every PR, including backend-only changes. It is an
aggregate job. It requires the three `frontend coverage (shard N/3)` jobs and
`frontend coverage (merge)` to succeed on every event. The four
`page workflows (firefox|webkit, shard N/2)` jobs run only in the merge queue,
on `main` and on `workflow_dispatch` (see "Pull requests and the merge queue").
`viewerjs` requires them to be skipped on a pull request and to succeed
everywhere else. Any other failed, cancelled or skipped job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
aggregate job that passes only when every frontend job succeeds: the three
`frontend coverage (shard N/3)` jobs, `frontend coverage (merge)`, and the four
`page workflows (firefox|webkit, shard N/2)` jobs. A failed, cancelled or skipped
job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3`
(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins
and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3
@@ -262,31 +260,6 @@ watchdog dump, is a failure like any other: read the dump, find the blocking
call, fix it. Never raise a limit or rerun to get green; a limit changes only
with new measurements written next to it.
Browser jobs (#425): GitHub keeps no log of a job killed at its limit and skips
its upload steps, so the test step in each browser job in `viewerjs-check.yml`
has its own `timeout-minutes` (18, below the job's 20). At the step limit
GitHub sends SIGTERM, which would end Playwright without any report; under `CI`
`tools/vendor/running-tests-reporter.mjs` catches the first one, prints
`Still running when the run ended (SIGTERM): N` and one line per unfinished
test (path relative to the suite's `rootDir`), then sends Playwright its own
SIGINT, so the workers stop, the run ends interrupted and the blob report is
written to `tools/vendor/test-results/blob/`. In the coverage job npm signals
only `frontend-coverage.mjs`, which forwards SIGTERM and SIGINT to the child it
is running (started in its own process group, so a group-wide signal still
arrives once). Delivery is proven locally in the pinned image for a SIGTERM to
the whole process group (`timeout -s TERM`). A SIGTERM to npm alone
(`timeout --foreground`) reaches nothing: npm forwards it to the `sh -c` that
runs the script, and that shell exits without passing it on. Delivery in CI,
where the step runs through `docker exec`, is not yet observed; the next real
step timeout will show it. That directory and the traces
upload with `if: always()`; `npx playwright merge-reports` opens the blob.
Each test keeps the 30 s default and there is no `globalTimeout`. Two limits
of this: it assumes the npm cache hits (a cold cache adds up to 5 minutes to
`npm ci`, and the job limit can then still win), and coverage shard 1 (12m53s
measured) is the tight one in its 18-minute step. Local runs keep the `list`
reporter only; reproduce with `CI=1 timeout -s TERM 40 npm run --prefix
tools/vendor test:workflows -- --project=firefox`.
### After a push
Follow the run for the commit you pushed, never an older one:
@@ -298,13 +271,6 @@ gh run view <run id> --json jobs # shows a job that alr
gh run view --job <job id> --log-failed # that job's failed log, at once
```
Python 3.13/3.14, real ComfyUI and Firefox/WebKit failures first show up in the
merge queue, on the merge-group commit rather than the
pushed one. Find that run with `gh run list --event merge_group --workflow test.yml`
(or `viewerjs-check.yml`) and match its head branch
(`gh-readonly-queue/main/pr-<N>-...`). Windows and macOS failures show up only
after the merge, in `gh run list --workflow portability.yml`.
Start fixing as soon as a failed log shows the cause, but push the fix only after
the run for that commit has finished: pull-request runs use `cancel-in-progress`,
so a push cancels the running jobs and can hide a second failure.
@@ -416,10 +382,6 @@ A spec performance budget gets two tests:
`test_filter_budget` (Images filters, spec D12: p95 < 150 ms at 50,000 files
and 20,000 links).
A hot SQLite read that returns many rows aggregates them into one step
(`json_group_array`) instead of stepping once per row, so it does not wait for
the GIL once per row while ComfyUI generates (#354, `get_image_link_matches`).
The `perf budget` job in `.github/workflows/test.yml` is the reference
measurement: `ubuntu-latest`, one Python, no coverage, only the wall-clock tests.
It runs on every push to `main` and on `workflow_dispatch`, never on pull
@@ -577,95 +539,29 @@ Before changing paths, SQLite, streams, restore or cleanup:
open with `FirstWriterHold`) and give peers that must all succeed the
ordinary busy timeout; cross-thread handoffs wait on events with generous
ceilings, never a short wall clock (#242).
- Run native Windows CI early for these changes. Pull requests and the merge
queue run only Linux, so start Windows and macOS on the branch with
`gh workflow run portability.yml --ref <branch>`.
Include a Windows impact and evidence note in the PR. All six Windows/macOS
combinations in that run, every Linux leg and every applicable
- Run native Windows CI early for these changes and include a Windows impact and
evidence note in the PR. All twelve OS/Python combinations and every applicable
exact-head check must pass before merge. Portable `ntpath`/`PureWindowsPath`
controls do not establish native filesystem, locking or cleanup success.
## Pull requests and the merge queue
Each PR runs the full matrix once, in the merge queue (#420). The event decides
what runs:
| Job | `pull_request` | `merge_group`, push to `main`, `workflow_dispatch` |
| --- | --- | --- |
| `py3.12 on ubuntu-latest` | runs | runs |
| `py3.13 on ubuntu-latest`, `py3.14 on ubuntu-latest` | not created | run |
| `real ComfyUI compatibility` | skipped | runs |
| `coverage gate (85%)`, `black and flake8`, `frontend modules`, `bandit (high severity)` | run | run |
| `frontend coverage (shard N/3)`, `frontend coverage (merge)` (vendor pins and tooling tests on shard 3) | run | run |
| `page workflows (firefox\|webkit, shard N/2)` | skipped | run |
| `all tests`, `viewerjs` (gates) | run | run |
| `perf budget` | skipped | push and dispatch only (not in the queue) |
| `py3.12-3.14 on windows-latest/macos-latest` (`portability.yml`) | not run | push to `main`, nightly, `workflow_dispatch` only; never required |
GitHub has one required set for pull requests and the queue. A job skipped by
its `if:` reports Success. A dropped matrix leg never reports, so it would stay
pending. That is why `all tests` (in `test.yml`) and `viewerjs` carry the
guarantee:
- On a pull request they require the subset to succeed and the queue-only jobs
to be skipped.
- On every other event they require every job to succeed.
So a merge group cannot turn green unless all three Linux Python legs, real
ComfyUI and the four browser shards passed.
Windows and macOS (Python 3.12-3.14) are out of the blocking path for now
(maintainer, 2026-10-28). They run in `portability.yml` on every push to `main`,
nightly and on demand, and no context there is required. A red portability run
is still a failure: fix it at its cause, not by rerunning, and have every one
green before the workspace cutover (#334). `tests/vendor/ci-workflow.test.mjs` pins the
event split and runs both gate scripts for each event.
## Required merge checks
`.github/branch-protection.json` records the required checks:
`py3.12 on ubuntu-latest`, `coverage gate (85%)`, `black and flake8`,
`frontend modules`, `all tests`, `viewerjs`, `real ComfyUI compatibility` and
`bandit (high severity)`. Each one reports on a pull request.
The same set lives in two places, and both must change together: the `main`
ruleset (25960526, which also holds the merge queue) and the classic branch
protection on `main`, which the manifest mirrors. GitHub requires the union of
the two, so a context left in either one blocks every PR that does not report it.
`.github/branch-protection.json` records the required checks: all twelve Python/OS
combinations, Python coverage, formatting, frontend modules, `viewerjs`, `real ComfyUI compatibility`,
and `bandit (high severity)`.
The branch must be current with main, and administrators follow the same checks.
It does not require an additional human approval.
A repository administrator applies the configuration after the new workflow is
passing, then verifies it:
```sh
gh api repos/ComfyAssets/ComfyUI_PromptManager/rulesets/25960526 \
| jq '.rules[] | select(.type == "required_status_checks") | [.parameters.required_status_checks[].context]'
```
A PUT replaces the whole `rules` array, so edit the current ruleset in place.
Keep the `deletion`, `non_fast_forward` and `merge_queue` rules, and give every
context `integration_id` 15368 (GitHub Actions), so no other integration can
post a required check. Save the current ruleset first; it is the rollback:
```sh
gh api repos/ComfyAssets/ComfyUI_PromptManager/rulesets/25960526 > ruleset-before.json
jq '{name, target, enforcement, conditions, bypass_actors, rules: [.rules[]
| if .type == "required_status_checks"
then .parameters.required_status_checks = [$ARGS.positional[] | {context: ., integration_id: 15368}]
else . end]}' ruleset-before.json --args "py3.12 on ubuntu-latest" "..." > ruleset-new.json
gh api --method PUT repos/ComfyAssets/ComfyUI_PromptManager/rulesets/25960526 --input ruleset-new.json
```
Afterwards, read it back. Check that the `merge_queue` rule is still there and
that every context keeps its `integration_id`. Then apply the manifest to the
classic protection and read both back:
```sh
gh api --method PUT repos/ComfyAssets/ComfyUI_PromptManager/branches/main/protection \
--input .github/branch-protection.json
gh api repos/ComfyAssets/ComfyUI_PromptManager/branches/main/protection/required_status_checks
gh api repos/ComfyAssets/ComfyUI_PromptManager/rules/branches/main
```
Update the manifest whenever
required job names change. Keep the frontend check
Update the manifest whenever required job names change. Keep the frontend check
free of path filters so a required check is never silently skipped.
## Unified workspace
@@ -688,7 +584,7 @@ The shared inline and collection inspector reads the current rooted file's suppo
embedded records, exact generation facts and recorded or inferred prompt evidence.
It renders in two concept surfaces (#249 D20–D22, D38). The Images side panel shows
only the concept content: cover, title, filename, the evidence badge with a category
chip naming its origin (D146), the + and − prompt blocks with copy and send, tag chips (the file's own image
chip, the + and − prompt blocks with copy and send, tag chips (the file's own image
tags first, D88, then the linked tags; image tags and the linked positive's own tags
filter Images, D21), the six facts, and a footer with "Send to ComfyUI" and
"Prompt history" only when prompt roles resolved or a link is saved; the sticky panel
@@ -918,25 +814,6 @@ unattributed stage, when it is a plain first stage: not a detailer, no prompt ov
set, and no listed sampler in its own latent ancestry (else it stays unattributed) (`tests/test_resolver_saver_anchor.py`, shapes in
`tests/saver_anchor_graphs.py` from the maintainer's `gpu_unlock_bench`, dataset and
ANIMA runs; browser parity through the `anchor-*` cases, which carry `__file_name`).
Recorded run history (#386): both inspect routes (Images and a history occurrence)
ask `database.run_attribution.recorded_output` inside the file's stat window. It
answers only for a file that exactly one recorded run reported and that was written
during that run (the #342 rules): the one output node that run reported it from (none
when several did) and, per sampler anchor, the effective texts PromptManager recorded
at execution (a role only inferred from the graph is none). Only when the file alone
leaves the prompt open (`ambiguous` or `unresolved`) the inspector resolves again
anchored on that node (`recorded_saver_id`, walked like a file-name match; a file
name naming a different saver contradicts history and anchors nothing), and keeps
that result only when history's saver chose its samplers (`recorded_anchor`) and it
names the positive (resolved, or a guider with no negative); its `graph-inferred`
sides read `recorded-run` ("Recorded run" badge, ranked below every "Workflow
inferred" source). Still open at the one sampler that saver leads to, with that
sampler's positive text recorded (a language model's output feeding PromptManager),
the recorded texts show as `execution`. Outputs and texts are read in one
transaction. A result the file resolves alone never changes; a lookup failure is
logged and judges the file alone; local Inspect and ingestion are unchanged
(`tests/test_recorded_saver_anchor.py`, `tests/test_run_attribution.py`,
`tests/test_api_history_outputs.py`).
`tools/inspection_corpus.py --root DIR --out JSONL [--workers N] [--baseline JSONL]
[--summary TXT] [--code TREE]` measures a media folder read-only against the #240
@@ -1369,7 +1246,7 @@ How the earlier workspace Images filters moved (old → new). The legacy `/galle
| Folder | "Refine this view" → Folder (D14); shown in the scope chip |
| Media (All media / Images / Videos) | "Refine this view" → Media (D14) |
| Sort: Newest first / Oldest first / Filename A–Z / Filename Z–A | toolbar sort: Recently added / Oldest first / Name A–Z / Name Z–A, plus Highest rated (D13, D14) |
| (none) | toolbar category (the effective category, D141) and "Refine this view" → Tags, through the linked positive (D13, D14) |
| (none) | toolbar category and "Refine this view" → Tags, through the linked positive (D13, D14) |
| Refresh | Import → "Scan an output folder" → Rescan (D15) |
| Grid / List / Masonry, Comfortable / Large / Compact | the same choices in the concept filter row |
| Card image → side panel; preview icon → viewer | image → viewer; Info → side panel (D19) |
@@ -1399,18 +1276,22 @@ Recorded deviations on this page (spec §8):
inspector's own "Download metadata JSON" envelopes in listing order); the button is
`aria-disabled` and ignores a second press while it runs, the selection is kept, and
the toast names the unread images (none read: no download); leaving Images aborts the
reads with no download and no toast. Set category (amended by D140, #332) sets the
checked images' own categories: it opens the concept's `simpleInput` dialog
("Categorize selected images", `input#action-input` labelled Category) and writes
`PUT`/`DELETE /prompt_manager/images/category`, never a prompt (see D140). Prompt
categories are changed in the Prompt library editor and on the legacy admin page's
"Set Category"; `POST /prompt_manager/bulk/category` is unchanged and serves that
legacy page only. The write carries no page signal: leaving Images while it runs lets
it finish without a toast, and the return loads the collection afresh instead of the
saved visit. Its success reloads the categories and the listing together (clearing
the selection, focus to the heading, and, as any replaced collection does, the
inspector moves to the first listed image); a failure keeps the selection, except a
changed source (409), which reloads the listing as a success does. `images selection bar matches
reads with no download and no toast. Set category opens the
concept's `simpleInput` dialog ("Categorize selected images", `input#action-input`
labelled Category, Cancel / Apply) and posts the selection's distinct linked positive
ids to `/prompt_manager/bulk/category` (images have no category; the filter matches
the linked positive's). With no linked image checked it posts nothing and opens no
dialog; an empty value keeps the dialog open; the field stops at the server's 100
characters; success reloads the categories and the listing together (clearing the
selection, focus to the heading, and, as any replaced collection does, the inspector
moves to the first listed image); failure keeps the selection. The write carries no
page signal: leaving Images while it runs lets it finish without a toast, and the return
loads the collection afresh instead of the saved visit. Three deviations from the
concept's `simpleInput`: the field starts empty (the concept pre-fills its demo
"Landscapes"), the action reads "Apply" (not "Apply in prototype"), and the
nothing-linked check runs on the Set category press, so no dialog opens. The route
rejects anything but 1-1000 positive integer ids within SQLite's INTEGER range
(deduplicated) and a category that fails `validate_category`. `images selection bar matches
the concept` compares the bar and the checked card with one card checked; the omitted
actions flow again from the concept's own boxes, checked against the concept page with
them removed at runtime.
@@ -1448,8 +1329,7 @@ Recorded deviations on this page (spec §8):
`overscroll-behavior: contain`.
- **D35**: counts pluralize ("1 positive prompt", "1 image", "1 more image loaded").
- **D36**: the category list is the Library's, sorted alphabetically (case-insensitive)
by the page, whatever order the server sends (amended by D141: Images adds its
image categories).
by the page, whatever order the server sends.
- **D37**: production loads 40 images per page (the concept 12), so the continuation
and toast use 40; after Escape production returns focus to the opening image.
- **D44** (#249 PR E): Filters reads "Filters · N" while N of Source, Folder, Media and
@@ -1477,9 +1357,6 @@ Recorded deviations on this page (spec §8):
filters, while recipe, recorded-run and embedded tags stay plain `span.chip`.
Amended by D88 (#249 PR C): the file's own current image tags lead the same row
and are filters too; a linked tag already shown (case-insensitive) is left out.
Amended by D146 (#332): the category chip also names its origin in visible text
after the category (" · this image", " · prompt", " · recipe"), and an image's own
category shows for an unlinked or ambiguous file too.
- **D22**: an empty positive reads "No positive prompt (explicitly empty)" (the concept
applies its negative wording to both roles), and a confirmed `p.prompt-text` is
focusable (`tabindex=0`, marked focus ring) so long prompts stay keyboard-readable.
@@ -1538,97 +1415,15 @@ Recorded deviations on this page (spec §8):
- **D39** (ruled 2026-10-21: fallback): a recipe-linked image shows its recipe's own
category when set, otherwise the linked positive prompt's, and its recipe's own tags
when it has any, otherwise the positive's; each field falls back on its own and the
two tag sets are never merged. The Images Tags filter matches the positive's tags.
The category filter matches the same inherited category as the panel (D142, #332), and
an image's own category overrides both (D141, D146). The inspection response names each
two tag sets are never merged. Set category (#259) writes the positive's category and
the Images category and Tags filters match the positive's, so the panel now agrees
with them whenever the recipe adds nothing. The inspection response names each
field's source (`library.category_scope`, `library.tags_scope`: `pair`, `positive` or
null, or `image` for an own category, #332); the category chip's visually hidden
prefix (D21) says "Recipe category: ", "Positive prompt category: " or "Image category: "
from it, and its visible origin (D146) " · recipe", " · prompt" or " · this image", and tag chips are Tags filter buttons only when
null); the category chip's visually hidden prefix (D21) says "Recipe category: " or
"Positive prompt category: " from it, and tag chips are Tags filter buttons only when
they are the positive's (#260 D2). History's media inspector uses the same rule. The
concept shows the prompt's category and tags for every image and the concept
fixture's recipes have neither, so the inspector comparison needs no mapping for it.
- **D141** (#332, amends D13 and D36): the Images category filter matches each image's
effective category: its own category (Images Set category, PR 3) when it has one,
else its inherited one (D142), else none. An image whose own category differs never
matches its prompt's category, and an unlinked or ambiguous image with an own
category is listed under it (the members-only gate now applies to the Tags filter
only). "Select all N matching" and saved views use the same value. The caption's
positive-prompt count still counts the linked positives of the images shown. With a
category set the listing reads the own categories once per source (a primary-key
range); a failed read is the existing 500 `listing-failed`, never an unfiltered page.
The link read then keeps only links whose raw recipe or positive category is the
value (a superset of those that inherit it; Python decides), and reads the links of
files whose own category is the value by path, so their rating and positive count.
`test_filter_budget` seeds 2,000 own categories on its 50,000-file fixture.
The option list adds the image categories of listed files
(`GET /prompt_manager/images/categories`, PR 3), merged with the Library's and sorted
(D36); if only that read fails, the prompt categories show and the categories alert is
raised. `images category options and Set category dialog match the concept` sets one
image's own category ("Portraits") through the real route and compares the toolbar
with it sorted into the concept's options; unmapped, only the options differ.
- **D142** (#332 MC4, amends D39's filter note): the inherited category is one rule,
`inherited_category` in `utils/inspection_response.py`: a linked recipe's own
category when it is not blank, else the linked positive's. Before #332 the filter
matched the positive's even when the chip showed the recipe's. The link reads select
the raw recipe and positive categories and never decide inheritance in SQL
(`tests/test_api_workspace_image_categories.py` checks the filter and the chip agree
on blank values stored as spaces, tabs, U+00A0 and BLOB). The concept has no recipe
categories, so this is behavior only.
- **D145** (#332, DC4): History's media inspector shows an image's own category with
the chip rule of the side panel and viewer. Only the inspection request reads it, once,
inside the History snapshot: the file is the symlink-resolved output folder plus the
receipt's subfolder and filename, keyed with `image_category_key` over the configured
output sources, as Images keys it. A temp output or a file no source holds has none;
a failed read hides the chip (DC5). The outputs list and media requests read nothing
new. The chip's hidden prefix reads "Image category: " for an own category; the
visible origin text and its comparison are D146 (PR 3). `history media inspector
category chip matches the concept` (history-concept.spec) sets an own category on the
Quality baseline's newest output, opens its Recorded run and "View current file", and
compares the viewer's chip with the concept viewer's category chip (app.js a402) laid
out at runtime with the D146 origin and that category; its place is not compared (what
precedes it differs), only its size, tokens, text, prefix and origin. Unmapped, the two
differ only in the text, the prefix, the width and `span.chip-source`.
- **D140** (#332, amends D8): Set category sets the checked images' own categories, so
every checked file is a target (no nothing-linked check). More than 500 checked (or
"all N matching" over 500) toasts "Select at most 500 images." and opens nothing.
The dialog is the concept's `simpleInput` with three recorded differences: the field
starts empty (the concept pre-fills its demo "Landscapes") and suggests the toolbar's
current categories (`datalist#workspace-input-suggestions`, `openInputDialog`'s
`suggestions`); "Use prompt category", a plain button between Cancel and Apply
(`secondary`), closes it and opens the "Use prompt category" confirmation ("Remove the
image category from the 1 selected image? / … the N selected images? Each one then
shows its prompt's category, or none if it has no linked prompt. This cannot be
undone.", Cancel and a danger Remove, built like saved views' delete confirmation);
and the action reads "Apply" (D8). Apply sends `PUT /prompt_manager/images/category`
`{items | filter, category}`, Remove `DELETE` `{items | filter}`; Cancel sends
nothing. An empty value keeps the dialog open and the field stops at 100 characters.
Toasts: "Category “X” set on 1 image." / "… on N images." (unchanged files count as
set) plus " N could not be found." for unlisted files; "Removed the image category
from N images." or "None of the selected images had its own category."; 404 "None of
the selected images could be found. Refresh the images and try again."; 409 "The
sources changed. Refresh the images and try again." (the listing reloads); anything
else "Category could not be set. Try again.". `images category options and Set
category dialog match the concept` (images-concept.spec) checks a card, opens the
dialog in both documents and compares it (title, body tree, field, input, suggestions,
actions and every button's box and tokens) with the concept's own dialog changed at
runtime as recorded: field emptied, the datalist added after the field, a clone of
its Cancel reading "Use prompt category" after Cancel, the primary renamed "Apply".
Unmapped, the dialog differs only in the input's value and list, the body's tree, the
suggestions and the actions.
- **D146** (#332, amends D21 and D39): the category chip names its origin in visible
text. It is `span.chip` > `span.ca-visually-hidden` (the D21 prefix) + the category +
`span.chip-source` (" · this image", " · prompt" or " · recipe"; muted
`--ca-muted`, `white-space: nowrap`, so at 390px the origin never splits; the chip may wrap
before it or inside a long category). It shows when the inspection names a non-empty category and either the
scope is `image` (any link status, so an unlinked or ambiguous file with an own
category shows it) or the file is linked. The concept's chip shows the bare name
(app.js a402). The inspector comparison (inspector-concept.spec) measures each chip's
`span.chip-source` (text, font, color, `white-space`, box) and lays the concept's own
chip out with the recorded rule and " · prompt" added at runtime (`withChipSource` in
`concept-inventory.mjs`), in the side panel and the viewer's details; unmapped they
differ only in the chip's text, width and `span.chip-source`. Its mutation check
removes the origin. History's chip (D145) is compared the same way.
- **D42**: the concept never shows a toast and a page alert together. Production keeps
the concept's fixed toast (centred, 22px from the bottom, 12px at 700px and below)
and, when an Import → Rescan toast leaves the Images discovery alert
@@ -1653,8 +1448,7 @@ applies D3 (the `.file-inspector` class of the production-only rules), D20 (the
viewer's plain tag chips, laid out by the concept page itself with its chips made
plain at runtime), the D20/D21 side-panel scroll (`overflow: auto`,
`max-height: calc(100dvh - 40px)` above 700px), D21 (the category's hidden scope
prefix), D22 (focusable prompt text) and D146 (the chip's visible " · prompt", laid out
by the concept page itself). `inspector comparison detects mutations`
prefix) and D22 (focusable prompt text). `inspector comparison detects mutations`
changes `.prompt-heading .role-label` letter spacing and removes `.detail-footer`.
Manual check (quick), after a browser refresh on Images: hover an image (it zooms);
@@ -1989,13 +1783,11 @@ ranges include cards that bounded rendering has unmounted.
Over the limit, the client toasts "Narrow the view to at most 10,000 images." and the
server answers 400 `too-many-images`. No match is 400 `no-images`. Auto-tag in this
mode does not offer Review each image, because the review walks images in the
browser. Set category (D140, #332) sends `{filter, category}` to
`PUT /prompt_manager/images/category` (Use prompt category `{filter}` to `DELETE`),
which sets or removes each matching image's own category; over 500 matches the client
toasts "Select at most 500 images." and opens no dialog, and the server answers 400
`too-many-images`. (`POST /prompt_manager/bulk/category` still accepts
`{images_filter, category}` and sets the linked positives' category, but no workspace
control sends it any more.) Export metadata reads every image in
browser. Set category posts `{images_filter, category}` to `/prompt_manager/bulk/category`.
The server sets the category on the distinct linked positives and answers
`prompt_count` and `skipped`. A listing whose `positive_count` is 0 toasts the
nothing-linked sentence and opens no dialog, and over 10,000 matches it toasts the
limit sentence. Export metadata reads every image in
the browser, so it is `aria-disabled` in this mode, and its `title` says why.
Tests: `tests/browser/workspace-images.spec.mjs` ("Shift+click selects a range across
@@ -2667,14 +2459,6 @@ production hub `#/overview` is unchanged (D28); its System card opens
the pressed section button and disposes the previous section's scope, so its reads,
listeners and dialogs never outlive it.
System is the home for settings and maintenance (#330 System S13). The legacy pages keep
their prompt list, tags, Scan Images, auto-tag batch and review and gallery browsing; their
Settings, Diagnostics, View Logs, Maintenance, Backup DB, Restore DB, the gallery's
thumbnail, cache, rescan and duplicate tools and the auto-tag model downloads open the
System section that owns them in a new tab (`legacy-system-links.spec.mjs`). Each page
keeps only its own settings in Page options. What left the legacy pages is listed in
"Legacy System UI removed by S13".
Drafts (`system-state.mjs`) live for the page session (Q10). Save writes the dirty
browser fields, then the host fields (`POST /prompt_manager/settings` with
`expected_revision`, #220), then the runtime fields. A failed domain keeps its drafts and
@@ -4289,9 +4073,6 @@ auto-tag / review", which the Images selection bar owns, not a System section. A
is not `covered` names none (`—`). Test is the exact title of a test that runs, then its
file. Anchors are line numbers on `main` at this PR.
S13 (#330) removed the legacy copy of the `covered` rows the legacy pages linked to System;
"Legacy System UI removed by S13" below lists every removed element against its row.
`tests/vendor/legacy-coverage.test.mjs` fails on a row with any of these problems:
- no owner, or a status outside the three;
- a malformed Native control cell;
@@ -4372,11 +4153,7 @@ opens `workspace#/system/<id>` in a new tab) and removes the legacy copy of a ca
only when its row above is `covered`. One row per removed capability: Capability is the
exact Capability cell of its coverage row, Page is `admin` or `gallery`, Removed ids and
Removed methods list what went (`—` for none), and Native test copies that row's Test
cell. An id that stays on a System link is not listed, only its handler. The gallery's
Default View Mode left Page options too: System › Appearance's Default layout is the one
default-layout control. Nothing writes the legacy `defaultViewMode` any more; a stored
value still picks the view the legacy gallery opens in (grid otherwise), and Page options
Save and Reset keep it.
cell. An id that stays on a System link is not listed, only its handler.
`tests/vendor/legacy-coverage.test.mjs` fails when a row cites a capability that is not
`covered` or copies another test, when an id or method is listed twice for one page, or
@@ -4395,31 +4172,6 @@ legacy page workflows), not by this check.
| Log files list and download | admin | `#downloadLogsBtn`, `#logFilesList` | `loadLogFiles()`, `downloadLogs()`, `downloadLogFile()` | each listed file downloads by its returned name and a link has no Download (workspace-system-diagnostics.spec.mjs) |
| Clear/truncate logs | admin | `#clearLogsBtn` | `clearLogs()` | Clear all logs states both counts, confirms first and reads everything again (workspace-system-diagnostics.spec.mjs) |
| Set log level (incl. CRITICAL), console logging | admin | `#updateLogConfigBtn`, `#setLogLevel`, `#consoleLogging` | `loadLogConfig()`, `updateLogConfig()` | runtime level (including CRITICAL) and console logging save and say they apply until the server restarts (workspace-system-diagnostics.spec.mjs) |
| Image scan directories add/remove | admin | `#scanPathsList`, `#addScanPathBtn` | `renderScanPaths()`, `addScanPath()`, `removeScanPath()`, `_collectScanPaths()` | Add source validates on the server, then marks the host draft dirty; Save persists (workspace-system-sources.spec.mjs) |
| Currently monitoring list + Refresh | admin | `#refreshMonitoringStatus`, `#monitoringStatus` | `updateMonitoringStatus()` | watching only with a live observer; restart pending otherwise (workspace-system-sources.spec.mjs) |
| Search results auto-hide `result_timeout` | admin | `#resultTimeout` | — | host fields round-trip through /settings with revision (workspace-system.spec.mjs) |
| Web UI display mode | admin | `#webuiDisplayMode` | — | host fields round-trip through /settings with revision (workspace-system.spec.mjs) |
| Worker threads | admin | `#workerThreads`, `#workerThreadsValue` | `renderWorkerThreads()` | worker range max is cpu_count (workspace-system-sources.spec.mjs) |
| LoraManager badge, enable, path, trigger words, CivitAI key, clear key | admin | `#loraDetectionBadge`, `#loraEnabled`, `#loraSettings`, `#loraManagerPath`, `#loraTriggerWords`, `#civitaiApiKey`, `#clearCivitaiKey` | `detectLoraManager()`, `_bindLoraEvents()`, `saveLoraSettings()` | LoRA rows show detected / enabled / cache-loaded separately (workspace-system-integrations.spec.mjs) |
| LoRA data import with progress | admin | `#loraImportModal`, `#loraImportBtn`, `#loraImportStatus`, `#loraImportPercent`, `#loraImportBar`, `#loraImportProcessed`, `#loraImportImported` | `runLoraImport()` | Review import shows replacement, blocked items and network effects with the backup box; Start opens the job (D107) (workspace-system-integrations.spec.mjs) |
| Maintenance cleanup_duplicates, vacuum, cleanup_orphaned_images, prune_orphaned_prompts | admin | `#maintenanceModal`, `#cleanup_duplicates`, `#vacuum`, `#cleanup_orphaned_images`, `#prune_orphaned_prompts`, `#runMaintenanceBtn`, `#maintenanceResults` | `showMaintenanceModal()`, `closeMaintenance()`, `runMaintenance()`, `renderMaintenanceResults()` | Apply reviewed operation closes, toasts, opens the job in Activity (workspace-system-maintenance.spec.mjs) |
| Maintenance check_hash_duplicates, check_consistency | admin | `#check_hash_duplicates`, `#check_consistency` | — | Run health checks opens Health report and updates metric 4 (workspace-system.spec.mjs) |
| Maintenance statistics (database info) | admin | `#statistics` | — | meta-grid shows real database and source status, the pinned Viewer.js version and the host renderers (workspace-system-diagnostics.spec.mjs) |
| Maintenance select all / clear all / run | admin | `#selectAllMaintenanceBtn`, `#clearAllMaintenanceBtn` | `selectAllMaintenance()`, `clearAllMaintenance()` | plan dialog shows the server's scope, protected, backup policy and media lines with the backup box (D100); Cancel writes nothing (workspace-system-maintenance.spec.mjs) |
| Database backup | admin | — | `backupDatabase()` | Back up downloads through the api client and revokes the object URL (D103) (workspace-system-backups.spec.mjs) |
| Restore from file | admin | `#restoreModal`, `#restoreFileInput`, `#confirmRestore`, `#cancelRestore` | `showRestoreModal()`, `confirmRestore()` | Review restore stages the chosen file and shows the real summary with the backup box (D105) (workspace-system-backups.spec.mjs) |
| Gallery thumbnail generation | gallery | `#generateThumbnailsBtn`, `#thumbnailStatus`, `#thumbnailProgress`, `#progressText`, `#progressPercent`, `#cancelThumbnailsBtn`, `#progressBar`, `#progressDetails`, `#progressETA` | `generateThumbnails()`, `generateThumbnailsWithProgress()` | Start closes the dialog, toasts, and opens the job in Activity (workspace-system-sources.spec.mjs) |
| Default view mode, image quality | gallery | `#defaultViewModeSelect`, `#imageQualitySelect` | — | defaults apply to a new visit but not to the current view or a saved view (workspace-system.spec.mjs) |
| Image info on cards; sidebar collapsed | gallery | `#showImageInfoToggle`, `#sidebarCollapsedByDefaultToggle` | — | added rows reach their consumers (workspace-system.spec.mjs) |
| Video autoplay, mute, loop | gallery | `#videoAutoplayToggle`, `#videoMuteToggle`, `#videoLoopToggle` | — | video defaults reach the viewer (workspace-system.spec.mjs) |
| Auto-load metadata, cache metadata, show file paths | gallery | `#autoLoadMetadataToggle`, `#cacheMetadataToggle`, `#showFilePathsToggle` | — | metadata off shows an explicit load and loads once on request (workspace-system.spec.mjs) |
| Browser cache clear | gallery | `#clearCacheBtn` | `clearCache()` | Metadata cache reviews and clears only owned browser entries when there are some (D102) (workspace-system-maintenance.spec.mjs) |
| Clear server thumbnails | gallery | — | — | Thumbnail cache opens the S5 dialog (workspace-system-maintenance.spec.mjs) |
| Refresh file listing | gallery | `#rescanFolderBtn` | `rescanFolder()` | Refresh files calls /gallery/rescan and toasts its counts (workspace-system-sources.spec.mjs) |
| Duplicate image scan, review, delete | gallery | `#scanDuplicatesBtn`, `#dupScanPill`, `#dupScanPillText`, `#duplicatesModal`, `#closeDuplicatesBtn`, `#duplicatesScanStatus`, `#dupProgressText`, `#dupProgressBar`, `#duplicatesContent`, `#duplicatesList`, `#duplicatesFooter`, `#duplicatesCount`, `#selectAllBtn`, `#removeDuplicatesBtn` | `scanDuplicates()`, `attachDuplicateStream()`, `resumeDuplicateScan()`, `applyDuplicateEvent()`, `renderDuplicateProgress()`, `setDuplicateButton()`, `showDuplicatePill()`, `hideDuplicatePill()`, `openDuplicatesModal()`, `showDuplicatesModal()`, `hideDuplicatesModal()`, `displayDuplicates()`, `selectAllDuplicates()`, `removeDuplicates()` | Find duplicates starts a job; its view links Review groups; Keep radios build a duplicate_files plan (D101) (workspace-system-maintenance.spec.mjs) |
| Auto-tag model download | admin | `#autoTagDownloadModal`, `#downloadModelName`, `#downloadStatus`, `#downloadProgressPercent`, `#downloadProgressBar`, `#cancelDownloadBtn` | `downloadModel()`, `cancelDownload()` | Download starts the job and Activity shows it without Cancel (workspace-system-tagging.spec.mjs) |
| Auto-tag model download | gallery | `#autoTagDownloadModal`, `#downloadModelName`, `#downloadStatus`, `#downloadProgressPercent`, `#downloadProgressBar`, `#cancelDownloadBtn` | `downloadModel()`, `cancelDownload()` | Download starts the job and Activity shows it without Cancel (workspace-system-tagging.spec.mjs) |
| Unload model | gallery | — | `unloadModel()` | Unload is enabled only while a model is loaded, and a held gate's refusal names the operation (workspace-system-tagging.spec.mjs) |
### Browser cache ownership (#222)
@@ -4438,15 +4190,17 @@ page writes a key that is neither a preference nor inventoried, writes one it ca
CacheStorage or IndexedDB; `the storage scan reports every way of writing browser
storage` pins those patterns. It skips `web/lib` (third-party builds) and the pinned
Toolkit build (`web/js/shared/toolkit`, never hand-edited, verified by `npm run check`;
its storage belongs to the Toolkit repository and a new pin is reviewed on its own). Since
System S13 (#330) the legacy gallery no longer offers Clear cache: System › Maintenance's
Metadata cache and Thumbnail cache reviews own both parts (removal table above), and
`cache-ownership.js` serves the native review.
its storage belongs to the Toolkit repository and a new pin is reviewed on its own). The
gallery's server thumbnail clear stays its own request and runs first: refused or
unreachable, it changes nothing and says so (the old code toasted success when the
request failed); refused browser storage is reported, never hidden
(`tests/js/legacy-refusals.test.js`; the browser case `the gallery cache clear keeps
preferences and other tools caches` in `workflows.spec.mjs`).
The duplicates dialog's View Full named its source (`?root=<index>`, with the path
URL-encoded) when the gallery's output folder is a listed source
(`test_view_full_names_the_gallery_folder_source`); S13 removed the legacy dialog, and the
route keeps that behaviour for System › Maintenance's duplicate review.
The legacy duplicates dialog's View Full now names its source (`?root=<index>`, with the
path URL-encoded) when the gallery's output folder is a listed source; before, the serve
route opened the first source holding a file at that path
(`test_view_full_names_the_gallery_folder_source`).
### Library location contract (#249 System S6b)
@@ -4738,18 +4492,8 @@ Filters (`mode=workspace`; anything invalid is 400 "Invalid gallery filters"):
rating 5 to 1, then linked but unrated, then unlinked and ambiguous; ties break
newest first. A signed cursor covers every filter and the sort, so reusing it
with a change is 400.
- One link read (`get_image_link_matches`) serves the filters, the rating sort
and the count. SQLite returns its rows as one JSON value in one step (#354):
Python's `sqlite3` releases and retakes the GIL around every step, and while
ComfyUI generates each retake can wait on the busy thread, so a read that
stepped once per linked row took seconds. The result keeps only the listed
paths; peak memory grows with the database's matching linked rows (about
530 bytes each, freed on return). When the JSON step fails (a SQLite without
JSON functions, a BLOB value, a JSON text over SQLite's length limit) the read
streams the rows as before and logs one warning per process; a missing JSON
function stops the attempt for the rest of the process. An interrupted read
(Move library) raises as before. `tests/test_database_link_matches.py` proves both reads
equal on every filter and on odd values.
- One link read (`get_image_link_matches`, streamed and bounded to the listed
paths) serves the filters, the rating sort and the count.
`positive_count` is the number of distinct linked positives over the full matching
set, not the page:
@@ -4800,85 +4544,6 @@ concept fixture stars the concept's favorites (each other prompt's first image).
restore. Old backups without the table restore, and backup plus restore keeps
WAL-resident favorites and closes every handle (`tests/test_database_backup.py`).
Image categories (#332, store and routes): an image file may carry its own
category; without one it shows the category its link supplies.
- The table is `image_categories(root_key, relative_path, category, updated_at)`
with primary key `(root_key, relative_path)` and the index
`idx_image_categories_category`. `migrate_image_categories` creates both empty
inside a savepoint, with no backfill. It reads only `sqlite_master` and its own
table's PRAGMAs, and writes only its two objects. `tests/test_image_categories.py`
checks this with an SQLite authorizer on a 23,000-link library, and checks that
`prompts` and `prompt_pairs` `(id, category, updated_at)` are byte-for-byte equal
afterwards.
- The **file key** is `image_category_key(sources, absolute path)` in
`py/api/images.py`: the deepest output source that contains the path (compared
lexically, never resolving the path, so a symlinked subfolder keeps its listed
spelling) and the path below it, keyed with the favorites helpers. A source
matches by its real path or its configured spelling (a detected output folder
can be a symlink). Without nested sources it equals a favorite's key. Every write and read uses it. A path
that no source contains has no key, so it has no own category.
- `image_categories_schema_error` checks the structure at startup (a value saved
under older rules never stops the library opening; a table without its index,
which this code never leaves, is repaired by the install). Restore also checks the
values: text columns, keys within the favorites bound, `category` 1 to 100
characters with no character below U+0020 and no leading or trailing space, and
`updated_at` of at most 64 bytes. Backups made before #332 restore without the
table; a migrated restore stage must have it (`require_library_state`).
- `PUT /prompt_manager/images/category` `{items, category}` answers
`{success, category, updated, unchanged, not_found}`.
`DELETE /prompt_manager/images/category` `{items}` answers `{success, cleared}`.
`GET /prompt_manager/images/categories` answers `{success, categories}`: the
sorted distinct own values whose file is in some source's listing.
- `items` are 1 to 500 Images targets `{root_index, root_scope, path}`, checked by
`py/api/media_target.py` (shared with Collections, #331). Refusals:
- 400: a malformed body, a field other than `items` and `category`, a
`root_index` outside the output sources (a LoRA folder), or a category that is
blank after stripping, over 100 characters, or holds a control character;
- 403: traversal, a drive, a backslash, a non-media name ("Only media files can
have a category") or a key over the stored bound;
- 409 `source-changed`: a stale `root_scope`;
- 413: a body over 1 MiB.
Any invalid item fails the whole request before a write. Targets that fold to one
key are written and counted once. Instead of `items` a request may send `filter`,
the Images listing's own query ("Select all N matching", #392, keys as in
`parse_listing_filter`), resolved now: over 500 matches is a 400
`too-many-images` ("Select at most 500 images."), a renumbered source a 409
`source-changed`.
- PUT writes only files the source's listing holds and counts the rest in
`not_found`; when none is listed it answers 404 `files-not-found` and writes
nothing. A file that already has the value counts as `unchanged` and keeps its
`updated_at`. DELETE accepts any well-formed target, so a missing file's
category can still be removed. An unexpected failure is a logged 500 ("The
category could not be changed." / "Image categories could not be read.") without
the exception text. The routes never read or write `prompts`, `prompt_pairs` or
`generated_images`; `POST /prompt_manager/bulk/category` is unchanged.
- Duplicate cleanup (System › Maintenance): the kept copy's own category wins. A
kept copy without one receives the removed copy's (`COPY_SQL`, part of the
cleanup's query-plan check). The removed copy's row is never deleted, so a
give-back (the kept copy changed) and a failed delete lose nothing. The job log
says "Kept N image categories on the kept copies".
`tests/test_duplicate_image_categories.py` covers nested sources in both orders.
- A moved, renamed or deleted file's row stays and is hidden; a file that later
appears at the same path shows it. The real-host check is the last test in
`tests/comfyui/image-attribution.spec.mjs`: PUT, list, DELETE on a generated
output.
- Interface (PR 3, a browser refresh; no restart beyond PR 1-2's): the Images
selection bar's Set category writes these routes (D140, which amends D8); the toolbar
category options add the listed images' own categories (D141); the inspector's
category chip names its origin, " · this image", " · prompt" or " · recipe" (D146), in
the side panel, the viewer and History's media inspector (D145); Tags & categories
counts images on each category row and lists image-only values (D143), and Manage
category previews and renames image rows too (D144). Prompt categories are changed in
the Prompt library editor and on the legacy admin page's "Set Category". Concept
comparisons, at 1600x1100 and 390x844 in both themes: `images-concept.spec.mjs` (the
Set category dialog and the options), `inspector-concept.spec.mjs` (the chip),
`history-concept.spec.mjs` (History's chip) and `vocabulary-visual.spec.mjs` (rows and
preview, with `vocabulary-canonical-comparison.mjs`). Behavior: `workspace-images`,
`full-inspector`, `workspace-history` and `workspace-vocabulary` specs, and
`tests/js/workspace-images-category.test.js`, `workspace-file-inspector.test.js`,
`workspace-vocabulary-state.test.js`, `workspace-toolbar.test.js`.
Image annotations schema (#249, `database/annotation_schema.py`): the group
`media_annotation_metadata` (one row: schema version, `image_index_revision`),
`media_annotation_files` (one private locator per physical file,
@@ -5408,7 +5073,7 @@ which used to save `null` under a link.
Primitive or through `PreviewAny`, for PromptManager, PromptManagerText, Focused, Split
and the Pair (positive, negative, both, neither linked, and a Primitive negative). A
wildcard Primitive on PromptManager (queued as written) and Focused (one choice) saves
and counts what was queued. The linked text is saved, counted once per job, recorded as
and counts what was queued. The linked text is saved, counted once, recorded as
the run's raw positive/negative, and after a scan the run's image links to it. Text
only known at run time (`PreviewAny`) cannot be read from the image; the case only
requires that it is never linked to another prompt (its attribution from the recorded
@@ -5422,39 +5087,6 @@ PM_COMFY_ROOT=... PM_TEST_PYTHON=... npm run --prefix tools/vendor test:comfyui
PM_COMFY_ROOT=... PM_TEST_PYTHON=... npm run --prefix tools/vendor test:executions -- execution-text-link
```
## Cached runtime-linked runs (#347)
Text only known at run time (a `PreviewAny` or LLM link, a list node) is counted by the
node when it executes; the queue hook cannot read it. When ComfyUI serves such a node from
its output cache, `utils/cached_runs.py` counts the job instead: every execution records
a receipt (digests of its STRING outputs and hashes of the text it stores, per run key and
queue item), and on `execution_cached` the receipt whose digests equal the cache entry's
text is replayed once. The role (positive, negative, which Pair outputs are used) and what
the hook already saw are recomputed from the job's own graph, and the node and replay
decide through the same `own_counts`.
- Fail closed: no receipt in this process (after a restart, an external cache provider),
changed cached text, more than 64 list values, a node the job's outputs do not need, or
a host without the queue/executor interfaces: nothing is counted, as before #347.
- A cached node counts at job start like the hook: a job failing later still counts it.
A node that executes after its replay in the same job (RAM-pressure freed the entry)
skips its own count but still consumes the pending first-use catch-up.
- No backfill: earlier cached jobs keep their counts. The history page's "tracked runs"
can stay below the Library count for these prompts (#403).
- Tests: `tests/test_cached_runs.py` (whole jobs with the real nodes, a real database,
fake host caches); `tests/comfyui/execution-text-link.spec.mjs` "#347" cases on the
managed host: PreviewAny into each node kind, 3 jobs changing only the sampler seed,
count 3 in both renderers; in one renderer the Classic, LRU, RAM-pressure and no-cache
modes, A, B, A from the owned `PMCycledText` under LRU, typed text, a downstream
failure, a "run selected output" job that does not need the node, and a rewire to the
negative input. The RAM-pressure eviction path cannot be forced on a real host and is
covered by the Python tests.
```sh
python3 -m unittest discover -s tests -p "test_cached_runs.py"
PM_COMFY_ROOT=... PM_TEST_PYTHON=... npm run --prefix tools/vendor test:executions -- execution-text-link --grep "#347"
```
## Durable execution ledger
`npm run --prefix tools/vendor test:executions` runs 52 real execution cases
@@ -5874,20 +5506,6 @@ rotates the epoch, so a pre-restore `expected_epoch` is refused with `storage-ch
Owner rows and their timestamps never change on a tag command; a category apply only
touches rows carrying the exact source.
Image categories (#332 §7, MC9, MC10, DC7): each `GET categories` row also carries
`image_count`, the stored `image_categories` rows holding that exact value (files
missing on disk included). A value prompts and images share appears once, ordered by
its oldest positive; a value only images hold appears once after every prompt-backed
value (`first_pid` 9223372036854775807 in the keyset cursor), ordered by value, with
`prompt_count` 0. `/library/options` keeps its own query and never lists image-only
values. Category preview and apply report `impact.images` (rows holding the source),
the target exists when an image row holds it, an image-only value can be renamed, the
apply renames the image rows in the same transaction (file keys and rowids kept, so a
merge cannot collide), and the impact token hashes the sorted image rowids, so an image
row gaining or losing the value between preview and apply is 409
`vocabulary-conflict` with nothing written. Every image-category statement reads
`idx_image_categories_category` (`test_category_reads_use_the_image_category_index`).
Two legacy routes change additively: `GET /tags` now lists unused registry tags, and
`/tags/suggest` without context appends unused names at count 0. Unused tags become
deletable only through the native Delete tag (vocabulary UI PR), not on the legacy
@@ -5905,18 +5523,13 @@ Deploying this backend needs a ComfyUI restart (Python and route changes).
longer hands off to the legacy tag manager. That page stays reachable from Overview's
"Open existing tag manager" card for tag search and sort, the untagged entry and
multi-tag AND/OR browsing; bulk prompt actions (delete, tag, category) stay on the
legacy admin page. A prompt's category is changed in the Prompt library editor or with
the legacy admin page's "Set Category" (`POST /prompt_manager/bulk/category`, unchanged);
the Images selection bar's Set category sets images' own categories only (D140, #332),
and Manage category renames a value on prompts, recipes and images together (D144). The screen consumes only the `/prompt_manager/vocabulary/*` routes
legacy admin page. The screen consumes only the `/prompt_manager/vocabulary/*` routes
above and adds no route, so PR B needs no ComfyUI restart: a browser refresh is enough.
Screen: a `+ Create tag` heading action, a Tag vocabulary table (columns Tag, Prompts,
Images counting the files that carry the tag (`image_count`, #249 C2), and Actions with Browse and
Rename / merge), and a Categories panel whose category rows end with the Manage
category row (Review). A category row reads "N positive prompts", "N positive prompts ·
M images" when images hold the value as their own category too, or "M images" for a
value only images hold (D143, #332). Lists load 100 rows at a time, append when scrolled into view
category row (Review). Lists load 100 rows at a time, append when scrolled into view
and keep `Load more tags` / `Load more categories` while more remain. Prompts counts
are the Library Browse totals.
@@ -5959,34 +5572,7 @@ Recorded deviations from the concept (maintainer acceptance): bounded lists with
Load more; title "Tags"; Manage categories adds a Category select and Preview before
Apply; "Apply" instead of "Apply in prototype"; Delete tag… inside Rename or merge;
Browse sets the exact filter; singular "1 positive prompt"; the production list appends
fixture-only tags after the concept's sample tags.
- **D143** (#332): category rows count the images whose own category is the value
(`image_count`, the stored rows, missing files included): "N positive prompts · M
images" beside prompts, "M images" for an image-only value, which follows every
prompt-backed row (by value) before Manage category. Browse on an image-only row opens
Images with exactly that Category filter (its Library would be empty), as a fresh
Images visit that becomes the Images return state, the way Browse seeds the Library's; every other
row opens the Library as before. Manage category's select lists image-only values
last, so past 1,000 categories they are the first left out (the dialog says the list is
capped).
- **D144** (#332): Manage category's preview counts the images holding the value ("“X”
is used by N positive prompts and M images. Apply renames it to “Y”."; D9's sentence
with "M images" added when above 0), an image-only value can be renamed, and Apply
renames the image rows in the same transaction; an image row gaining or losing the
value between Preview and Apply is 409 `vocabulary-conflict` with nothing written.
- `category rows and the Manage category preview match the concept with image
categories` (vocabulary-visual.spec, 1600x1100 and 390x844, both themes) seeds the
fixture with `image_categories: true` ("Landscapes" on two images, "Portraits" on one
image only, rows keyed as Images keys them) and compares the page with D143 mapped
(`mapVocabularyConcept(reference, imageCategories)`, whose row model is checked against
the concept page with those rows added at runtime) and the Manage dialog after
Preview with D9 and D144 (`mapVocabularyDialog`'s `preview`). Before the preview is
measured the pointer leaves the pressed Apply and focus returns to ×, the state the
opening comparison in workspace-vocabulary.spec measures. Unmapped, the page differs
in the Landscapes row's text and the added Portraits row, the dialog only in the
sentence. The default fixture seeds no image category, so the other vocabulary
comparisons are unchanged. Paired concept/production captures
fixture-only tags after the concept's sample tags. Paired concept/production captures
for both themes and widths are attached as test artifacts; no PNG baseline is added
until acceptance. Not compared: hover states and the dialog focus ring (the page-level
keyboard ring and link ring offsets are asserted). No coverage floor was lowered;
-20
View File
@@ -195,26 +195,6 @@ except Exception as e:
f"Failed to register usage hook: {e}"
)
# A cached node counts what its last execution of the same text counted (#347).
try:
from .py.config import server_instance
from .utils.cached_runs import register_cached_runs
register_cached_runs(server_instance, _db_factory)
except Exception as e:
try:
from .utils.logging_config import get_logger
get_logger("prompt_manager.init").error(
f"Failed to register cached-run counting: {e}"
)
except Exception:
import logging
logging.getLogger("prompt_manager.init").error(
f"Failed to register cached-run counting: {e}"
)
def _history_db_factory() -> "PromptDatabase":
"""Initialize history on its worker even if monitoring initialization failed."""
+5 -42
View File
@@ -9,12 +9,8 @@ IMMEDIATE`` transaction, so no row ever names a deleted file:
already links the kept copy: then its saved-pair links join that record
and it is deleted;
- its favorite;
- its collection memberships (#331), joined onto the kept copy's as their
union under the same keys a favorite moves by;
- its image tags (``media_image_tags`` of its byte record), joined onto the
kept copy's byte record as their union, in order;
- its own image category (#332), copied to the kept copy when that copy has
none; the removed copy's row is kept, so a give-back loses nothing;
- its pending Needs review items, settled: only an item still ``pending`` at
revision 0 with no decision row is removed (``review._SETTLE_ITEM``), so a
decided item is never touched.
@@ -26,8 +22,6 @@ import os
from .annotation_schema import BYTES_TABLE, FILES_TABLE
from .annotations import _BUMP_INDEX
from .image_categories import COPY_SQL as COPY_CATEGORY_SQL
from .image_categories import copy_image_category
from .models import normalize_image_path, utc_now_iso
from .review import _BUMP_REVIEW_INDEX, _PENDING_OF_LOCATOR, _SETTLE_ITEM
from .review_changes import notify_review_change
@@ -56,15 +50,6 @@ INSERT_FAVORITE_SQL = "INSERT OR IGNORE INTO image_favorites VALUES (?, ?, ?)"
DELETE_FAVORITE_SQL = (
"DELETE FROM image_favorites WHERE root_key = ? AND relative_path = ?"
)
COPY_MEMBERSHIPS_SQL = (
"INSERT OR IGNORE INTO image_collection_items "
"(collection_id, root_key, relative_path, added_at) "
"SELECT collection_id, ?, ?, added_at FROM image_collection_items "
"WHERE root_key = ? AND relative_path = ?"
)
DELETE_MEMBERSHIPS_SQL = (
"DELETE FROM image_collection_items WHERE root_key = ? AND relative_path = ?"
)
COPY_BYTE_SQL = (
f"SELECT b.id FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b " # nosec B608
"ON b.file_id=f.id WHERE f.locator=? AND b.sha256=?"
@@ -88,12 +73,9 @@ FILE_LINK_SQL = (
DELETE_LINK_SQL,
FAVORITE_SQL,
DELETE_FAVORITE_SQL,
COPY_MEMBERSHIPS_SQL,
DELETE_MEMBERSHIPS_SQL,
COPY_BYTE_SQL,
COPY_TAG_COUNT_SQL,
PENDING_COUNT_SQL,
COPY_CATEGORY_SQL,
)
@@ -153,17 +135,12 @@ class DuplicateCopyOperations:
by), ``targets`` (``{removed spelling: kept spelling}``, the spelling
a moved record takes; else ``kept_path``), ``favorites`` (``(root key,
removed key, kept root key, kept key)`` per listed root holding the
file; they move its favorite and its collection memberships),
``removed_locator``, ``kept_locator``, ``sha256`` and the kept copy's
``kept_size`` and ``kept_mtime_ns``; ``categories``: ``(removed file
key, kept file key)`` pairs of the image categories (#332). Returns
``{moved, merged, favorites, tags, review, collections, categories}``
(``collections``: the memberships inserted on the kept copy;
``categories``: the image categories copied to the kept copy).
file), ``removed_locator``,
``kept_locator``, ``sha256`` and the kept copy's ``kept_size`` and
``kept_mtime_ns``. Returns ``{moved, merged, favorites, tags,
review}``.
"""
counts = dict.fromkeys(
("moved", "merged", "favorites", "tags", "review", "collections"), 0
)
counts = dict.fromkeys(("moved", "merged", "favorites", "tags", "review"), 0)
kept_keys = [normalize_image_path(path) for path in move["kept"]]
with self._pair_transaction() as conn:
for spelled in dict.fromkeys(move["removed"]):
@@ -196,12 +173,7 @@ class DuplicateCopyOperations:
counts["moved"] += 1
for favorite in move.get("favorites", ()):
counts["favorites"] += _move_favorite(conn, *favorite)
counts["collections"] += _move_memberships(conn, *favorite)
counts["tags"] = self._move_copy_tags(conn, move)
counts["categories"] = sum(
copy_image_category(conn, removed, kept)
for removed, kept in move.get("categories", ())
)
for (item_id,) in conn.execute(
_PENDING_OF_LOCATOR, (move["removed_locator"],)
).fetchall():
@@ -241,12 +213,3 @@ def _move_favorite(conn, root_key, removed_key, kept_root, kept_key):
conn.execute(INSERT_FAVORITE_SQL, (kept_root, kept_key, row[0]))
conn.execute(DELETE_FAVORITE_SQL, (root_key, removed_key))
return 1
def _move_memberships(conn, root_key, removed_key, kept_root, kept_key):
"""Copy a removed copy's memberships to the kept copy; the rows inserted."""
inserted = conn.execute(
COPY_MEMBERSHIPS_SQL, (kept_root, kept_key, root_key, removed_key)
).rowcount
conn.execute(DELETE_MEMBERSHIPS_SQL, (root_key, removed_key))
return inserted
-249
View File
@@ -1,249 +0,0 @@
"""Image categories (#332): an image file's own category.
An image shows its own category when it has one, otherwise the category its
link supplies (its recipe's, else its positive prompt's); this table holds
only the own values. A row is keyed by the file key: the deepest configured
output source that contains the file (its real, case-normalized path) and the
path below it, case-folded where the OS folds case
(``py/api/images.py`` ``image_category_key``). Without nested sources the key
equals a favorite's key. Nothing follows a moved file: its row stays and is
hidden until a file appears at the same path again.
Writes validate strictly (the route strips the value and applies
``validate_category``). Startup checks the structure only, so a value saved
under older rules never stops the library opening; restore also checks the
values. Optional in old backups, required once a database is current.
"""
import re
import sqlite3
from typing import Iterable
TABLE = "image_categories"
INDEX = "idx_image_categories_category"
MAX_KEY_LENGTH = 4096
MAX_CATEGORY_LENGTH = 100
_DDL = (
f"CREATE TABLE {TABLE} (root_key TEXT NOT NULL, relative_path TEXT NOT NULL, "
"category TEXT NOT NULL, updated_at TEXT NOT NULL, "
"PRIMARY KEY (root_key, relative_path))"
)
_INDEX_DDL = f"CREATE INDEX {INDEX} ON {TABLE}(category)"
_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1"
_COLUMNS = (
("root_key", "TEXT", 1, 1),
("relative_path", "TEXT", 1, 2),
("category", "TEXT", 1, 0),
("updated_at", "TEXT", 1, 0),
)
# A category may not hold a character below U+0020 (validate_category).
_CONTROL_GLOB = "*[" + "".join(chr(code) for code in range(1, 32)) + "]*"
CATEGORY_OF_SQL = (
f"SELECT category FROM {TABLE} WHERE root_key = ? AND relative_path = ?"
)
CATEGORIES_OF_ROOT_SQL = (
f"SELECT relative_path, category FROM {TABLE} WHERE root_key = ?"
)
UPSERT_SQL = (
f"INSERT INTO {TABLE} (root_key, relative_path, category, updated_at) "
"VALUES (?, ?, ?, ?) ON CONFLICT(root_key, relative_path) "
"DO UPDATE SET category = excluded.category, updated_at = excluded.updated_at"
)
DELETE_SQL = f"DELETE FROM {TABLE} WHERE root_key = ? AND relative_path = ?"
# The duplicate cleanup's copy (#332 MC11): the removed copy's row is copied
# to the kept copy only when the kept copy has none; it is never deleted.
COPY_SQL = (
f"INSERT OR IGNORE INTO {TABLE} (root_key, relative_path, category, updated_at) "
f"SELECT ?, ?, category, updated_at FROM {TABLE} "
"WHERE root_key = ? AND relative_path = ?"
)
STATEMENTS = (CATEGORY_OF_SQL, CATEGORIES_OF_ROOT_SQL, UPSERT_SQL, DELETE_SQL, COPY_SQL)
def _compact(sql: str) -> str:
return re.sub(r"\s+", "", sql or "").lower()
def _structure_error(
conn: sqlite3.Connection, allow_missing: bool, allow_missing_index: bool = False
) -> str | None:
objects = conn.execute(
"SELECT type,name,sql FROM sqlite_master WHERE name=? "
"OR (type IN ('index','trigger') AND tbl_name=?)",
(TABLE, TABLE),
).fetchall()
if not objects:
return None if allow_missing else "Missing image categories schema"
tables = [sql for kind, _name, sql in objects if kind == "table"]
others = {(kind, name) for kind, name, _sql in objects if kind != "table"}
expected = {("index", _AUTOINDEX), ("index", INDEX)}
# Before its install, a table without the category index is repaired by
# CREATE INDEX IF NOT EXISTS rather than refused.
if allow_missing_index and others == {("index", _AUTOINDEX)}:
expected = others
if len(tables) != 1 or others != expected:
return "Image categories name is reserved by an unsupported schema object"
if _compact(tables[0]) != _compact(_DDL):
return "Image categories has an invalid table definition"
index_sql = next((sql for kind, name, sql in objects if name == INDEX), None)
if ("index", INDEX) in expected and _compact(index_sql) != _compact(_INDEX_DDL):
return "Image categories has an invalid index"
autoindex_sql = next(sql for kind, name, sql in objects if name == _AUTOINDEX)
if autoindex_sql is not None:
return "Image categories has an invalid index"
columns = tuple(
(row[1], row[2].upper(), row[3], row[5])
for row in conn.execute(f"PRAGMA table_info({TABLE})")
)
if columns != _COLUMNS:
return "Image categories has invalid columns"
return None
def _values_error(conn: sqlite3.Connection) -> str | None:
# Bytes bound characters from above (UTF-8 is at most 4 bytes each).
limit = MAX_KEY_LENGTH * 4
if conn.execute(
f"SELECT 1 FROM {TABLE} WHERE typeof(root_key) != 'text' " # nosec B608
"OR typeof(relative_path) != 'text' OR typeof(category) != 'text' "
"OR typeof(updated_at) != 'text' "
f"OR length(CAST(root_key AS BLOB)) > {limit} "
f"OR length(CAST(relative_path AS BLOB)) > {limit} "
"OR length(CAST(updated_at AS BLOB)) > 64 "
"OR instr(CAST(category AS BLOB), x'00') > 0 "
f"OR length(category) < 1 OR length(category) > {MAX_CATEGORY_LENGTH} "
"OR category != TRIM(category) OR category GLOB ? LIMIT 1",
(_CONTROL_GLOB,),
).fetchone():
return "Image categories contains invalid values"
for root, path in conn.execute(
f"SELECT CAST(root_key AS BLOB), CAST(relative_path AS BLOB) " # nosec B608
f"FROM {TABLE} WHERE length(CAST(root_key AS BLOB)) > {MAX_KEY_LENGTH} "
f"OR length(CAST(relative_path AS BLOB)) > {MAX_KEY_LENGTH}"
):
for raw in (root, path):
if len(bytes(raw).decode("utf-8", "replace")) > MAX_KEY_LENGTH:
return "Image categories contains invalid values"
return None
def image_categories_schema_error(
conn: sqlite3.Connection,
*,
allow_missing: bool = True,
check_values: bool = True,
) -> str | None:
"""Reject an unsupported image categories schema (and, optionally, values).
Reads only. A missing table is acceptable for backups that predate it
unless ``allow_missing`` is false. ``check_values`` also refuses a stored
value the write path could never have written.
"""
error = _structure_error(conn, allow_missing)
if error is not None or not check_values:
return error
exists = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
).fetchone()
return _values_error(conn) if exists else None
def migrate_image_categories(conn: sqlite3.Connection) -> None:
"""Install the table and its index atomically; structure is checked
before and after. Reads only ``sqlite_master`` and this table's PRAGMAs
and writes only its own two objects (no backfill)."""
conn.execute("SAVEPOINT image_categories_schema")
try:
error = _structure_error(conn, True, allow_missing_index=True)
if error:
raise sqlite3.DatabaseError(error)
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
conn.execute(
_INDEX_DDL.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS", 1)
)
error = image_categories_schema_error(
conn, allow_missing=False, check_values=False
)
if error:
raise sqlite3.DatabaseError(error)
conn.execute("RELEASE image_categories_schema")
except BaseException:
conn.execute("ROLLBACK TO image_categories_schema")
conn.execute("RELEASE image_categories_schema")
raise
def copy_image_category(conn: sqlite3.Connection, removed_key, kept_key) -> int:
"""Give the kept copy the removed copy's category when it has none.
Runs inside the caller's transaction; never deletes the removed copy's
row, so a give-back or a failed delete loses nothing. Returns 1 when a
row was inserted, else 0.
"""
if tuple(removed_key) == tuple(kept_key):
return 0
return max(conn.execute(COPY_SQL, (*kept_key, *removed_key)).rowcount, 0)
class ImageCategoryOperations:
"""PromptDatabase mixin: set, clear and read images' own categories.
Never reads or writes ``prompts``, ``prompt_pairs`` or
``generated_images``.
"""
def image_category(self, root_key: str, relative_key: str) -> str | None:
"""One file's own category, or None."""
row = (
self.model.get_connection()
.execute(CATEGORY_OF_SQL, (root_key, relative_key))
.fetchone()
)
return row[0] if row else None
def own_categories_of(self, root_keys: Iterable[str]) -> dict:
"""``{(root key, relative key): category}`` under the given sources.
One primary-key range read per source.
"""
found = {}
conn = self.model.get_connection()
for root_key in dict.fromkeys(root_keys):
for relative, category in conn.execute(CATEGORIES_OF_ROOT_SQL, (root_key,)):
found[(root_key, relative)] = category
return found
def set_image_categories(self, file_keys, category: str) -> dict:
"""Set *category* on every file key, in one transaction.
``file_keys``: ``(root key, relative key)`` pairs from any number of
sources. Returns ``{updated, unchanged}``; a file that already had
exactly this value is ``unchanged`` and keeps its ``updated_at``.
"""
from .models import utc_now_iso # models imports this module's schema
keys = list(dict.fromkeys(tuple(key) for key in file_keys))
counts = {"updated": 0, "unchanged": 0}
now = utc_now_iso()
with self._pair_transaction() as conn:
for key in keys:
row = conn.execute(CATEGORY_OF_SQL, key).fetchone()
if row is not None and row[0] == category:
counts["unchanged"] += 1
continue
conn.execute(UPSERT_SQL, (*key, category, now))
counts["updated"] += 1
return counts
def clear_image_categories(self, file_keys) -> int:
"""Remove the own category of every file key (one transaction).
Returns the number of rows removed.
"""
keys = list(dict.fromkeys(tuple(key) for key in file_keys))
removed = 0
with self._pair_transaction() as conn:
for key in keys:
removed += conn.execute(DELETE_SQL, key).rowcount
return removed
-490
View File
@@ -1,490 +0,0 @@
"""Collections (#331): named, hand-picked groups of image files.
A member names its file exactly as a favorite does (``image_favorites.py``):
the source folder's real, case-normalized path (``root_key``) and the path
below it (``relative_path``), both server-side only. A member whose file is
moved, renamed or deleted outside PromptManager is kept and simply not listed;
nothing follows a file to a new place.
Names are trimmed with each run of whitespace collapsed to one space, 1 to 100
characters without control characters; ``name_key`` (NFC of the case-folded
NFC name) is UNIQUE, so names are unique ignoring case and Unicode form.
The two tables install like ``image_favorites`` and ``saved_views``: optional
in old backups, required once a database is current, created empty with no
backfill. Startup checks only the structure, so a value saved under older
rules never stops the library from opening; restore checks the values too,
but never recomputes ``name_key`` (case tables differ between Python
versions).
"""
import re
import sqlite3
import unicodedata
from typing import Optional
from .image_favorites import MAX_KEY_LENGTH
COLLECTIONS_TABLE = "image_collections"
ITEMS_TABLE = "image_collection_items"
FILE_INDEX = "idx_image_collection_items_file"
TABLES = (COLLECTIONS_TABLE, ITEMS_TABLE)
MAX_NAME_LENGTH = 100
_TIMESTAMP_BYTES = 64
# A name key's byte bound: case folding expands a character to at most
# three, each at most 4 UTF-8 bytes.
_MAX_KEY_BYTES = MAX_NAME_LENGTH * 3 * 4
_WHITESPACE = re.compile(r"\s+")
_TABLE_DDL = {
COLLECTIONS_TABLE: (
f"CREATE TABLE {COLLECTIONS_TABLE} (id INTEGER PRIMARY KEY AUTOINCREMENT, "
"name TEXT NOT NULL, name_key TEXT NOT NULL UNIQUE, "
"created_at TEXT NOT NULL)"
),
ITEMS_TABLE: (
f"CREATE TABLE {ITEMS_TABLE} (collection_id INTEGER NOT NULL REFERENCES "
f"{COLLECTIONS_TABLE}(id) ON DELETE CASCADE, root_key TEXT NOT NULL, "
"relative_path TEXT NOT NULL, added_at TEXT NOT NULL, "
"PRIMARY KEY (collection_id, root_key, relative_path))"
),
}
_INDEX_DDL = f"CREATE INDEX {FILE_INDEX} ON {ITEMS_TABLE}(root_key, relative_path)"
# (name, declared type, not null, default, primary key position)
_COLUMNS = {
COLLECTIONS_TABLE: (
("id", "INTEGER", 0, None, 1),
("name", "TEXT", 1, None, 0),
("name_key", "TEXT", 1, None, 0),
("created_at", "TEXT", 1, None, 0),
),
ITEMS_TABLE: (
("collection_id", "INTEGER", 1, None, 1),
("root_key", "TEXT", 1, None, 2),
("relative_path", "TEXT", 1, None, 3),
("added_at", "TEXT", 1, None, 0),
),
}
# (column, parent table, parent column, on delete)
_FOREIGN_KEYS = {
COLLECTIONS_TABLE: set(),
ITEMS_TABLE: {("collection_id", COLLECTIONS_TABLE, "id", "CASCADE")},
}
class CollectionInvalid(ValueError):
"""A name, id or target breaks the write rules (a 400)."""
def _has_control(text: str) -> bool:
"""Any Unicode control character (category Cc: C0, DEL and C1)."""
return any(unicodedata.category(char) == "Cc" for char in text)
def normalize_collection_name(name: object) -> str:
"""The stored spelling of *name*: trimmed, whitespace collapsed, checked.
Raises ``CollectionInvalid`` unless the result is 1 to 100 characters
without control characters.
"""
if not isinstance(name, str):
raise CollectionInvalid("Name must be a string")
collapsed = _WHITESPACE.sub(" ", name).strip()
if not 1 <= len(collapsed) <= MAX_NAME_LENGTH:
raise CollectionInvalid(f"Name must be 1 to {MAX_NAME_LENGTH} characters")
if _has_control(collapsed):
raise CollectionInvalid("Name contains control characters")
return collapsed
def collection_name_key(name: str) -> str:
"""The uniqueness key of a name: NFC of the case-folded NFC spelling."""
return unicodedata.normalize("NFC", unicodedata.normalize("NFC", name).casefold())
def _compact(sql: str) -> str:
return re.sub(r"\s+", "", sql).lower()
def _expected_catalog() -> set:
objects = {("table", name, _compact(sql)) for name, sql in _TABLE_DDL.items()}
objects.add(("index", FILE_INDEX, _compact(_INDEX_DDL)))
objects |= {("index", f"sqlite_autoindex_{table}_1", "") for table in TABLES}
return objects
def _catalog(conn: sqlite3.Connection) -> list:
# sqlite_sequence (AUTOINCREMENT) is shared with other tables and ignored.
return conn.execute(
"SELECT type,name,sql FROM sqlite_master "
"WHERE name IN (?,?,?) OR tbl_name IN (?,?)",
(*TABLES, FILE_INDEX, *TABLES),
).fetchall()
def _structure_error(conn: sqlite3.Connection) -> Optional[str]:
for table, expected in _COLUMNS.items():
columns = tuple(
(row[1], row[2].upper(), row[3], row[4], row[5])
for row in conn.execute(f"PRAGMA table_info({table})")
)
if columns != expected:
return f"Image collections table {table} has invalid columns"
foreign_keys = {
(row[3], row[2], row[4], row[6])
for row in conn.execute(f"PRAGMA foreign_key_list({table})")
}
if foreign_keys != _FOREIGN_KEYS[table]:
return f"Image collections table {table} has invalid foreign keys"
return None
def _decoded_too_long(conn, sql: str, limit: int) -> bool:
"""Whether a byte-bounded candidate row decodes to over *limit* characters."""
for row in conn.execute(sql):
for raw in row:
if len(bytes(raw).decode("utf-8", "replace")) > limit:
return True
return False
def _value_error(conn: sqlite3.Connection) -> Optional[str]:
# Only this module's constants are interpolated into the SQL below. Bytes
# bound characters from above (UTF-8 is at most 4 bytes each); the few
# candidates over the character bound are measured decoded in Python.
if conn.execute(
f"SELECT 1 FROM {COLLECTIONS_TABLE} WHERE " # nosec B608
"typeof(id) != 'integer' OR typeof(name) != 'text' "
"OR typeof(name_key) != 'text' OR typeof(created_at) != 'text' "
f"OR length(CAST(name AS BLOB)) > {MAX_NAME_LENGTH * 4} "
"OR length(CAST(name_key AS BLOB)) = 0 "
f"OR length(CAST(name_key AS BLOB)) > {_MAX_KEY_BYTES} "
f"OR length(CAST(created_at AS BLOB)) > {_TIMESTAMP_BYTES} LIMIT 1"
).fetchone():
return "Image collections contains invalid values"
# The key is not recomputed: Unicode case tables differ between Python
# versions, and a key written by another version must stay restorable.
# Its UNIQUE constraint keeps names distinct; writes compute it.
for (name,) in conn.execute(f"SELECT name FROM {COLLECTIONS_TABLE}"): # nosec B608
if (
not 1 <= len(name) <= MAX_NAME_LENGTH
or _has_control(name)
or name != name.strip()
):
return "Image collections contains invalid names"
limit = MAX_KEY_LENGTH * 4
if conn.execute(
f"SELECT 1 FROM {ITEMS_TABLE} WHERE " # nosec B608
"typeof(collection_id) != 'integer' OR typeof(root_key) != 'text' "
"OR typeof(relative_path) != 'text' OR typeof(added_at) != 'text' "
f"OR length(CAST(root_key AS BLOB)) > {limit} "
f"OR length(CAST(relative_path AS BLOB)) > {limit} "
f"OR length(CAST(added_at AS BLOB)) > {_TIMESTAMP_BYTES} LIMIT 1"
).fetchone() or _decoded_too_long(
conn,
f"SELECT CAST(root_key AS BLOB), CAST(relative_path AS BLOB) " # nosec B608
f"FROM {ITEMS_TABLE} WHERE length(CAST(root_key AS BLOB)) > {MAX_KEY_LENGTH} "
f"OR length(CAST(relative_path AS BLOB)) > {MAX_KEY_LENGTH}",
MAX_KEY_LENGTH,
):
return "Image collections contains invalid members"
if conn.execute(f"PRAGMA foreign_key_check({ITEMS_TABLE})").fetchone():
return "Image collections contains members of a missing collection"
return None
def image_collections_schema_error(
conn: sqlite3.Connection,
*,
allow_missing: bool = True,
check_values: bool = True,
) -> Optional[str]:
"""Reject partial or unsupported collections schemas and invalid rows.
Reads only: nothing is ever repaired. A database without either table is
acceptable for backups that predate them unless ``allow_missing`` is
false. ``check_values=False`` (live startup) checks the structure only.
"""
objects = _catalog(conn)
if not objects:
return None if allow_missing else "Missing image collections schema"
present = {name for kind, name, _sql in objects if kind == "table"}
if present != set(TABLES):
return "Incomplete image collections schema"
actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
if actual != _expected_catalog():
return "Image collections has an invalid schema definition"
error = _structure_error(conn)
if error or not check_values:
return error
return _value_error(conn)
def migrate_image_collections(conn: sqlite3.Connection) -> None:
"""Install both tables atomically; validate before and after creating them."""
conn.execute("SAVEPOINT image_collections_schema")
try:
error = image_collections_schema_error(conn, check_values=False)
if error:
raise sqlite3.DatabaseError(error)
for sql in _TABLE_DDL.values():
conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
conn.execute(_INDEX_DDL.replace("INDEX", "INDEX IF NOT EXISTS", 1))
error = image_collections_schema_error(
conn, allow_missing=False, check_values=False
)
if error:
raise sqlite3.DatabaseError(error)
conn.execute("RELEASE image_collections_schema")
except BaseException:
conn.execute("ROLLBACK TO image_collections_schema")
conn.execute("RELEASE image_collections_schema")
raise
class CollectionMissing(LookupError):
"""No collection has the id (a 404 ``collection-not-found``)."""
class CollectionNameTaken(ValueError):
"""Another collection already has the name's key (a 409 ``name-taken``)."""
def __init__(self, collection_id: int):
super().__init__("Another collection has this name")
self.collection_id = collection_id
class CollectionFilesMissing(LookupError):
"""A new name with nothing to insert (a 404 ``files-not-found``)."""
LIST_SQL = f"SELECT id, name FROM {COLLECTIONS_TABLE} ORDER BY name_key, id"
GET_SQL = f"SELECT id, name FROM {COLLECTIONS_TABLE} WHERE id = ?"
BY_KEY_SQL = f"SELECT id, name FROM {COLLECTIONS_TABLE} WHERE name_key = ?"
MEMBERS_SQL = (
f"SELECT relative_path FROM {ITEMS_TABLE} "
"WHERE collection_id = ? AND root_key = ?"
)
ROOT_MEMBERS_SQL = (
f"SELECT collection_id, relative_path FROM {ITEMS_TABLE} WHERE root_key = ?"
)
COUNTS_SQL = (
f"SELECT c.id, (SELECT COUNT(*) FROM {ITEMS_TABLE} i "
f"WHERE i.collection_id = c.id) FROM {COLLECTIONS_TABLE} c"
)
FILE_SQL = (
f"SELECT collection_id FROM {ITEMS_TABLE} "
"WHERE root_key = ? AND relative_path = ? ORDER BY collection_id"
)
# Every read, for the EXPLAIN QUERY PLAN check: none scans the items table and
# only the two whole-list reads (LIST_SQL, COUNTS_SQL) scan the collections.
READ_SQL = (
LIST_SQL,
GET_SQL,
BY_KEY_SQL,
MEMBERS_SQL,
ROOT_MEMBERS_SQL,
COUNTS_SQL,
FILE_SQL,
)
_INSERT_ITEM_SQL = f"INSERT OR IGNORE INTO {ITEMS_TABLE} VALUES (?, ?, ?, ?)"
_DELETE_ITEM_SQL = (
f"DELETE FROM {ITEMS_TABLE} "
"WHERE collection_id = ? AND root_key = ? AND relative_path = ?"
)
def _collection_id(value: object) -> int:
if type(value) is not int or not 1 <= value <= 999_999_999:
raise CollectionInvalid("Collection id must be a positive integer")
return value
def _items(items: object) -> list:
"""Distinct ``(root key, relative key)`` pairs, in order."""
if not isinstance(items, (list, tuple)):
raise CollectionInvalid("Items must be a list")
found = []
for item in items:
if (
not isinstance(item, (list, tuple))
or len(item) != 2
or not all(
isinstance(part, str) and 1 <= len(part) <= MAX_KEY_LENGTH
for part in item
)
):
raise CollectionInvalid("Each item is a root key and a relative key")
found.append((item[0], item[1]))
return list(dict.fromkeys(found))
def _row(row) -> dict:
return {"id": row[0], "name": row[1]}
class ImageCollectionOperations:
"""PromptDatabase mixin: collections and their members.
Each write runs in its own ``BEGIN IMMEDIATE`` transaction
(``_pair_transaction``), so two tabs creating one name are serialized.
"""
def list_collections(self) -> list[dict]:
"""Every collection ``{id, name}`` in name-key order, then id."""
rows = self.model.get_connection().execute(LIST_SQL)
return [_row(row) for row in rows]
def get_collection(self, collection_id: int) -> Optional[dict]:
"""``{id, name}`` of one collection, or None."""
row = (
self.model.get_connection()
.execute(GET_SQL, (_collection_id(collection_id),))
.fetchone()
)
return _row(row) if row else None
def collection_exists(self, collection_id: int) -> bool:
return self.get_collection(collection_id) is not None
def collection_members(self, collection_id: int, root_key: str) -> set[str]:
"""The relative keys stored for one collection under one source."""
rows = self.model.get_connection().execute(
MEMBERS_SQL, (_collection_id(collection_id), root_key)
)
return {path for (path,) in rows}
def root_collection_members(self, root_key: str) -> dict[int, set[str]]:
"""``{collection id: relative keys}`` stored under one source."""
found: dict[int, set[str]] = {}
for collection_id, path in self.model.get_connection().execute(
ROOT_MEMBERS_SQL, (root_key,)
):
found.setdefault(collection_id, set()).add(path)
return found
def collection_count_snapshot(self, root_keys: list) -> tuple[dict, dict]:
"""``({root key: {id: relative keys}}, {id: stored rows})`` read in
one read transaction, so a write between the reads cannot skew them.
"""
conn = self.model.get_connection()
if conn.in_transaction:
raise RuntimeError("Collection counts require their own transaction")
conn.execute("BEGIN")
try:
members = {}
for root_key in dict.fromkeys(root_keys):
found: dict[int, set[str]] = {}
for collection_id, path in conn.execute(ROOT_MEMBERS_SQL, (root_key,)):
found.setdefault(collection_id, set()).add(path)
members[root_key] = found
stored = {cid: count for cid, count in conn.execute(COUNTS_SQL)}
finally:
conn.rollback() # it only read
return members, stored
def collection_counts(self) -> dict[int, int]:
"""``{collection id: stored rows}`` for every collection."""
rows = self.model.get_connection().execute(COUNTS_SQL)
return {collection_id: count for collection_id, count in rows}
def file_collections(self, root_key: str, relative_path: str) -> list[int]:
"""Ids of the collections holding one file."""
rows = self.model.get_connection().execute(FILE_SQL, (root_key, relative_path))
return [collection_id for (collection_id,) in rows]
def add_to_collection(
self, *, items: object, collection_id: object = None, name: object = None
) -> dict:
"""Add files to an existing id or a name; create the name when needed.
A name whose key exists uses that collection; otherwise a collection
is created only when at least one item is inserted. Returns
``{collection, created, added, already}``.
"""
from .models import utc_now_iso # models imports this module's schema
if (collection_id is None) == (name is None):
raise CollectionInvalid("Give either a collection id or a name")
pairs = _items(items)
if name is not None:
spelled = normalize_collection_name(name)
else:
collection_id = _collection_id(collection_id)
now = utc_now_iso()
with self._pair_transaction() as conn:
created = False
if name is not None:
row = conn.execute(
BY_KEY_SQL, (collection_name_key(spelled),)
).fetchone()
if row is None:
if not pairs:
raise CollectionFilesMissing("Nothing to add")
cursor = conn.execute(
f"INSERT INTO {COLLECTIONS_TABLE} " # nosec B608
"(name, name_key, created_at) VALUES (?, ?, ?)",
(spelled, collection_name_key(spelled), now),
)
row, created = (cursor.lastrowid, spelled), True
else:
row = conn.execute(GET_SQL, (collection_id,)).fetchone()
if row is None:
raise CollectionMissing("No such collection")
collection = _row(row)
added = sum(
conn.execute(
_INSERT_ITEM_SQL, (collection["id"], root, path, now)
).rowcount
for root, path in pairs
)
return {
"collection": collection,
"created": created,
"added": added,
"already": len(pairs) - added,
}
def remove_from_collection(self, collection_id: object, items: object) -> dict:
"""Remove files from one collection; ``{collection, removed}``."""
collection_id = _collection_id(collection_id)
pairs = _items(items)
with self._pair_transaction() as conn:
row = conn.execute(GET_SQL, (collection_id,)).fetchone()
if row is None:
raise CollectionMissing("No such collection")
removed = sum(
conn.execute(_DELETE_ITEM_SQL, (collection_id, root, path)).rowcount
for root, path in pairs
)
return {"collection": _row(row), "removed": removed}
def rename_collection(self, collection_id: object, name: object) -> dict:
"""The renamed ``{id, name}``; a case variant of its own name is fine."""
collection_id = _collection_id(collection_id)
spelled = normalize_collection_name(name)
key = collection_name_key(spelled)
with self._pair_transaction() as conn:
if conn.execute(GET_SQL, (collection_id,)).fetchone() is None:
raise CollectionMissing("No such collection")
taken = conn.execute(BY_KEY_SQL, (key,)).fetchone()
if taken is not None and taken[0] != collection_id:
raise CollectionNameTaken(taken[0])
conn.execute(
f"UPDATE {COLLECTIONS_TABLE} SET name = ?, name_key = ? " # nosec B608
"WHERE id = ?",
(spelled, key, collection_id),
)
return {"id": collection_id, "name": spelled}
def delete_collection(self, collection_id: object) -> bool:
"""Delete one collection and its members (never a file). True."""
collection_id = _collection_id(collection_id)
with self._pair_transaction() as conn:
cursor = conn.execute(
f"DELETE FROM {COLLECTIONS_TABLE} WHERE id = ?", # nosec B608
(collection_id,),
)
if cursor.rowcount == 0:
raise CollectionMissing("No such collection")
return True
+2 -21
View File
@@ -22,10 +22,6 @@ from .managed_connection import (
from . import restore_stage
from .library_schema import library_schema_error, migrate_library_state
from .image_favorites import image_favorites_schema_error, migrate_image_favorites
from .image_collections import (
image_collections_schema_error,
migrate_image_collections,
)
from .annotation_schema import annotation_schema_error, migrate_annotation_state
from .autotag_schema import (
autotag_schema_error,
@@ -38,7 +34,6 @@ from .review_schema import (
review_without_annotations_error,
)
from .saved_views import migrate_saved_views, saved_views_schema_error
from .image_categories import image_categories_schema_error, migrate_image_categories
from .workflow_documents import (
DOCUMENT_ID_COLUMNS,
migrate_workflow_documents,
@@ -342,7 +337,6 @@ class PromptModel:
migrate_library_state(conn)
migrate_image_favorites(conn)
migrate_saved_views(conn)
migrate_image_collections(conn)
# A review group without its annotation group is refused before the
# annotation install could make it look valid (catalog only; the rows
# are validated once, in migrate_review_state).
@@ -356,8 +350,6 @@ class PromptModel:
migrate_annotation_state(conn)
migrate_review_state(conn)
migrate_autotag_state(conn)
# Image categories (#332): independent of every table above.
migrate_image_categories(conn)
# Last: the older generated_images rebuilds run only before file_path.
migrate_workflow_documents(conn)
@@ -1163,9 +1155,8 @@ class PromptModel:
Args:
path: File to inspect
require_library_state: Also require the tables every current
database has (Library state, image favorites, image
categories, saved views, collections, image annotations and
the review queue), as a migrated
database has (Library state, image favorites, saved views,
image annotations and the review queue), as a migrated
restore stage must; old backups may lack them.
documents: Read every workflow document and refuse a damaged
one. A restore checks its upload without it and repairs
@@ -1229,21 +1220,11 @@ class PromptModel:
)
if favorites_error:
return False, favorites_error
categories_error = image_categories_schema_error(
conn, allow_missing=not require_library_state
)
if categories_error:
return False, categories_error
views_error = saved_views_schema_error(
conn, allow_missing=not require_library_state
)
if views_error:
return False, views_error
collections_error = image_collections_schema_error(
conn, allow_missing=not require_library_state
)
if collections_error:
return False, collections_error
annotation_error = annotation_schema_error(
conn, allow_missing=not require_library_state
)
+41 -188
View File
@@ -7,18 +7,12 @@ import json
import math
import datetime
import os
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple
try:
from ..utils.inspection_response import inherited_category
except ImportError:
from utils.inspection_response import inherited_category
from typing import Optional, List, Dict, Any, Set, Tuple
from .models import PromptModel, normalize_image_path, utc_now_iso
from .pairs import PairDependencyError, PairOperations # noqa: F401 (re-exported)
from .library import LibraryOperations, titled
from .image_favorites import ImageFavoriteOperations
from .image_collections import ImageCollectionOperations
from .annotations import AnnotationOperations
from .review import ReviewOperations
from .review_decisions import ReviewDecisionOperations
@@ -33,7 +27,6 @@ from .health import HealthOperations, iter_consistency_issues
from .duplicate_copies import DuplicateCopyOperations
from .workflow_storage import WorkflowStorageOperations
from .category_import import CategoryImportOperations, EmptyReplacement # noqa: F401
from .image_categories import ImageCategoryOperations
from .workflow_documents import (
DocumentReader,
documents_of_rows,
@@ -244,17 +237,12 @@ LEFT JOIN prompt_library_state pls ON pls.prompt_id = pp.positive_prompt_id
WHERE gi.image_path IN ({marks}) OR gi.file_path IN ({marks}){fallback}"""
# One image row describes a card only when no other row claims the same file
# (as in get_image_link_summaries, several rows are ambiguous). The pair scope
# and the linked positive follow _LINK_SUMMARY_SQL; rating, tags and the
# searchable text and titles come from that positive (spec D12/D13). The raw
# recipe and positive categories are read for the inherited category, which
# Python decides (``inherited_category``, #332 MC4); SQL never filters by
# category. Every
# and the linked positive follow _LINK_SUMMARY_SQL; category, rating, tags and
# the searchable text and titles come from that positive (spec D12/D13). Every
# writer since the 3.2.4 migration stores file_path, so it is the path key.
# The prompt-level conditions are uncorrelated IN sets SQLite builds once, and
# the CASE evaluates the per-row ambiguity probe only for admitted rows.
_LINK_FILTER_SQL = """SELECT gi.file_path AS file_path, p.rating AS rating,
{hit} AS matched, p.id AS positive, pp.id IS NOT NULL AS has_pair,
pp.category AS pair_category, p.category AS category
_LINK_FILTER_SQL = """SELECT gi.file_path, p.rating, {hit}, p.id
FROM generated_images gi
LEFT JOIN prompt_pairs pp ON pp.id = (
SELECT CASE WHEN COUNT(DISTINCT ip.pair_id) = 1 THEN MIN(ip.pair_id) END
@@ -263,31 +251,10 @@ JOIN prompts p ON p.id = COALESCE(pp.positive_prompt_id, gi.prompt_id)
WHERE gi.file_path IS NOT NULL AND CASE WHEN {admit} THEN NOT EXISTS (
SELECT 1 FROM generated_images other
WHERE other.file_path = gi.file_path AND other.id <> gi.id) ELSE 0 END"""
# _LINK_FILTER_SQL's rows as one JSON array, built by SQLite in one step (#354).
# Python's sqlite3 releases and retakes the GIL around every step, so a read
# that steps once per linked row waits on a busy ComfyUI thread thousands of
# times. A BLOB value (no writer stores one) makes the value NULL (or
# json_array refuses it), and the read streams instead. The raw categories are
# put in only for a category read (#332), so plain reads stay as small as #354
# measured them.
_LINK_MATCHES_JSON_SQL = """SELECT CASE
WHEN max({blob}) THEN NULL
ELSE json_group_array(json_array({columns})) END
FROM ({select})"""
_LINK_PLAIN_COLUMNS = ("file_path", "rating", "matched", "positive")
_LINK_CATEGORY_COLUMNS = _LINK_PLAIN_COLUMNS + ("has_pair", "pair_category", "category")
# The columns a BLOB can reach: matched, positive and has_pair are computed.
_LINK_STORED_COLUMNS = ("file_path", "rating", "pair_category", "category")
# Set when this SQLite has no JSON functions: every later read streams.
_one_step_unsupported = False
# One warning per process when a read falls back to streaming.
_one_step_warned = False
# The linked positive of each given path (one unambiguous link, pair scope as
# in _LINK_FILTER_SQL) with the raw categories of its inherited category; {hit}
# as in link_filter_query. Used for the few files whose current image tags meet
# a Tags filter (#249 B3).
_LINK_FACTS_SQL = """SELECT gi.file_path, p.rating, {hit}, p.id,
pp.id IS NOT NULL, pp.category, p.category
# in _LINK_FILTER_SQL) with its category; {hit} as in link_filter_query. Used
# for the few files whose current image tags meet a Tags filter (#249 B3).
_LINK_FACTS_SQL = """SELECT gi.file_path, p.rating, {hit}, p.id, p.category
FROM generated_images gi
LEFT JOIN prompt_pairs pp ON pp.id = (
SELECT CASE WHEN COUNT(DISTINCT ip.pair_id) = 1 THEN MIN(ip.pair_id) END
@@ -316,24 +283,8 @@ _LINK_FILTER_TAG = (
)
# One unambiguously linked image's facts for the Images filters, a plain tuple
# (hit, rating, positive id, inherited category): built per linked row, so no
# NamedTuple. The category is the inherited one (#332 MC4: the recipe's when
# set, else the positive's), never the image's own category.
LinkMatch = Tuple[bool, Optional[int], int, Any]
def _link_match(rating, matched, positive, has_pair, pair_category, category):
inherited, _scope = inherited_category(category, pair_category, bool(has_pair))
return (bool(matched), rating, positive, inherited)
def link_filter_query(
q: str,
tags: List[str],
*,
every_link: bool = False,
category: Optional[str] = None,
q: str, category: Optional[str], tags: List[str], *, every_link: bool = False
) -> Tuple[str, Dict[str, Any]]:
"""The workspace filter SELECT and its named parameters (all bound).
@@ -347,9 +298,7 @@ def link_filter_query(
params["pattern"] = f"%{escape_like(q)}%"
admitted = []
if category is not None:
# Only narrows: an inherited category is one of the two raw values;
# which one is decided in Python (inherited_category, #332 MC4).
admitted.append("(p.category = :category OR pp.category = :category)")
admitted.append("p.category = :category")
params["category"] = category
for index, tag in enumerate(tags):
admitted.append(_LINK_FILTER_TAG.format(index=index))
@@ -360,50 +309,6 @@ def link_filter_query(
return sql, params
def link_matches_json_query(sql: str, categories: bool = False) -> str:
"""*sql* from link_filter_query, wrapped to return its rows as one JSON value.
With *categories* each row also carries the raw values the inherited
category is decided from; otherwise only path, rating, hit and positive.
"""
columns = _LINK_CATEGORY_COLUMNS if categories else _LINK_PLAIN_COLUMNS
blob = " OR ".join(
f"typeof({name}) = 'blob'" for name in _LINK_STORED_COLUMNS if name in columns
)
return _LINK_MATCHES_JSON_SQL.format(
blob=blob, columns=", ".join(columns), select=sql
)
def _link_matches_from_rows(
rows: Iterable[Sequence[Any]], categories: bool, wanted: Optional[Set[str]]
) -> Dict[str, LinkMatch]:
"""LinkMatch per wanted path from link rows (one-step or streamed).
Without *categories* the inherited category is None, as the caller then
never compares it.
"""
matches: Dict[str, LinkMatch] = {}
for path, rating, matched, positive, *raw in rows:
if wanted is None or path in wanted:
if categories:
matches[path] = _link_match(rating, matched, positive, *raw[:3])
else:
matches[path] = (bool(matched), rating, positive, None)
return matches
def _streamed_link_matches(
conn: sqlite3.Connection,
sql: str,
params: Dict[str, Any],
wanted: Optional[Set[str]],
categories: bool = False,
) -> Dict[str, LinkMatch]:
"""The link read stepped once per row (the fallback of the one-step read)."""
return _link_matches_from_rows(conn.execute(sql, params), categories, wanted)
_ASCII_FOLD = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
@@ -454,7 +359,6 @@ class PromptDatabase(
ExecutionOutputOperations,
HistoryReadOperations,
ImageFavoriteOperations,
ImageCollectionOperations,
AnnotationOperations,
ReviewOperations,
ReviewDecisionOperations,
@@ -464,7 +368,6 @@ class PromptDatabase(
DuplicateCopyOperations,
WorkflowStorageOperations,
CategoryImportOperations,
ImageCategoryOperations,
):
"""Database operations class for managing prompts."""
@@ -2638,94 +2541,38 @@ class PromptDatabase(
def get_image_link_matches(
self,
q: str,
category: Optional[str],
tags: List[str],
*,
every_link: bool = False,
wanted: Optional[Set[str]] = None,
category: Optional[str] = None,
) -> Dict[str, LinkMatch]:
"""Map normalized paths of unambiguously linked images to a LinkMatch
(hit, rating, positive id, inherited category).
) -> Dict[str, Tuple[bool, Optional[int], int]]:
"""Map normalized paths of unambiguously linked images to
(hit, rating, positive id).
Rows are limited to images whose linked positive has every tag in
*tags* (exact names). The category filter is applied by the caller to
the effective category (#332). ``hit`` says whether *q* occurs in
Rows are limited to images whose linked positive has *category* and
every tag in *tags* (exact names). ``hit`` says whether *q* occurs in
the positive's text, Library title or a tag name; a pair's own title
names its negative variant and is not searched.
Without tags only hits are returned, unless *every_link*
Without category or tags only hits are returned, unless *every_link*
asks for every linked image (the rating sort and the positive count).
The positive id is the card's linked positive. With *category* only
links whose raw recipe or positive category is that value are read (a
superset of the links that inherit it; the caller compares the
inherited value) and the inherited category is decided per row;
otherwise it is None, which keeps the plain reads cheap.
One SELECT, returned as one JSON value (#354): SQLite steps once, so
the read takes the GIL back a constant two or three times instead of
once per linked row while ComfyUI generates. Peak memory therefore
grows with the database's matching linked rows (about 530 bytes each,
more with the raw categories of a category read, freed on return); the
result is bounded by *wanted* (the listing's normalized paths) when
given. Where the JSON step fails (no JSON functions, a BLOB value, a
JSON text over SQLite's length limit) the rows are streamed as before.
The positive id is the card's linked positive. One SELECT whose rows
are streamed; with *wanted* (the listing's normalized paths) only those
paths are kept, so memory is bounded by the listing, not the database.
"""
sql, params = link_filter_query(
q, tags, every_link=every_link, category=category
)
categories = category is not None
sql, params = link_filter_query(q, category, tags, every_link=every_link)
matches: Dict[str, Tuple[bool, Optional[int], int]] = {}
with self.model.get_connection() as conn:
rows = None
if not _one_step_unsupported:
rows = self._one_step_link_rows(conn, sql, params, categories)
if rows is None:
return _streamed_link_matches(conn, sql, params, wanted, categories)
return _link_matches_from_rows(rows, categories, wanted)
def _one_step_link_rows(
self,
conn: sqlite3.Connection,
sql: str,
params: Dict[str, Any],
categories: bool = False,
) -> Optional[List[List[Any]]]:
"""*sql*'s rows from one JSON value, or None when this read must stream.
An interrupted read (Move library retiring the connection) is raised
as before, never retried row by row.
"""
global _one_step_unsupported, _one_step_warned
error: Optional[str] = None
try:
cursor = conn.execute(link_matches_json_query(sql, categories), params)
try:
((document,),) = cursor.fetchall()
finally:
cursor.close() # resets the statement: its snapshot ends now
except (sqlite3.OperationalError, sqlite3.DataError) as failure:
# DataError: the JSON text is longer than SQLITE_LIMIT_LENGTH.
error = str(failure)
if error == "interrupted":
raise
if error.startswith("no such function"):
_one_step_unsupported = True
else:
if document is not None:
return json.loads(document)
error = "a BLOB value"
if not _one_step_warned:
_one_step_warned = True
self.logger.warning(
"Reading image links row by row (slower while ComfyUI "
"generates); the one-step read failed: %s",
error,
)
return None
for path, rating, matched, positive in conn.execute(sql, params):
if wanted is None or path in wanted:
matches[path] = (bool(matched), rating, positive)
return matches
def get_image_link_facts(
self, paths: List[str], q: str = ""
) -> Dict[str, Tuple[bool, Optional[int], int, Any, Tuple[str, ...]]]:
) -> Dict[str, Tuple[bool, Optional[int], int, Optional[str], Tuple[str, ...]]]:
"""Map each normalized path with one unambiguous link to its positive's
(hit, rating, positive id, inherited category, tag names).
(hit, rating, positive id, category, tag names).
The link is resolved as in get_image_link_matches, and ``hit`` says
whether *q* occurs in the positive's text, Library title or a tag
@@ -2736,7 +2583,7 @@ class PromptDatabase(
if q:
hit = _LINK_FILTER_HIT
params["pattern"] = f"%{escape_like(q)}%"
rows: Dict[str, LinkMatch] = {}
rows: Dict[str, Tuple[bool, Optional[int], int, Optional[str]]] = {}
names: Dict[int, List[str]] = {}
unique = list(dict.fromkeys(paths))
with self.model.get_connection() as conn:
@@ -2745,8 +2592,10 @@ class PromptDatabase(
bound = {f"path{i}": path for i, path in enumerate(chunk)}
marks = ",".join(f":{key}" for key in bound)
sql = _LINK_FACTS_SQL.format(hit=hit, marks=marks)
for path, *row in conn.execute(sql, {**params, **bound}):
rows[path] = _link_match(*row)
for path, rating, matched, positive, category in conn.execute(
sql, {**params, **bound}
):
rows[path] = (bool(matched), rating, positive, category)
positives = sorted({row[2] for row in rows.values()})
for start in range(0, len(positives), _FACTS_CHUNK):
chunk = positives[start : start + _FACTS_CHUNK]
@@ -2759,16 +2608,20 @@ class PromptDatabase(
path: (*row, tuple(names.get(row[2], ()))) for path, row in rows.items()
}
def get_image_paths_matching(self, q: str, tags: List[str]) -> Set[str]:
"""Normalized paths whose linked positive matches *q* and *tags*.
def get_image_paths_matching(
self, q: str, category: Optional[str], tags: List[str]
) -> Set[str]:
"""Normalized paths whose linked positive matches *q*, *category* and *tags*.
See get_image_link_matches; an empty *q* matches every linked image
the tags admit.
the category and tags admit.
"""
return {
path
for path, match in self.get_image_link_matches(q, tags).items()
if match[0] or not q
for path, (hit, _rating, _positive) in self.get_image_link_matches(
q, category, tags
).items()
if hit or not q
}
def get_prompt_id_for_image(self, image_path: str) -> Optional[int]:
+4 -8
View File
@@ -59,7 +59,6 @@ SUMMARY_KEYS = (
"image_links",
"runs",
"saved_views",
"collections",
)
@@ -145,8 +144,7 @@ def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
``positives`` are the Library's prompts (``library_counts``'s
``positive_prompts``); ``pairs`` the active saved pairs and
``archived_pairs`` the archived ones; ``image_links`` the indexed image
rows; ``runs`` the recorded executions; ``collections`` the image
collections (#331).
rows; ``runs`` the recorded executions.
"""
from .library import _ELIGIBLE
@@ -171,7 +169,6 @@ def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
"image_links": _count(conn, "generated_images"),
"runs": _count(conn, "execution_runs"),
"saved_views": _count(conn, "saved_views"),
"collections": _count(conn, "image_collections"),
}
@@ -183,7 +180,6 @@ _FINGERPRINT_SQL = (
("prompt_pairs", "SELECT MAX(id), MAX(updated_at) FROM prompt_pairs"),
("execution_runs", "SELECT COUNT(*), MAX(rowid) FROM execution_runs"),
("saved_views", "SELECT MAX(rowid) FROM saved_views"),
("image_collections", "SELECT MAX(rowid) FROM image_collections"),
)
@@ -194,9 +190,9 @@ def live_fingerprint(conn: sqlite3.Connection):
tables a restore replaces, or when a prompt, pair, run or saved view is
the newest edited (counts plus the newest row and edit of each table);
the apply refuses with ``plan-changed`` then. It does not see every
edit (a tag, favorite or collection membership change on an older row);
those are kept in the restore's safety backup. A read transaction the
caller already has is used and left open; one opened here is ended here.
edit (a tag or favorite change on an older row); those are kept in the
restore's safety backup. A read transaction the caller already has is
used and left open; one opened here is ended here.
"""
import json
+26 -150
View File
@@ -15,10 +15,6 @@ only when all of this holds:
Anything else returns None: the caller leaves the file unlinked. Read-only.
``recorded_output`` applies the first two rules and returns, for the
inspector (#386), the one output node (save node) that run reported the file
from and the texts PromptManager recorded at execution per sampler.
Lookups are batched (``stored_run_prompts``): one parameterized query per
chunk of at most ``LOOKUP_CHUNK`` distinct file names, matched in memory. The
history schema is left exactly as it is (no extra index), so older builds can
@@ -30,7 +26,6 @@ import os
import sqlite3
import threading
import weakref
from dataclasses import dataclass
from typing import Any, Callable, Dict, Iterable, List, Optional
try:
@@ -50,8 +45,7 @@ LOOKUP_CHUNK = 500
_RUNS_FOR_NAMES = (
"SELECT DISTINCT o.filename, o.subfolder, o.root_key, o.path_style, o.run_id, "
"r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at), "
"o.output_node_id "
"r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at) "
"FROM execution_run_outputs o JOIN execution_runs r ON r.run_id = o.run_id "
"WHERE o.folder_type = 'output' AND o.filename IN ({marks})"
)
@@ -144,10 +138,9 @@ def stored_run_prompt(
def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]:
"""``{(filename, subfolder, root_key, style): {run_id: (start, end, nodes)}}``.
"""``{(filename, subfolder, root_key, style): {run_id: (start, end)}}``.
``nodes`` is the set of output node ids that reported the location in that
run. One query per ``LOOKUP_CHUNK`` names; None without history tables.
One query per ``LOOKUP_CHUNK`` names; None without history tables.
"""
found: Dict[tuple, dict] = {}
conn = db.model.get_connection()
@@ -158,49 +151,12 @@ def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]:
rows = conn.execute(query, chunk).fetchall() # nosec B608
except sqlite3.Error:
return None # no history tables (a library without history)
for name, subfolder, root_key, style, run_id, began, ended, node in rows:
for name, subfolder, root_key, style, run_id, began, ended in rows:
runs = found.setdefault((name, subfolder, root_key, style), {})
runs.setdefault(run_id, (began, ended, set()))[2].add(node)
runs[run_id] = (began, ended)
return found
def _recorded_runs(
db: Any,
image_paths: Iterable[str],
output_root: Optional[str],
path_style: Optional[str],
) -> Dict[str, tuple]:
"""``{path: (run_id, output node ids)}`` for each file history attributes.
A file is attributed when exactly one recorded run reported its location
and it was last written while that run executed.
"""
paths = list(dict.fromkeys(path for path in image_paths if path))
if not paths or not output_root:
return {}
style = path_style or ("windows" if os.name == "nt" else "posix")
located = {}
for path in paths:
location = _location(path, output_root, style)
if location is not None:
located[path] = location
if not located:
return {}
names = sorted({filename for _key, _sub, filename in located.values()})
runs = _runs_by_location(db, names)
if not runs:
return {}
attributed = {}
for path, (root_key, subfolder, filename) in located.items():
reported = runs.get((filename, subfolder, root_key, style), {})
if len(reported) != 1:
continue
((run_id, (began, ended, nodes)),) = reported.items()
if _written_during(path, began, ended):
attributed[path] = (run_id, nodes)
return attributed
def _named_prompt(db: Any, run_id: str) -> Optional[int]:
contexts = db.get_execution_contexts(run_id).get("contexts") or []
named = {
@@ -227,115 +183,35 @@ def stored_run_prompts(
"""
paths = list(dict.fromkeys(path for path in image_paths if path))
result: Dict[str, Optional[int]] = dict.fromkeys(paths)
if not paths or not output_root:
return result
style = path_style or ("windows" if os.name == "nt" else "posix")
located = {}
for path in paths:
location = _location(path, output_root, style)
if location is not None:
located[path] = location
if not located:
return result
names = sorted({filename for _key, _sub, filename in located.values()})
runs = _runs_by_location(db, names)
if not runs:
return result
prompts: Dict[str, Optional[int]] = {}
for path, (run_id, _nodes) in _recorded_runs(
db, paths, output_root, path_style
).items():
for path, (root_key, subfolder, filename) in located.items():
reported = runs.get((filename, subfolder, root_key, style), {})
if len(reported) != 1:
continue
((run_id, (began, ended)),) = reported.items()
if not _written_during(path, began, ended):
continue
if run_id not in prompts:
prompts[run_id] = _named_prompt(db, run_id)
result[path] = prompts[run_id]
return result
@dataclass(frozen=True)
class RecordedOutput:
"""What recorded history knows about the run that wrote one file (#386).
``saver_id`` is the one output node the run reported the file from (None
when several did). ``texts`` maps each sampler anchor of the run to the
effective texts PromptManager recorded at execution for its roles
(``{"positive": text or None, "negative": text or None}``); a role whose
text was only inferred from the graph is None.
"""
saver_id: Optional[str]
texts: Dict[str, Dict[str, Optional[str]]]
def _recorded_texts(db: Any, run_id: str) -> Dict[str, Dict[str, Optional[str]]]:
texts = {}
for context in db.get_execution_contexts(run_id).get("contexts") or []:
evidence = context.get("evidence") or {}
roles = {}
for role in ("positive", "negative"):
values = context.get(role) or {}
proof = (evidence.get(role) or {}).get("effective") or {}
value = values.get("effective")
recorded = proof.get("provenance") == "recorded"
roles[role] = value if recorded and isinstance(value, str) else None
texts[context["anchor_id"]] = roles
return texts
def recorded_output(
db: Any,
image_path: str,
output_root: Optional[str],
path_style: Optional[str] = None,
) -> Optional[RecordedOutput]:
"""The recorded run facts for ``image_path``, else None (#386).
The file must be attributed to one run (``_recorded_runs``: one run
reported its location and it was written during that run). Read-only; it
never links anything.
"""
if not image_path:
return None
conn = db.model.get_connection()
own_transaction = not conn.in_transaction
if own_transaction: # outputs and texts from one state of history
conn.execute("BEGIN")
try:
found = _recorded_runs(db, [image_path], output_root, path_style)
if image_path not in found:
return None
run_id, nodes = found[image_path]
saver = next(iter(nodes)) if len(nodes) == 1 else None
return RecordedOutput(saver, _recorded_texts(db, run_id))
finally:
if own_transaction:
conn.rollback()
_LOGGER = logging.getLogger("prompt_manager.run_attribution")
def recorded_output_lookup(
db: Any, *, close: bool = False, reference: Optional[dict] = None
) -> Callable[[Any], Any]:
"""``lookup(path) -> RecordedOutput or None`` for the inspect routes (#386).
Uses ComfyUI's output folder. Any failure (no history, a locked library)
is logged and names no node, so the file is judged from its own metadata.
``close`` closes the calling thread's connection afterwards, for routes
that close theirs after reading. ``reference`` (a history occurrence's
output reference) spells the file as history recorded it: the route's
target is case-normalized on Windows, and history matches names exactly.
"""
def lookup(path: Any) -> Optional[RecordedOutput]:
root = host_output_directory()
if reference is not None:
if reference.get("folder_type") != "output" or not root:
return None
path = os.path.join(
root, reference.get("subfolder") or "", reference.get("filename") or ""
)
try:
return recorded_output(db, str(path), root)
except (sqlite3.Error, OSError, ValueError):
_LOGGER.warning("Recorded run lookup failed", exc_info=True)
return None
except Exception:
_LOGGER.exception("Recorded run lookup failed unexpectedly")
return None
finally:
if close:
db.model.close()
return lookup
_SEALED_LISTENERS: List[weakref.WeakMethod] = []
_SEALED_LOCK = threading.Lock()
+15 -55
View File
@@ -13,9 +13,7 @@ same connection before commit. Owner rows and their timestamps never change.
Category consolidation is previewed in one ``library_snapshot`` and applied in
one ``_pair_transaction`` that recomputes the owner projection and requires the
preview's ``impact_token``; only rows carrying the exact source change. Image
rows holding the source (``image_categories``, #332) are owners too: they are
counted, hashed into the token and renamed with the prompts and pairs.
preview's ``impact_token``; only rows carrying the exact source change.
"""
import sqlite3
@@ -99,30 +97,16 @@ def _validate_tag_id(tag_id: object) -> None:
raise ValueError("Invalid tag ID")
# Category values in Tags & categories (#332 §7): the prompt-backed values
# (nonblank values on eligible parents or on active pairs, grouped exactly, so
# whitespace and case variants stay distinct; /library/options lists the same
# prompt-backed set with its own query) plus the values only image rows hold.
# A prompt-backed value is ordered by its oldest owning positive; an image-only
# value sorts after all of them (first_pid is SQLite's INTEGER maximum, above
# any prompt id), then by value. ``image_count`` counts stored rows, including
# files missing on disk. No FULL OUTER JOIN, so it runs on SQLite 3.31+.
_IMAGE_ONLY_FIRST_PID = 2**63 - 1
_CATEGORY_ROWS = (
# The /library/options universe: nonblank values on eligible parents or on
# active pairs, grouped exactly (whitespace and case variants stay distinct).
_CATEGORY_GROUPS = (
"WITH owners(value,pid) AS ("
"SELECT p.category,p.id FROM prompts p WHERE p.category IS NOT NULL "
"AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
"UNION SELECT pp.category,pp.positive_prompt_id FROM prompt_pairs pp "
"WHERE pp.archived=0 AND pp.category IS NOT NULL AND TRIM(pp.category)!=''),"
"image_values(value,image_count) AS ("
"SELECT category,COUNT(*) FROM image_categories GROUP BY category),"
"grouped(value,first_pid,prompt_count) AS ("
"SELECT value,MIN(pid),COUNT(DISTINCT pid) FROM owners GROUP BY value "
"UNION ALL SELECT value," + str(_IMAGE_ONLY_FIRST_PID) + ",0 FROM image_values "
"WHERE value NOT IN (SELECT value FROM owners)),"
"counted AS (SELECT g.value,g.first_pid,g.prompt_count,"
"COALESCE(i.image_count,0) AS image_count FROM grouped g "
"LEFT JOIN image_values i ON i.value=g.value) "
"grouped AS (SELECT value,MIN(pid) AS first_pid,"
"COUNT(DISTINCT pid) AS prompt_count FROM owners GROUP BY value) "
)
@@ -133,23 +117,15 @@ _CATEGORY_COMPONENTS = (
"WHERE p.category=? ORDER BY p.id"
)
_CATEGORY_PAIRS = "SELECT id,archived FROM prompt_pairs WHERE category=? ORDER BY id"
# Image rows holding one exact value (#332 DC7), an index search.
_IMAGE_CATEGORY_ROWIDS = (
"SELECT rowid FROM image_categories WHERE category=? ORDER BY rowid"
)
_CATEGORY_EXISTS = (
"SELECT EXISTS(SELECT 1 FROM prompts WHERE category=:v)"
" OR EXISTS(SELECT 1 FROM prompt_pairs WHERE category=:v)"
" OR EXISTS(SELECT 1 FROM image_categories WHERE category=:v)"
)
# Pairs after components and image rows last, so a failing update rolls every
# earlier one back. An image row is keyed by its file, never by its value, so
# merging into a value other files already hold cannot conflict.
# Pairs after components, so a failing pair update rolls both back.
_CATEGORY_UPDATES = (
"UPDATE prompts SET category=?,updated_at=? WHERE category=?",
"UPDATE prompt_pairs SET category=?,updated_at=? WHERE category=?",
"UPDATE image_categories SET category=?,updated_at=? WHERE category=?",
)
@@ -160,15 +136,8 @@ def category_impact_token(
target_exists: bool,
components: list,
pairs: list,
*,
images: list = (),
) -> str:
"""Hash a category projection like ``library_edit_token`` (no secret).
``images`` are the rowids of the image rows holding ``source`` (#332 DC7).
With none the preimage has no ``images`` key, so a prompt-only token keeps
the value it had before image categories existed.
"""
"""Hash a category projection like ``library_edit_token`` (no secret)."""
preimage = {
"epoch": epoch,
"source": source,
@@ -177,8 +146,6 @@ def category_impact_token(
"components": sorted([int(i), bool(flag)] for i, flag in components),
"pairs": sorted([int(i), bool(flag)] for i, flag in pairs),
}
if images:
preimage["images"] = sorted(int(i) for i in images)
return canonical_token(_CATEGORY_DOMAIN, preimage)
@@ -358,8 +325,7 @@ class VocabularyOperations:
pairs = [
[row[0], bool(row[1])] for row in conn.execute(_CATEGORY_PAIRS, (source,))
]
images = [row[0] for row in conn.execute(_IMAGE_CATEGORY_ROWIDS, (source,))]
if not components and not pairs and not images:
if not components and not pairs:
raise LibraryCommandError("vocabulary-conflict", epoch)
target_exists = bool(
conn.execute(_CATEGORY_EXISTS, {"v": target}).fetchone()[0]
@@ -373,10 +339,9 @@ class VocabularyOperations:
"other_prompts": len(components) - eligible,
"negative_variants": len(pairs) - archived,
"archived_negative_variants": archived,
"images": len(images),
},
"impact_token": category_impact_token(
epoch, source, target, target_exists, components, pairs, images=images
epoch, source, target, target_exists, components, pairs
),
}
@@ -400,8 +365,7 @@ class VocabularyOperations:
def apply_vocabulary_category(
self, *, expected_epoch: str, source: str, target: str, impact_token: str
) -> dict:
"""Set the exact ``source`` to ``target`` on every component, pair
and image row.
"""Set the exact ``source`` to ``target`` on every component and pair.
The recomputed projection must still match the preview's token; a
mismatch is ``vocabulary-conflict`` with nothing written.
@@ -458,15 +422,15 @@ class VocabularyOperations:
_limit(limit)
key = _category_after(after)
total = conn.execute(
_CATEGORY_ROWS + "SELECT COUNT(*) FROM counted"
_CATEGORY_GROUPS + "SELECT COUNT(*) FROM grouped"
).fetchone()[0]
where, params = "", []
if key is not None:
where = " WHERE first_pid>? OR (first_pid=? AND value>?)"
params = [key[0], key[0], key[1]]
rows = conn.execute(
_CATEGORY_ROWS
+ "SELECT value,first_pid,prompt_count,image_count FROM counted"
_CATEGORY_GROUPS
+ "SELECT value,first_pid,prompt_count FROM grouped"
+ where
+ " ORDER BY first_pid,value LIMIT ?",
[*params, limit + 1],
@@ -476,11 +440,7 @@ class VocabularyOperations:
return {
"library_epoch": epoch,
"categories": [
{
"value": row["value"],
"prompt_count": row["prompt_count"],
"image_count": row["image_count"],
}
{"value": row["value"], "prompt_count": row["prompt_count"]}
for row in rows
],
"total": total,
+6 -14
View File
@@ -26,13 +26,13 @@ except ImportError:
InputTypeDict = dict
try:
from .prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
from .prompt_manager_base import PromptManagerBase
except ImportError:
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
from prompt_manager_base import PromptManagerBase
class PromptManager(PromptManagerBase, ComfyNodeABC):
@@ -115,7 +115,6 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
)
},
),
"save_to_library": SAVE_TO_LIBRARY_INPUT,
},
"hidden": {
# The queued graph and this node's id: used to tell the positive
@@ -153,7 +152,6 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
append_text: str = "",
prompt: Optional[Dict[str, Any]] = None,
unique_id: Optional[str] = None,
save_to_library: bool = True,
) -> Tuple[Any]:
"""
Encode the text prompt and save it to the database.
@@ -166,7 +164,6 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
search_text: Text to search for in past prompts
prepend_text: Text to prepend to the main prompt
append_text: Text to append to the main prompt
save_to_library: Off: nothing is saved, counted or tracked (#90)
Returns:
Tuple containing the conditioning for the diffusion model and the final
@@ -208,9 +205,7 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
# Save prompt to database and set execution context for gallery tracking
prompt_id = None
extended_tags = []
if not save_to_library:
self._skip_not_saved(prompt, unique_id)
elif storage_text and storage_text.strip():
if storage_text and storage_text.strip():
self.logger.debug(f"Processing prompt text: {storage_text[:100]}...")
# Add prepend/append info to tags if they exist
@@ -254,11 +249,6 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
tokens = clip.tokenize(encoding_text)
conditioning = clip.encode_from_tokens_scheduled(tokens)
self.logger.info(f"CLIP encoding completed, text: {repr(encoding_text)[:80]}")
result = (conditioning, encoding_text)
if not save_to_library:
return result # #90: no metadata registration, no history text
# Register with ComfyUI integration for standard metadata compatibility
node_id = f"promptmanager_{int(time.time() * 1000)}"
self.comfyui_integration.register_prompt(
@@ -272,10 +262,12 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
"append_text": append_text.strip() if append_text else None,
},
)
self.logger.info(f"CLIP encoding completed, text: {repr(encoding_text)[:80]}")
result = (conditioning, encoding_text)
record_runtime_context(
type(self).__name__, text, prepend_text, append_text, encoding_text
)
self._record_cached_item(unique_id, (encoding_text,), storage_text)
return result
@classmethod
+5 -96
View File
@@ -23,16 +23,9 @@ try:
from .utils.image_monitor import get_image_monitor
from .utils.prompt_tracker import get_prompt_tracker
from .utils.hashing import generate_prompt_hash
from .utils.prompt_graph import resolve_text, run_prompt_nodes, save_setting
from .utils.prompt_graph import resolve_text, run_prompt_nodes
from .utils.run_keys import executing_run_id, prompt_node_key
from .utils.usage_tracking import PENDING_FIRST_USE
from .utils.cached_runs import (
CACHED_RUNS,
Role,
Seen,
own_counts,
stored_facts,
)
except ImportError:
import sys
@@ -42,32 +35,9 @@ except ImportError:
from utils.image_monitor import get_image_monitor
from utils.prompt_tracker import get_prompt_tracker
from utils.hashing import generate_prompt_hash
from utils.prompt_graph import resolve_text, run_prompt_nodes, save_setting
from utils.prompt_graph import resolve_text, run_prompt_nodes
from utils.run_keys import executing_run_id, prompt_node_key
from utils.usage_tracking import PENDING_FIRST_USE
from utils.cached_runs import (
CACHED_RUNS,
Role,
Seen,
own_counts,
stored_facts,
)
# The optional input every PromptManager node declares last (#90): declared
# last so no widget value of a workflow saved before it shifts.
SAVE_TO_LIBRARY_INPUT = (
"BOOLEAN",
{
"default": True,
"label_on": "save",
"label_off": "skip",
"tooltip": (
"Save this node's prompt to the library, count its runs and link its "
"images. Off: PromptManager ignores the run; the output is unchanged."
),
},
)
class PromptManagerBase:
@@ -101,10 +71,8 @@ class PromptManagerBase:
"""Save the prompt to the SQLite database.
Runs are normally counted by the queue hook (utils/usage_tracking.py), so a
save only counts one when asked to (text the hook could not see, unless
this job already counted the node from ComfyUI's cache) or when the hook
queued this prompt before it existed. The two are independent: the
pending catch-up is written even when ``count_run`` is False.
save only counts one when asked to (batch items whose text the hook could not
see) or when the hook queued this prompt before it existed.
Args:
text: The prompt text
@@ -172,9 +140,7 @@ class PromptManagerBase:
resolved = resolve_text(prompt_graph, node_id)
return {
"is_run_prompt": node_id in run_prompt_nodes(prompt_graph),
# The hook skips a switch it cannot read (#90): the node counts
"seen_at_queue": resolved is not None
and save_setting(prompt_graph, node_id) is True
and generate_prompt_hash(resolved) == generate_prompt_hash(text),
"known": True,
}
@@ -201,19 +167,8 @@ class PromptManagerBase:
batch_item = (
role["known"] and role["is_run_prompt"] and not role["seen_at_queue"]
)
counts = own_counts(
type(self).__name__,
stored_facts(text),
Role(
frozenset({"text"} if role["known"] and role["is_run_prompt"] else ())
),
Seen(text=role["seen_at_queue"]),
)
prompt_id = self._save_prompt_to_database(
text=text,
category=category,
tags=tags,
count_run=bool(counts) and not self._replayed(unique_id),
text=text, category=category, tags=tags, count_run=batch_item
)
if role["is_run_prompt"] and not prompt_id:
# The save failed: an older entry of this key must not stand in (#342)
@@ -227,34 +182,6 @@ class PromptManagerBase:
)
return prompt_id
def _cached_runs(self) -> Any:
return getattr(self, "cached_runs", CACHED_RUNS)
def _replayed(self, unique_id: Any) -> bool:
"""This job already counted the node from ComfyUI's cache (#347).
A RAM-pressure cache can free the entry after ``execution_cached``, so
the node executes in a job that replayed it: its own count is skipped.
"""
try:
return unique_id is not None and self._cached_runs().replayed(unique_id)
except Exception as error:
self.logger.debug(f"Could not check the cached-run replay: {error}")
return False
def _record_cached_item(
self, unique_id: Any, outputs: tuple, text: Any, negative_text: Any = None
) -> None:
"""Remember this execution's outputs and stored text for cached jobs (#347)."""
if unique_id is None:
return
try:
self._cached_runs().record_item(
unique_id, outputs, stored_facts(text, negative_text)
)
except Exception as error:
self.logger.debug(f"Could not record the cached-run receipt: {error}")
@staticmethod
def _run_identity(prompt_graph: Any, unique_id: Any) -> Dict[str, Any]:
"""The tracker's proof that an image belongs to this execution (#342)."""
@@ -263,24 +190,6 @@ class PromptManagerBase:
"run_id": executing_run_id(),
}
def _skip_not_saved(self, prompt_graph: Any = None, unique_id: Any = None) -> None:
"""``save_to_library`` is off: nothing is saved, counted or tracked (#90).
A literal off makes the node no run prompt (``utils.prompt_graph``), so
no image looks up its run key. A switch wired from a node the graph
cannot read keeps the node a run prompt: its run key is marked like an
empty editor's, so an earlier run's entry of that key never links this
run's images.
"""
message = (
"Save to library is off for this node: its prompt is not saved, the "
"run is not counted and its images are not linked"
)
if isinstance(prompt_graph, dict) and unique_id is not None:
self._mark_untracked(prompt_graph, unique_id, message)
else:
self.logger.info(message)
def _skip_empty_prompt(self, prompt_graph: Any, unique_id: Any) -> None:
"""The ``text`` editor is empty: save and track nothing (#342).
+26 -63
View File
@@ -12,21 +12,9 @@ else:
try:
from .prompt_manager import IO, InputTypeDict, PromptManagerFocused
from .utils.cached_runs import (
PAIR,
Role,
Seen,
own_counts,
pair_components,
stored_facts,
)
from .utils.hashing import generate_prompt_hash
from .utils.pair_execution import record_pair_execution
from .utils.prompt_graph import (
intrinsic_pair_nodes,
run_prompt_components,
save_setting,
)
from .utils.prompt_graph import intrinsic_pair_nodes, run_prompt_components
from .utils.prompt_pair_resolver import resolve_text_input
from .utils.usage_tracking import (
PENDING_FIRST_USE,
@@ -35,21 +23,9 @@ try:
)
except ImportError:
from prompt_manager import IO, InputTypeDict, PromptManagerFocused
from utils.cached_runs import (
PAIR,
Role,
Seen,
own_counts,
pair_components,
stored_facts,
)
from utils.hashing import generate_prompt_hash
from utils.pair_execution import record_pair_execution
from utils.prompt_graph import (
intrinsic_pair_nodes,
run_prompt_components,
save_setting,
)
from utils.prompt_graph import intrinsic_pair_nodes, run_prompt_components
from utils.prompt_pair_resolver import resolve_text_input
from utils.usage_tracking import (
PENDING_FIRST_USE,
@@ -98,13 +74,8 @@ class PromptManagerPair(PromptManagerFocused):
prompt: Optional[Dict[str, Any]] = None,
unique_id: Optional[str] = None,
extra_pnginfo: Optional[Dict[str, Any]] = None,
save_to_library: bool = True,
) -> Tuple[Any, Any, str, str]:
"""Encode both sides once; database and tracking failures never block CLIP.
With ``save_to_library`` off nothing is saved, counted, tracked or
recorded (#90); the four outputs are the same.
"""
"""Encode both sides once; database and tracking failures never block CLIP."""
if clip is None:
raise RuntimeError("ERROR: clip input is invalid: None")
if not isinstance(text, str) or not isinstance(negative_text, str):
@@ -122,10 +93,6 @@ class PromptManagerPair(PromptManagerFocused):
negative_conditioning = clip.encode_from_tokens_scheduled(
clip.tokenize(negative_text)
)
result = positive_conditioning, negative_conditioning, positive, negative_text
if not save_to_library:
self._skip_not_saved(prompt, unique_id)
return result
record_pair_execution(
extra_pnginfo,
unique_id,
@@ -186,6 +153,7 @@ class PromptManagerPair(PromptManagerFocused):
)
except Exception as error:
self.logger.warning(f"Could not register paired metadata: {error}")
result = positive_conditioning, negative_conditioning, positive, negative_text
record_runtime_context(
type(self).__name__,
text,
@@ -197,9 +165,6 @@ class PromptManagerPair(PromptManagerFocused):
record_runtime_context(
type(self).__name__, negative_text, "", "", negative_text, role="negative"
)
self._record_cached_item(
unique_id, (positive, negative_text), text, negative_text
)
return result
def _save_and_track_pair(
@@ -235,9 +200,6 @@ class PromptManagerPair(PromptManagerFocused):
resolved_negative = (
resolve_text_input(prompt, node_id, "negative_text") if known else None
)
if known and save_setting(prompt, node_id) is not True:
# The hook skips a switch it cannot read (#90): the node counts
resolved_positive = resolved_negative = None
positive_seen = resolved_positive is not None and generate_prompt_hash(
resolved_positive
) == generate_prompt_hash(text)
@@ -247,35 +209,36 @@ class PromptManagerPair(PromptManagerFocused):
and generate_prompt_hash(resolved_negative)
== generate_prompt_hash(negative_text)
)
negative_seen = resolved_negative is not None and generate_prompt_hash(
resolved_negative
) == generate_prompt_hash(negative_text)
positive_pending = getattr(self, "pending_first_use", PENDING_FIRST_USE)
pair_pending = getattr(self, "pending_pair_first_use", PENDING_PAIR_FIRST_USE)
facts = stored_facts(text, negative_text)
role = Role(frozenset(used_fields), pair_used)
# Own counts are skipped when this job already counted the node from
# ComfyUI's cache (#347); the pending first-use catch-up never is.
counts = (
[]
if self._replayed(unique_id)
else own_counts(
PAIR, facts, role, Seen(positive_seen, negative_seen, pair_seen)
)
)
component_ids = {"text": prompt_id, "negative_text": pair["negative_prompt_id"]}
for field, component_hash in pair_components(facts, role):
component_id = component_ids[field]
if not component_id:
counted = set()
for field, component_text, resolved_text, component_id in (
("text", text, resolved_positive, prompt_id),
(
"negative_text",
negative_text,
resolved_negative,
pair["negative_prompt_id"],
),
):
component_hash = generate_prompt_hash(component_text)
if (
field not in used_fields
or not component_id
or component_hash in counted
):
continue
runs = positive_pending.consume(component_hash) + int(
("prompt", component_hash) in counts
counted.add(component_hash)
seen_at_queue = (
resolved_text is not None
and generate_prompt_hash(resolved_text) == component_hash
)
runs = positive_pending.consume(component_hash) + int(not seen_at_queue)
if runs:
self.db.record_prompt_use(component_id, times=runs)
if pair_used:
runs = pair_pending.consume(pair_run_key(text, negative_text)) + int(
("pair", facts.positive_hash, facts.negative_hash) in counts
not pair_seen
)
if runs:
self.db.record_prompt_pair_use(pair_id, times=runs)
+6 -14
View File
@@ -26,13 +26,13 @@ except ImportError:
InputTypeDict = dict
try:
from .prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
from .prompt_manager_base import PromptManagerBase
except ImportError:
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
from prompt_manager_base import PromptManagerBase
class PromptManagerText(PromptManagerBase, ComfyNodeABC):
@@ -115,7 +115,6 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
),
},
),
"save_to_library": SAVE_TO_LIBRARY_INPUT,
},
"hidden": {
# The queued graph and this node's id: used to tell the positive
@@ -150,7 +149,6 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
append_text: str = "",
prompt: Optional[Dict[str, Any]] = None,
unique_id: Optional[str] = None,
save_to_library: bool = True,
) -> Tuple[str]:
"""
Process the text prompt and save it to the database.
@@ -162,7 +160,6 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
search_text: Text to search for in past prompts
prepend_text: Text to prepend to the main prompt
append_text: Text to append to the main prompt
save_to_library: Off: nothing is saved, counted or tracked (#90)
Returns:
Tuple containing the final processed text string
@@ -187,9 +184,7 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
# Save prompt to database and set execution context for gallery tracking
prompt_id = None
extended_tags = []
if not save_to_library:
self._skip_not_saved(prompt, unique_id)
elif storage_text and storage_text.strip():
if storage_text and storage_text.strip():
self.logger.debug(f"Processing prompt text: {storage_text[:100]}...")
# Add prepend/append info to tags if they exist
@@ -225,11 +220,6 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
else:
self._skip_empty_prompt(prompt, unique_id)
self.logger.debug(f"Text processing completed: {final_text[:100]}...")
result = (final_text,)
if not save_to_library:
return result # #90: no metadata registration, no history text
# Register with ComfyUI integration for standard metadata compatibility
node_id = f"promptmanagertext_{int(time.time() * 1000)}"
self.comfyui_integration.register_prompt(
@@ -243,10 +233,12 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
"append_text": append_text.strip() if append_text else None,
},
)
self.logger.debug(f"Text processing completed: {final_text[:100]}...")
result = (final_text,)
record_runtime_context(
type(self).__name__, text, prepend_text, append_text, final_text
)
self._record_cached_item(unique_id, (final_text,), storage_text)
return result
@classmethod
-6
View File
@@ -48,7 +48,6 @@ from .review import ReviewRoutesMixin
from .review_recording import ReviewRecordingMixin
from .review_events import ReviewEventsMixin
from .saved_views import SavedViewRoutesMixin
from .collections import CollectionRoutesMixin
from .history import HistoryRoutesMixin
from .images import ImageRoutesMixin
from .gallery_listing import WALK_STOPPED, RootListing, WalkOutcome
@@ -73,7 +72,6 @@ from .diagnostics_report import DiagnosticsReportMixin
from .autotag_routes import AutotagRoutesMixin
from .lora_integration import LoraIntegrationMixin
from .lora_import_routes import LoraImportRoutesMixin
from .image_categories import ImageCategoryRoutesMixin
try:
from ...database.operations import PromptDatabase
@@ -316,7 +314,6 @@ class PromptManagerAPI(
ReviewRecordingMixin,
ReviewEventsMixin,
SavedViewRoutesMixin,
CollectionRoutesMixin,
HistoryRoutesMixin,
PromptRoutesMixin,
ImageRoutesMixin,
@@ -339,7 +336,6 @@ class PromptManagerAPI(
AutotagRoutesMixin,
LoraIntegrationMixin,
LoraImportRoutesMixin,
ImageCategoryRoutesMixin,
):
"""REST API handler for PromptManager operations and web interface.
@@ -688,7 +684,6 @@ class PromptManagerAPI(
self._register_review_routes(routes)
self._register_review_event_routes(routes)
self._register_saved_view_routes(routes)
self._register_collection_routes(routes)
self._register_history_routes(routes)
self._register_prompt_routes(routes)
self._register_image_routes(routes)
@@ -705,7 +700,6 @@ class PromptManagerAPI(
self._register_autotag_routes(routes)
self._register_lora_routes(routes)
self._register_lora_import_routes(routes)
self._register_image_category_routes(routes)
# Register gzip compression middleware (once)
global _gzip_registered
-31
View File
@@ -52,7 +52,6 @@ try:
extract_generation_params,
parse_page_json,
)
from ...utils.prompt_graph import metadata_opted_out, side_opted_out
from ...utils.prompt_pair_resolver import resolve_prompt_pair, sampler_less_prompt
from ...utils.review_capture import reviewable
except ImportError:
@@ -68,7 +67,6 @@ except ImportError:
extract_generation_params,
parse_page_json,
)
from utils.prompt_graph import metadata_opted_out, side_opted_out
from utils.prompt_pair_resolver import resolve_prompt_pair, sampler_less_prompt
from utils.review_capture import reviewable
@@ -262,11 +260,6 @@ SCAN_DETAILS = {
"failed": "Error: the prompt could not be stored",
"error": "Error: the file could not be processed (see the server log)",
"found": "Metadata found, but no prompt text to store",
"not_saved": "Skipped · its PromptManager node does not save to the library",
"not_saved_wired": (
"Skipped · its Save to library switch is wired from another node, so only "
"the run itself can link it"
),
None: "No prompt metadata in the file",
}
@@ -1727,34 +1720,10 @@ class AdminRoutesMixin:
except (ValueError, TypeError):
# Preserve malformed metadata for conservative resolver rejection.
pass
if metadata_opted_out(canonical):
# Save to library was off for the run's prompt: no prompt, link,
# recorded-run fallback or Needs review item (#90)
return "not_saved"
if metadata_opted_out(canonical, unreadable=True):
# The switch is wired from a node the file cannot show: only the
# live run (the node saw the value) can link it, never a scan
return "not_saved_wired"
if not canonical.get("prompt") and not canonical.get("parameters"):
return self._link_recorded_run(media_file, canonical, detailed, history)
_with_file_size(canonical, media_file)
resolution = resolve_prompt_pair(canonical)
if resolution["state"] == "resolved" and side_opted_out(
canonical.get("prompt"), resolution["negative"]
):
# Only the negative's node is off: store the positive alone (#90)
prompt_text = (
resolution["positive"]
.get("storage_text", resolution["positive"]["text"])
.strip()
)
status = self._store_scanned_prompt(media_file, canonical, prompt_text)
if not detailed or status not in SCAN_TITLED_DETAILS:
return status
return {
"status": status,
"title": self._scan_title(media_file, prompt_text),
}
if blank_positive(resolution):
# Only prepend/append text: no stored prompt can own it (#342)
resolution = {**resolution, "state": "unresolved"}
-17
View File
@@ -86,23 +86,6 @@ def unavailable_message(model_name: str) -> str:
return f"{model_name} is not downloaded. {UNAVAILABLE_HINT}"
def model_refusal(status: dict, model_type: str, model_name: str):
"""Why *model_type* cannot run, from ``get_models_status()``, else None.
Not downloaded first (``unavailable_message``), then a backend package
that cannot be imported (#424). Both are a 409 ``model-unavailable``.
"""
entry = status.get(model_type) or {}
if not entry.get("downloaded"):
return unavailable_message(model_name)
missing = entry.get("missing_packages") or []
if missing:
from ..autotag import missing_packages_message
return missing_packages_message(model_type, [str(name) for name in missing])
return None
def run_suggestions(service, capture_path: str, model_type: str, options: dict):
"""Blocking: load (or swap to) *model_type*, then suggest for the capture.
+41 -17
View File
@@ -21,18 +21,13 @@ from aiohttp import web
from .annotation_suggest import (
SUGGEST_FIELDS,
OptionError,
model_refusal,
run_suggestions,
suggestion_rows,
unavailable_message,
validate_options,
)
from .autotag_routes import model_busy_response
# The target check is shared with the collection routes (#331); both names
# stay importable from here.
from .media_target import MAX_TARGET_PATH # noqa: F401
from .media_target import media_target as _target
from .images import _unavailable_link, _verified_title
from .images import _forbidden, _has_traversal, _unavailable_link, _verified_title
from .prompts import bad_request
from .. import model_gate
@@ -45,7 +40,7 @@ try:
)
from ...database.execution_history import subscribe_history_restore
from ...database.library_commands import LibraryCommandError
from ...utils.media_capture import CaptureError, CaptureStore
from ...utils.media_capture import CAPTURE_EXTENSIONS, CaptureError, CaptureStore
except ImportError:
from database.annotations import (
MAX_IMAGE_TAGS,
@@ -55,13 +50,15 @@ except ImportError:
)
from database.execution_history import subscribe_history_restore
from database.library_commands import LibraryCommandError
from utils.media_capture import CaptureError, CaptureStore
from utils.media_capture import CAPTURE_EXTENSIONS, CaptureError, CaptureStore
# Request bodies of the annotation routes are small (a target plus a tag set).
MAX_BODY_BYTES = 64 * 1024
MAX_TARGET_PATH = 1024
CAPTURE_URL = "/prompt_manager/image-annotations/capture/"
_PREVIEW_CHUNK = 64 * 1024
_TARGET_FIELDS = frozenset({"root_index", "root_scope", "path"})
_OBSERVE_FIELDS = frozenset({"target"})
_SAVE_FIELDS = frozenset(
{
@@ -191,6 +188,37 @@ def _capture_error(exc: CaptureError) -> web.Response:
return _coded(exc.code, _MESSAGES.get(exc.code) or str(exc))
def _target(data: object):
"""``((index, scope, relative), None)`` for a target, else ``(None, error)``.
Only the listing identity is accepted: an int root index, the root's
64-hex scope and a bounded root-relative POSIX media path, never an
absolute, drive-qualified, backslash or ``..`` path. Nothing here touches
the filesystem or the database.
"""
if not isinstance(data, dict) or set(data) != _TARGET_FIELDS:
return None, bad_request("target must name root_index, root_scope and path")
index, scope, relative = data["root_index"], data["root_scope"], data["path"]
if type(index) is not int or index < 0:
return None, bad_request("Invalid gallery root")
if not isinstance(scope, str) or _HEX64.fullmatch(scope) is None:
return None, bad_request("Invalid gallery root scope")
if not isinstance(relative, str) or not relative or len(relative) > MAX_TARGET_PATH:
return None, bad_request("path must be a relative media path")
parts = relative.split("/")
if (
"\0" in relative
or "\\" in relative
or relative.startswith("/")
or _has_traversal(relative)
or any(part in ("", ".", "..") or ":" in part for part in parts)
):
return None, _forbidden()
if os.path.splitext(parts[-1])[1].lower() not in CAPTURE_EXTENSIONS:
return None, _forbidden("Only media files can be tagged")
return (index, scope, relative), None
def _expires_at(seconds: float | None) -> str:
moment = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
seconds=max(0.0, seconds or 0.0)
@@ -595,7 +623,7 @@ class AnnotationRoutesMixin:
it, including a legacy batch), never downloads (``model-unavailable``)
and never writes.
"""
from ..autotag import MODELS, MissingBackendPackages, get_autotag_service
from ..autotag import MODELS, get_autotag_service
from ..config import MAX_SUGGESTION_LIMIT, AutotagConfig
data, error = await self._annotation_body(request, SUGGEST_FIELDS)
@@ -633,10 +661,9 @@ class AnnotationRoutesMixin:
return error
service = get_autotag_service()
status = await self._run_in_executor(service.get_models_status)
name = MODELS.get(model_type, {}).get("name", model_type)
refusal = model_refusal(status, model_type, name)
if refusal is not None:
return _coded("model-unavailable", refusal)
if not status.get(model_type, {}).get("downloaded"):
name = MODELS.get(model_type, {}).get("name", model_type)
return _coded("model-unavailable", unavailable_message(name))
owner = model_gate.new_owner("image-annotations")
if not model_gate.try_acquire(owner):
return model_busy_response()
@@ -655,9 +682,6 @@ class AnnotationRoutesMixin:
options,
on_skipped=functools.partial(store.release_use, token),
)
except MissingBackendPackages as exc:
# A package gone since the status read (load_model logged it).
return _coded("model-unavailable", str(exc))
except Exception:
self.logger.exception("Tag suggestion failed")
return _error(
+4 -20
View File
@@ -102,17 +102,6 @@ def size_label(size: str) -> str:
return f"{match.group(1)} {match.group(2)}" if match else str(size or "")
def package_entry(flags: dict) -> dict:
"""The public package fields of one status entry (#424): the pip names
that cannot be imported, the install command and note (strings or null)."""
missing = flags.get("missing_packages") or []
text = {
key: flags[key] if isinstance(flags.get(key), str) else None
for key in ("install_command", "install_note")
}
return {"missing_packages": [str(name) for name in missing], **text}
def public_models(status: dict, loaded_type: Optional[str]) -> dict:
"""Each engine model's public entry: names and sizes from ``MODELS``, the
flags from ``status``; never a path (``model_path`` is dropped)."""
@@ -133,7 +122,6 @@ def public_models(status: dict, loaded_type: Optional[str]) -> dict:
for key in ("model_exists", "mmproj_exists"):
if key in flags:
entry[key] = bool(flags[key])
entry.update(package_entry(flags))
models[model_type] = entry
return models
@@ -762,7 +750,6 @@ class AutotagRoutesMixin:
"downloaded": bool(
(status.get(model_type) or {}).get("downloaded")
),
**package_entry(status.get(model_type) or {}),
},
"existing": existing,
"review_required": existing == "replace_after_review",
@@ -1086,8 +1073,8 @@ class AutotagRoutesMixin:
return self.db.current_image_tags({locator: fingerprint}).get(locator, [])
async def _suggest_single(self, target, model_type, options, base):
from ..autotag import MODELS, MissingBackendPackages, get_autotag_service
from .annotation_suggest import model_refusal
from ..autotag import MODELS, get_autotag_service
from .annotation_suggest import unavailable_message
from .annotations import _pinned_suggestions, _settled, _submit
try:
@@ -1097,8 +1084,8 @@ class AutotagRoutesMixin:
service = get_autotag_service()
status = await self._run_in_executor(service.get_models_status)
message = model_refusal(status, model_type, MODELS[model_type]["name"])
if message is not None:
if not (status.get(model_type) or {}).get("downloaded"):
message = unavailable_message(MODELS[model_type]["name"])
return self._coded_error(409, "model-unavailable", message)
owner = model_gate.new_owner("legacy-single")
if not model_gate.try_acquire(owner):
@@ -1144,9 +1131,6 @@ class AutotagRoutesMixin:
options,
on_skipped=functools.partial(store.release_use, capture.token),
)
except MissingBackendPackages as exc:
# A package gone since the status read (load_model logged it).
return self._coded_error(409, "model-unavailable", str(exc))
finally:
store.release(capture.token)
finally:
-350
View File
@@ -1,350 +0,0 @@
"""Collections (#331): named, hand-picked groups of images.
Routes under ``/prompt_manager/collections``. A file is named by its listing
identity ``{root_index, root_scope, path}`` (``media_target.py``) and stored
under the favorites' keys (``_favorite_root_key``, ``_favorite_relative_key``),
which never leave the server. A stale scope (the sources were reordered) is a
409 ``source-changed`` before anything is written. Responses are
``{"success": bool, ...}``; an unexpected failure is a logged 500 with a
generic message, never the exception text.
"""
import hmac
import re
from functools import wraps
from typing import Callable
from aiohttp import web
from .images import (
_favorite_relative_key,
_favorite_root_key,
_forbidden,
_json_body_or_error,
collection_not_found,
)
from .media_target import media_target
from .prompts import bad_request
try:
from ...database.image_collections import (
CollectionFilesMissing,
CollectionInvalid,
CollectionMissing,
CollectionNameTaken,
normalize_collection_name,
)
from ...database.image_favorites import MAX_KEY_LENGTH
except ImportError:
from database.image_collections import (
CollectionFilesMissing,
CollectionInvalid,
CollectionMissing,
CollectionNameTaken,
normalize_collection_name,
)
from database.image_favorites import MAX_KEY_LENGTH
BASE = "/prompt_manager/collections"
# The most files one add or remove names (the bulk tagging cap, MC12).
MAX_ITEMS = 500
MAX_ID = 999_999_999
_PATH_ID = re.compile(r"[1-9][0-9]{0,8}")
_ROOT_INDEX = re.compile(r"[0-9]{1,9}")
_ADD_TARGETS = (frozenset({"collection_id", "items"}), frozenset({"name", "items"}))
_REMOVE_FIELDS = frozenset({"collection_id", "items"})
_RENAME_FIELDS = frozenset({"name"})
_MEDIA_MESSAGE = "Only media files can be added to a collection"
READ_FAILED = "Collections could not be read."
CHANGE_FAILED = "The collection could not be changed."
SOURCES_CHANGED = "The sources changed. Refresh the images and try again."
FILES_NOT_FOUND = (
"None of the selected images could be found. Refresh the images and try again."
)
NAME_TAKEN = "Another collection already has this name."
class _Refused(Exception):
"""A request refused before any write; carries its response."""
def __init__(self, response: web.Response):
super().__init__(response.status)
self.response = response
def _coded(status: int, code: str, message: str, **extra) -> web.Response:
return web.json_response(
{"success": False, "code": code, "error": message, **extra},
status=status,
headers={"Cache-Control": "no-store"},
)
def _collection_response(failure: str) -> Callable:
"""Map the database's refusals to their status; anything else to a 500."""
def decorate(handler: Callable) -> Callable:
@wraps(handler)
async def wrapped(self, request: web.Request) -> web.Response:
try:
return await handler(self, request)
except _Refused as refused:
return refused.response
except CollectionInvalid as exc:
return bad_request(str(exc))
except CollectionMissing:
return collection_not_found()
except CollectionFilesMissing:
return _coded(404, "files-not-found", FILES_NOT_FOUND)
except CollectionNameTaken as exc:
return _coded(
409, "name-taken", NAME_TAKEN, collection_id=exc.collection_id
)
except Exception:
self.logger.exception("Collection request failed")
return web.json_response(
{"success": False, "error": failure}, status=500
)
return wrapped
return decorate
def _path_id(request: web.Request) -> int:
raw = request.match_info["collection_id"]
if not raw.isascii() or _PATH_ID.fullmatch(raw) is None:
raise _Refused(bad_request("Invalid collection id"))
return int(raw)
def _body_id(value: object) -> int:
if type(value) is not int or not 1 <= value <= MAX_ID:
raise _Refused(bad_request("Invalid collection id"))
return value
async def _body(request: web.Request, allowed: tuple) -> dict:
data, error = await _json_body_or_error(request)
if error is not None:
raise _Refused(error)
if frozenset(data) not in allowed:
names = " or ".join(", ".join(sorted(fields)) for fields in allowed)
raise _Refused(bad_request(f"Request must contain exactly {names}"))
return data
class CollectionRoutesMixin:
"""Mixin providing the collection routes (#331)."""
def _register_collection_routes(self, routes: web.RouteTableDef) -> None:
# The fixed paths go first: "/items" must never match "/{collection_id}".
routes.get(BASE)(self.list_collections)
routes.post(BASE + "/items")(self.add_collection_items)
routes.delete(BASE + "/items")(self.remove_collection_items)
routes.get(BASE + "/membership")(self.collection_membership)
routes.patch(BASE + "/{collection_id}")(self.rename_collection)
routes.delete(BASE + "/{collection_id}")(self.delete_collection)
# -- reads -------------------------------------------------------------
@_collection_response(READ_FAILED)
async def list_collections(self, request: web.Request) -> web.Response:
"""Every collection with its listed ``count`` and its ``missing`` rows.
``count`` is the members listed under every configured source (one
primary-key range read per source, matched against its cached
listing); ``missing`` the stored rows beyond it. When a scan or a read
fails both are null for every collection and the names still return.
"""
collections = await self._run_in_executor(self.db.list_collections)
counts = None
if collections:
try:
counts = await self._collection_counts()
except Exception:
self.logger.warning(
"Collection counts could not be read", exc_info=True
)
rows = []
for collection in collections:
listed, stored = (None, None) if counts is None else counts
count = None if listed is None else listed.get(collection["id"], 0)
stored_rows = None if stored is None else stored.get(collection["id"], 0)
rows.append(
{
**collection,
"count": count,
# Clamped: never below zero, whatever the snapshot saw.
"missing": None if count is None else max(0, stored_rows - count),
}
)
return web.json_response({"success": True, "collections": rows})
async def _collection_counts(self) -> tuple[dict, dict]:
"""``({id: listed members}, {id: stored rows})`` over every source.
The scans come from the gallery cache; the root keys (realpath), the
listings and one read snapshot of every member and stored count run
in the executor, so a write between two reads cannot skew them.
"""
roots = list(self._get_all_output_dirs())
root_keys = await self._run_in_executor(
lambda: [_favorite_root_key(root) for root in roots]
)
distinct = {} # one folder configured twice counts once
for root, root_key in zip(roots, root_keys):
distinct.setdefault(root_key, root)
scans = {
root_key: await self._get_gallery_files(root)
for root_key, root in distinct.items()
}
return await self._run_in_executor(self._count_members, distinct, scans)
def _count_members(self, roots: dict, scans: dict) -> tuple[dict, dict]:
"""Blocking: match one snapshot's members against each listing."""
members, stored = self.db.collection_count_snapshot(list(roots))
listed: dict[int, int] = {}
for root_key, root in roots.items():
listing = self._favorite_listing(root, scans[root_key])
for collection_id, keys in members.get(root_key, {}).items():
found = sum(1 for key in keys if key in listing)
listed[collection_id] = listed.get(collection_id, 0) + found
return listed, stored
@_collection_response(READ_FAILED)
async def collection_membership(self, request: web.Request) -> web.Response:
"""GET ``?root_index&root_scope&path``: the ids holding that file."""
query = request.query
if len(query) != 3 or set(query) != {"root_index", "root_scope", "path"}:
# len() counts a repeated key (?path=a&path=b) twice.
return bad_request("Query must name root_index, root_scope and path once")
raw_index = query["root_index"]
if not raw_index.isascii() or _ROOT_INDEX.fullmatch(raw_index) is None:
return bad_request("Invalid gallery root")
roots = list(self._get_all_output_dirs())
target = {
"root_index": int(raw_index),
"root_scope": query["root_scope"],
"path": query["path"],
}
root_key, key = await self._run_in_executor(
self._collection_key, target, roots, {}
)
ids = await self._run_in_executor(self.db.file_collections, root_key, key)
return web.json_response({"success": True, "collection_ids": ids})
# -- writes ------------------------------------------------------------
@_collection_response(CHANGE_FAILED)
async def add_collection_items(self, request: web.Request) -> web.Response:
"""POST ``{collection_id | name, items}``: add the listed files.
Only files the source's listing holds are added; the rest count in
``not_found``. A new name with nothing to add creates nothing (404
``files-not-found``).
"""
data = await _body(request, _ADD_TARGETS)
target = {}
if "name" in data:
target["name"] = normalize_collection_name(data["name"])
else:
target["collection_id"] = _body_id(data["collection_id"])
roots = list(self._get_all_output_dirs())
keys = await self._run_in_executor(self._collection_items, data["items"], roots)
found, not_found = [], 0
listings = {}
for index, root_key, key in keys:
if index not in listings:
root = roots[index]
files = await self._get_gallery_files(root)
listings[index] = await self._run_in_executor(
self._favorite_listing, root, files
)
if key in listings[index]:
found.append((root_key, key))
else:
not_found += 1
result = await self._run_in_executor(
lambda: self.db.add_to_collection(items=found, **target)
)
return web.json_response({"success": True, **result, "not_found": not_found})
@_collection_response(CHANGE_FAILED)
async def remove_collection_items(self, request: web.Request) -> web.Response:
"""DELETE ``{collection_id, items}``: listed or not, every named file."""
data = await _body(request, (_REMOVE_FIELDS,))
collection_id = _body_id(data["collection_id"])
roots = list(self._get_all_output_dirs())
keys = await self._run_in_executor(self._collection_items, data["items"], roots)
result = await self._run_in_executor(
self.db.remove_from_collection,
collection_id,
[(root_key, key) for _index, root_key, key in keys],
)
return web.json_response({"success": True, **result})
@_collection_response(CHANGE_FAILED)
async def rename_collection(self, request: web.Request) -> web.Response:
"""PATCH ``{name}``: a case variant of its own name is allowed."""
collection_id = _path_id(request)
data = await _body(request, (_RENAME_FIELDS,))
name = normalize_collection_name(data["name"])
collection = await self._run_in_executor(
self.db.rename_collection, collection_id, name
)
return web.json_response({"success": True, "collection": collection})
@_collection_response(CHANGE_FAILED)
async def delete_collection(self, request: web.Request) -> web.Response:
"""DELETE: the collection and its members; files are never touched."""
collection_id = _path_id(request)
await self._run_in_executor(self.db.delete_collection, collection_id)
return web.json_response({"success": True, "deleted": True})
# -- request steps -----------------------------------------------------
def _collection_items(self, items: object, roots: list) -> list[tuple]:
"""``(index, root key, relative key)`` per distinct file, in order.
Blocking (realpath per root, once per request): run in the executor.
Every item is checked before anything is written: 400 for a malformed
shape, 403 for an unsafe path, 409 for a stale scope. Two spellings
that fold to one key (Windows) count once.
"""
if not isinstance(items, list) or not 1 <= len(items) <= MAX_ITEMS:
raise _Refused(bad_request(f"items must list 1 to {MAX_ITEMS} images"))
keys, sources = {}, {}
for item in items:
index = item.get("root_index") if isinstance(item, dict) else None
root_key, key = self._collection_key(item, roots, sources)
keys.setdefault((root_key, key), index)
return [(index, root_key, key) for (root_key, key), index in keys.items()]
def _collection_key(
self, item: object, roots: list, sources: dict
) -> tuple[str, str]:
"""The stored keys of one target, or raise its refusal (blocking).
*sources* caches ``{index: (scope, root key)}`` for one request, so
each root's realpath runs once however many items name it.
"""
target, error = media_target(item, media_message=_MEDIA_MESSAGE)
if error is not None:
raise _Refused(error)
index, scope, relative = target
if index >= len(roots):
raise _Refused(_coded(409, "source-changed", SOURCES_CHANGED))
if index not in sources:
sources[index] = (
self._gallery_root_scope(roots[index], index),
_favorite_root_key(roots[index]),
)
current, root_key = sources[index]
if not hmac.compare_digest(scope, current):
raise _Refused(_coded(409, "source-changed", SOURCES_CHANGED))
key = _favorite_relative_key(relative)
if max(len(root_key), len(key)) > MAX_KEY_LENGTH:
# A stored key past the schema bound would fail the next restore.
raise _Refused(_forbidden("Path is too long to add to a collection"))
return root_key, key
+1 -12
View File
@@ -588,10 +588,7 @@ class DuplicateFilesMixin:
job.update(total=len(targets), status_line="Removing duplicate files")
spellings = await self._run_in_executor(self._source_spellings)
removed, failure, keepers = 0, None, {}
totals = dict.fromkeys(
("moved", "merged", "favorites", "tags", "review", "collections"), 0
)
totals["categories"] = 0 # image categories kept on the kept copies (#332)
totals = dict.fromkeys(("moved", "merged", "favorites", "tags", "review"), 0)
for done, target in enumerate(targets, start=1):
if failure is not None:
self._record_target(job, "kept", target, NOT_ATTEMPTED)
@@ -634,18 +631,10 @@ class DuplicateFilesMixin:
job.log_line(
f"Moved {_plural(totals['favorites'], 'favorite')} to the kept copies"
)
if totals["collections"]:
job.log_line(
f"Moved {_plural(totals['collections'], 'collection membership')} "
"to the kept copies"
)
if totals["tags"]:
job.log_line(
f"Added {_plural(totals['tags'], 'image tag')} to the kept copies"
)
if totals.get("categories"):
kept = _plural(totals["categories"], "image category", "image categories")
job.log_line(f"Kept {kept} on the kept copies")
if totals["review"]:
job.log_line(
f"Settled {_plural(totals['review'], 'Needs review item')} "
+2 -26
View File
@@ -18,7 +18,7 @@ Blocking: each lookup stats the file's folders once (cached).
import os
from pathlib import Path
from .images import _favorite_relative_key, _favorite_root_key, image_category_key
from .images import _favorite_relative_key, _favorite_root_key
def _identity(path):
@@ -78,15 +78,6 @@ class Spellings:
)
return spelled
@property
def listed_roots(self):
"""The listed roots (the output sources), in order."""
return [root for root, _identity in self._listed]
def category_key(self, path):
"""*path*'s image category file key (#332), or None outside every source."""
return image_category_key(self.listed_roots, path)
def favorites(self, path):
"""``{listed root: path below it (POSIX)}`` for each listed root holding it."""
path = os.path.abspath(str(path))
@@ -104,8 +95,7 @@ def copy_move(target, kept_stat, spellings):
A record found under a folder spelling moves to the kept copy under
the same spelling when that folder holds it too, else to its real
path. Its favorite and its collection memberships move under every
listed root holding the file.
path. Its favorite moves under every listed root holding the file.
"""
removed = spellings.of(target["abs"])
kept = spellings.of(target["keeper_abs"])
@@ -135,23 +125,9 @@ def copy_move(target, kept_stat, spellings):
"targets": targets,
"kept_path": target["keeper_abs"],
"favorites": favorites,
"categories": _category_pairs(target, spellings),
"removed_locator": target["locator"],
"kept_locator": target["keeper_locator"],
"sha256": target["hash"],
"kept_size": kept_stat.st_size,
"kept_mtime_ns": kept_stat.st_mtime_ns,
}
def _category_pairs(target, spellings):
"""``[(removed file key, kept file key)]`` for the image category (#332).
From the two absolute paths, not the favorites tuples: those follow the
listed root, which differs from the file key when sources nest.
"""
removed = spellings.category_key(target["abs"])
kept = spellings.category_key(target["keeper_abs"])
if removed is None or kept is None:
return []
return [(removed, kept)]
+5 -58
View File
@@ -5,7 +5,6 @@ import base64
import binascii
from functools import wraps
import json
import os
import re
import sqlite3
from typing import Callable
@@ -28,12 +27,10 @@ from .history_current import (
snapshot_current_output_scope,
)
from .inspection_service import CurrentFileChanged, inspect_current_file
from .images import image_category_key
try:
from ...database.execution_history import HistoryEpochChanged
from ...database.history_gallery import read_history_gallery
from ...database.run_attribution import recorded_output_lookup
from ...database.history_reads import (
HistoryCaptureChanged,
OutputPosition,
@@ -41,15 +38,10 @@ try:
WindowPosition,
)
from ...utils.history_read_fence import HistoryReadFence
from ...utils.inspection_response import (
OWN_CATEGORY_UNREAD,
apply_own_category,
encode_history_inspection_response,
)
from ...utils.inspection_response import encode_history_inspection_response
except ImportError:
from database.execution_history import HistoryEpochChanged
from database.history_gallery import read_history_gallery
from database.run_attribution import recorded_output_lookup
from database.history_reads import (
HistoryCaptureChanged,
OutputPosition,
@@ -57,11 +49,7 @@ except ImportError:
WindowPosition,
)
from utils.history_read_fence import HistoryReadFence
from utils.inspection_response import (
OWN_CATEGORY_UNREAD,
apply_own_category,
encode_history_inspection_response,
)
from utils.inspection_response import encode_history_inspection_response
_RUN_MAX_BYTES = 8 * 1024 * 1024
@@ -687,32 +675,6 @@ class HistoryRoutesMixin:
) -> web.Response:
return await self._history_current(request, "inspection")
def _history_own_category(self, reference: dict, scope):
"""The recorded output's own category (#332), inside the snapshot.
The path is the symlink-resolved output folder plus the receipt's
subfolder and filename, keyed by ``image_category_key`` over the
configured output sources as Images keys it. A temp output or a path
no source holds has none. A failed read is OWN_CATEGORY_UNREAD.
"""
if (
reference.get("folder_type") != "output"
or scope.canonical_output is None
or candidate_locator(reference, scope) is None
):
return None
path = os.path.join(
str(scope.canonical_output), reference["subfolder"], reference["filename"]
)
try:
key = image_category_key(self._get_all_output_dirs(), path)
return None if key is None else self.db.image_category(*key)
except Exception:
self.logger.warning(
"History image category could not be read", exc_info=True
)
return OWN_CATEGORY_UNREAD
async def _history_current(
self, request: web.Request, kind: str
) -> web.StreamResponse:
@@ -737,7 +699,6 @@ class HistoryRoutesMixin:
if scope.current_scope != current_scope:
raise _current_stale()
result = db.get_history_occurrence(run_id, node, ordinal)
own = None
if result is not None:
receipt = result["receipt"]
path = candidate_locator(receipt["reference"], scope)
@@ -745,10 +706,6 @@ class HistoryRoutesMixin:
db.model.get_connection(),
(path,) if path is not None else (),
)
if kind == "inspection":
own = self._history_own_category(
receipt["reference"], scope
)
finally:
db.model.close()
if result is None:
@@ -767,19 +724,9 @@ class HistoryRoutesMixin:
resolved.canonical_target,
source_label=resolved.public["source_label"],
saved=saved.get("saved"),
library=apply_own_category(
saved.get(
"library",
{
"status": (
"unavailable" if saved_unavailable else "none"
)
},
),
own,
),
recorded=recorded_output_lookup(
db, close=True, reference=receipt["reference"]
library=saved.get(
"library",
{"status": "unavailable" if saved_unavailable else "none"},
),
)
_reauthorize_current_target(resolved, scope)
-374
View File
@@ -1,374 +0,0 @@
"""Image category routes (#332): an image file's own category.
``PUT /prompt_manager/images/category`` sets the own category of up to 500
listed files; ``DELETE`` removes it (listed or not, so a missing file's
category can still be removed); ``GET /prompt_manager/images/categories``
lists the own-category values of files the sources list. Targets are the
Images listing identity (``{root_index, root_scope, path}``, checked by
``media_target``) of an output source; LoRA folders are not sources here.
Instead of ``items`` a request may name ``filter``, the Images listing's own
query ("Select all N matching", #392), resolved now up to 500 images.
Each file is stored under its file key (``images.image_category_key``).
These routes never read or write prompts, recipes or image links.
"""
import os
from pathlib import Path
from aiohttp import web
from .image_tag_jobs import MAX_JOB_TARGETS, _parse_targets
from .images import (
_favorite_relative_key,
_favorite_root_key,
_forbidden,
_json_body_or_error,
image_category_key,
)
from .prompts import bad_request
try:
from ...database.image_categories import MAX_CATEGORY_LENGTH, MAX_KEY_LENGTH
from ...database.models import normalize_image_path
from ...utils.validators import validate_category
except ImportError:
from database.image_categories import MAX_CATEGORY_LENGTH, MAX_KEY_LENGTH
from database.models import normalize_image_path
from utils.validators import validate_category
CATEGORY_MEDIA_MESSAGE = "Only media files can have a category"
_PUT_FIELDS = frozenset({"category"})
_DELETE_FIELDS = frozenset()
TOO_MANY = f"Select at most {MAX_JOB_TARGETS} images."
SOURCE_CHANGED_MESSAGE = "The sources changed. Refresh the images and try again."
WRITE_FAILED = "The category could not be changed."
READ_FAILED = "Image categories could not be read."
def _coded(status: int, code: str, message: str) -> web.Response:
return web.json_response(
{"success": False, "code": code, "error": message},
status=status,
headers={"Cache-Control": "no-store"},
)
def _category_value(raw) -> tuple[str | None, web.Response | None]:
"""The stripped category, or a 400 (an empty value is a 400: use DELETE)."""
if not isinstance(raw, str):
return None, bad_request("category must be a string")
value = raw.strip()
if not value:
return None, bad_request("category must not be empty")
if len(value) > MAX_CATEGORY_LENGTH:
return None, bad_request(
f"Category cannot exceed {MAX_CATEGORY_LENGTH} characters"
)
try:
value.encode("utf-8")
validate_category(value)
except UnicodeEncodeError:
return None, bad_request("category must be valid text")
except ValueError as exc:
return None, bad_request(str(exc))
return value, None
class _KeyRefused(Exception):
"""A target has no storable file key (outside every source, or too long)."""
def __init__(self, message: str = "Access denied"):
super().__init__(message)
self.message = message
def _file_key(roots, root, relative) -> tuple[str, str]:
"""The target's file key (blocking: resolves the sources)."""
key = image_category_key(roots, Path(root) / relative)
if key is None:
raise _KeyRefused()
if max(len(key[0]), len(key[1])) > MAX_KEY_LENGTH:
raise _KeyRefused("Path is too long")
return key
def _nested_sources(root: Path, sources) -> list[tuple[str, str]]:
"""``[(relative key prefix, source root key)]`` of sources inside *root*.
Deepest first, so the first prefix that matches is the file's source.
"""
base = os.path.join(os.path.normcase(os.path.realpath(str(root))), "")
nested = []
for source in sources:
real = os.path.realpath(str(source))
if not os.path.normcase(real).startswith(base):
continue
relative = os.path.relpath(real, os.path.realpath(str(root)))
nested.append(
(
_favorite_relative_key(relative.replace(os.sep, "/")),
_favorite_root_key(source),
)
)
nested.sort(key=lambda item: item[0].count("/"), reverse=True)
return nested
def _source_prefix(relative_key: str, nested) -> str | None:
"""The prefix of the deepest nested source holding *relative_key*."""
for prefix, _source_key in nested:
if relative_key.startswith(prefix + "/"):
return prefix
return None
class ImageCategoryRoutesMixin:
"""Routes for images' own categories (#332)."""
def _register_image_category_routes(self, routes):
# Fixed paths beside DELETE /prompt_manager/images/{image_id}, as the
# favorite routes are. Registered after it, they still win: aiohttp
# (3.10+) looks fixed paths up before dynamic resources
# (test_delete_category_never_reaches_delete_image).
@routes.put("/prompt_manager/images/category")
async def set_image_category_route(request):
return await self.set_image_category(request)
@routes.delete("/prompt_manager/images/category")
async def clear_image_category_route(request):
return await self.clear_image_category(request)
@routes.get("/prompt_manager/images/categories")
async def list_image_categories_route(request):
return await self.list_image_categories(request)
# -- reading ---------------------------------------------------------------
def _own_category_map(self, sources, listings) -> dict:
"""``{normalized listed path: own category}`` for the listed files (blocking).
One primary-key range read per source; each stored row is matched to
the listing of every selected root that holds its source, so the work
follows the stored rows, not the files. A file under a nested source
is keyed by the deepest one (``image_category_key``). A row whose file
is gone matches nothing. A failed read raises.
"""
sources = list(sources)
rows = self.db.own_categories_of(_favorite_root_key(s) for s in sources)
if not rows:
return {}
by_source: dict[str, dict[str, str]] = {}
for (root_key, relative), category in rows.items():
by_source.setdefault(root_key, {})[relative] = category
found = {}
for _index, root, files in listings:
listing = self._favorite_listing(root, files)
root_key = _favorite_root_key(root)
# normalize_image_path(root / listed) without a call per file: the
# listed spellings are clean relative POSIX paths. Windows abspath
# also drops a component's trailing dots and spaces, so it keeps
# the call there.
base = os.path.join(normalize_image_path(str(root)), "")
def normalized(listed, base=base, root=root):
if os.name == "nt":
return normalize_image_path(str(root / listed))
return base + os.path.normcase(listed.replace("/", os.sep))
nested = [
item for item in _nested_sources(root, sources) if item[1] != root_key
]
for relative, category in by_source.get(root_key, {}).items():
listed = listing.get(relative)
if listed is not None and _source_prefix(relative, nested) is None:
found[normalized(listed)] = category
for prefix, source_key in nested:
for relative, category in by_source.get(source_key, {}).items():
full = f"{prefix}/{relative}"
listed = listing.get(full)
if listed is not None and _source_prefix(full, nested) == prefix:
found[normalized(listed)] = category
return found
async def _source_listings(self, roots) -> list[tuple]:
return [
(index, root, await self._get_gallery_files(root))
for index, root in enumerate(roots)
]
async def list_image_categories(self, request):
"""GET: the own-category values of listed files, sorted."""
try:
roots = list(self._get_all_output_dirs())
listings = await self._source_listings(roots)
found = await self._run_in_executor(self._own_category_map, roots, listings)
except Exception:
self.logger.exception("Image categories read failed")
return _coded(500, "categories-failed", READ_FAILED)
return web.json_response(
{"success": True, "categories": sorted(set(found.values()))},
headers={"Cache-Control": "no-store"},
)
# -- writing ---------------------------------------------------------------
async def _filter_targets(self, raw):
"""``(targets, None)`` for an Images listing filter, else ``(None, error)``.
"Select all N matching" (#392) sends the listing's own query; it is
resolved now, up to 500 images (a 400 ``too-many-images`` above).
"""
from .images import ListingSourceChanged, parse_listing_filter
try:
selection = await self.resolve_listing_filter(
parse_listing_filter(raw), MAX_JOB_TARGETS
)
except ListingSourceChanged:
return None, _coded(409, "source-changed", SOURCE_CHANGED_MESSAGE)
except ValueError as exc:
return None, bad_request(str(exc))
if selection.truncated:
return None, _coded(400, "too-many-images", TOO_MANY)
return tuple(selection.targets), None
async def _category_request(self, request, fields: frozenset):
"""``(data, targets, roots, None)`` or ``(None, None, None, error)``.
The images are ``items`` (1 to 500 targets) or ``filter`` (all images
an Images listing filter admits), besides *fields*.
"""
data, error = await _json_body_or_error(request)
if error is not None:
return None, None, None, error
names = set(data) - fields
if set(data) & fields != fields or names not in ({"items"}, {"filter"}):
return (
None,
None,
None,
bad_request(
"Request must contain "
+ ", ".join(sorted(fields))
+ (" and " if fields else "")
+ "either items or filter"
),
)
if "filter" in data:
targets, error = await self._filter_targets(data["filter"])
if error is not None:
return None, None, None, error
return data, targets, list(self._get_all_output_dirs()), None
items = data["items"]
if not isinstance(items, list) or not 1 <= len(items) <= MAX_JOB_TARGETS:
return (
None,
None,
None,
bad_request(f"items must list 1 to {MAX_JOB_TARGETS} images"),
)
targets, error = _parse_targets(items, media_message=CATEGORY_MEDIA_MESSAGE)
if error is not None:
return None, None, None, error
try:
for _index, _scope, relative in targets:
relative.encode("utf-8")
except UnicodeEncodeError:
return None, None, None, bad_request("path must be valid text")
roots = list(self._get_all_output_dirs())
for index, scope in dict.fromkeys((t[0], t[1]) for t in targets):
if index >= len(roots):
return None, None, None, bad_request("Invalid gallery root")
error = self._gallery_scope_error(
{"root_scope": scope, "root": str(index)}, roots
)
if error is not None:
if error.status == 409:
error = _coded(409, "source-changed", SOURCE_CHANGED_MESSAGE)
return None, None, None, error
return data, targets, roots, None
def _listed_file_keys(self, roots, targets, files) -> tuple[list, list]:
"""``(listed keys, unlisted keys)`` of the targets (blocking).
A target counts as listed when its source's listing holds it; its key
then follows the listed spelling. Targets folding to one key count
once, and a key that is listed through any target is listed.
"""
listings, listed, missing = {}, {}, {}
for index, _scope, relative in targets:
root = roots[index]
if index not in listings:
listings[index] = self._favorite_listing(root, files[index])
spelled = listings[index].get(_favorite_relative_key(relative))
key = _file_key(roots, root, spelled or relative)
(missing if spelled is None else listed)[key] = True
return list(listed), [key for key in missing if key not in listed]
async def set_image_category(self, request):
"""PUT ``{items, category}``: set the own category of listed files."""
data, targets, roots, error = await self._category_request(request, _PUT_FIELDS)
if error is not None:
return error
category, error = _category_value(data["category"])
if error is not None:
return error
try:
files = {}
for index in dict.fromkeys(t[0] for t in targets):
files[index] = await self._get_gallery_files(roots[index])
listed_keys, missing_keys = await self._run_in_executor(
self._listed_file_keys, roots, targets, files
)
except _KeyRefused as refused:
return _forbidden(refused.message)
except Exception:
self.logger.exception("Image category update failed")
return _coded(500, "category-failed", WRITE_FAILED)
try:
if not listed_keys:
return _coded(
404,
"files-not-found",
"None of the selected images could be found.",
)
counts = await self._run_in_executor(
self.db.set_image_categories, listed_keys, category
)
except Exception:
self.logger.exception("Image category update failed")
return _coded(500, "category-failed", WRITE_FAILED)
return web.json_response(
{
"success": True,
"category": category,
"updated": counts["updated"],
"unchanged": counts["unchanged"],
"not_found": len(missing_keys),
},
headers={"Cache-Control": "no-store"},
)
async def clear_image_category(self, request):
"""DELETE ``{items}``: remove the own category, listed or not."""
_data, targets, roots, error = await self._category_request(
request, _DELETE_FIELDS
)
if error is not None:
return error
try:
keys = await self._run_in_executor(
lambda: list(
dict.fromkeys(_file_key(roots, roots[i], r) for i, _s, r in targets)
)
)
cleared = await self._run_in_executor(self.db.clear_image_categories, keys)
except _KeyRefused as refused:
return _forbidden(refused.message)
except Exception:
self.logger.exception("Image category removal failed")
return _coded(500, "category-failed", WRITE_FAILED)
return web.json_response(
{"success": True, "cleared": cleared},
headers={"Cache-Control": "no-store"},
)
+12 -23
View File
@@ -42,7 +42,7 @@ from .. import model_gate
from .admin import _plural, _time_left
from .annotation_suggest import (
OptionError,
model_refusal,
unavailable_message,
validate_options,
)
from .annotations import (
@@ -58,9 +58,7 @@ from .annotations import (
from .autotag_routes import BATCH_CANCELLED, MODEL_BUSY_ERROR, model_busy_response
from .images import (
MAX_LISTING_SELECTION,
CollectionMissing,
ListingSourceChanged,
collection_not_found,
_current_fingerprint,
_listing_posix,
_public_path,
@@ -195,17 +193,13 @@ class _Spec:
return scope
def _parse_targets(raw, **target_options):
"""Unique validated targets in request order, or ``(None, error)``.
``target_options`` reach the target check (the image category routes
pass their own ``media_message``, #332).
"""
def _parse_targets(raw):
"""Unique validated targets in request order, or ``(None, error)``."""
if not isinstance(raw, list) or not 1 <= len(raw) <= MAX_JOB_TARGETS:
return None, bad_request(f"targets must list 1 to {MAX_JOB_TARGETS} images")
seen, targets = set(), []
for item in raw:
target, error = _target(item, **target_options)
target, error = _target(item)
if error is not None:
return None, error
if target not in seen:
@@ -512,8 +506,10 @@ class ImageTagJobsMixin:
)
)
label = MODELS[model_type]["label"]
message = model_refusal(status, model_type, label)
if message is not None:
if not (status.get(model_type) or {}).get("downloaded"):
message = (
f"{label} is not downloaded. Download it in System › Auto-tagging."
)
return None, _refusal(409, "model-unavailable", message)
if backup_on and short is not None:
return None, _refusal(507, "insufficient-space", short_space_message(short))
@@ -689,8 +685,6 @@ class ImageTagJobsMixin:
selection = await self.resolve_listing_filter(spec.filter, cap)
except ListingSourceChanged:
return None, _coded("source-changed")
except CollectionMissing:
return None, collection_not_found()
except ValueError as exc:
return None, bad_request(str(exc))
except Exception:
@@ -790,13 +784,13 @@ class ImageTagJobsMixin:
async def _prepare_tagging_model(self, job, spec: _Spec, owner):
"""The loaded service, or None (logged) when the job cannot run."""
from ..autotag import MODELS, MissingBackendPackages, get_autotag_service
from ..autotag import MODELS, get_autotag_service
service = get_autotag_service()
status = await self._run_in_executor(service.get_models_status)
name = MODELS.get(spec.model_type, {}).get("name", spec.model_type)
message = model_refusal(status, spec.model_type, name)
if message is not None:
if not status.get(spec.model_type, {}).get("downloaded"):
name = MODELS.get(spec.model_type, {}).get("name", spec.model_type)
message = unavailable_message(name)
job.update(status_line=message)
job.log_line(f"! {message}")
return None
@@ -808,11 +802,6 @@ class ImageTagJobsMixin:
spec.model_type,
spec.options["use_gpu"],
)
except MissingBackendPackages as exc:
# A package gone since the status read (load_model logged it).
job.update(status_line=str(exc))
job.log_line(f"! {exc}")
return None
except Exception:
self.logger.exception("The auto-tag job could not load its model")
job.update(status_line=INFERENCE_FAILED)
+58 -247
View File
@@ -18,7 +18,7 @@ import urllib.parse
from bisect import bisect_left, bisect_right, insort
from collections.abc import Mapping
from pathlib import Path
from typing import Callable, Iterable, NamedTuple
from typing import Callable, NamedTuple
from aiohttp import web
from PIL import Image, UnidentifiedImageError
@@ -58,17 +58,13 @@ from .inspection_service import (
)
try:
from ...database.image_collections import CollectionMissing
from ...database.image_favorites import MAX_KEY_LENGTH
from ...database.library import display_title
from ...database.models import normalize_image_path
from ...database.run_attribution import recorded_output_lookup
from ...utils.card_seed import card_seed, header_params, parsed_positive
from ...utils.parallel import map_parallel
from ...utils.json_strict import dumps_strict
from ...utils.inspection_response import (
OWN_CATEGORY_UNREAD,
apply_own_category,
encode_inspection_response,
library_metadata,
)
@@ -82,17 +78,13 @@ try:
reveal_in_file_manager,
)
except ImportError:
from database.image_collections import CollectionMissing
from database.image_favorites import MAX_KEY_LENGTH
from database.library import display_title
from database.models import normalize_image_path
from database.run_attribution import recorded_output_lookup
from utils.card_seed import card_seed, header_params, parsed_positive
from utils.parallel import map_parallel
from utils.json_strict import dumps_strict
from utils.inspection_response import (
OWN_CATEGORY_UNREAD,
apply_own_category,
encode_inspection_response,
library_metadata,
)
@@ -112,21 +104,6 @@ IMAGE_EXTENSIONS = _IMAGE_EXTENSIONS
_CARD_RECORDS = ("prompt", "workflow")
_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
_LISTING_FAILED = "The image list could not be filtered. Check server logs for details."
_COLLECTION_ID = re.compile(r"[1-9][0-9]{0,8}")
COLLECTION_NOT_FOUND = "That collection no longer exists."
def collection_not_found() -> web.Response:
"""The 404 a write or a listing answers for an unknown collection (#331)."""
return web.json_response(
{
"success": False,
"code": "collection-not-found",
"error": COLLECTION_NOT_FOUND,
},
status=404,
headers={"Cache-Control": "no-store"},
)
def _card_header(path: Path) -> tuple:
@@ -586,8 +563,7 @@ class _ImageTagMatch(NamedTuple):
tags: bool # every filter tag is in its image tags or linked positive tags
query: bool # ``q`` is a case-folded substring of one image tag name
# (hit, rating, positive id, inherited category) of its unambiguous link
link: tuple | None
link: tuple | None # (hit, rating, positive id) of its unambiguous link
def _current_fingerprint(path: Path) -> tuple[int, int] | None:
@@ -824,53 +800,6 @@ def _favorite_root_key(root) -> str:
return _favorite_relative_key(os.path.realpath(str(root)))
def _source_depth(source: str) -> int:
return len([part for part in re.split(r"[\\/]+", source) if part])
def deepest_source(sources, absolute_path):
"""``(source, POSIX path below it)`` of the deepest source holding the path.
A source matches by its real path or by its spelling as configured (a
detected output folder can be a symlink), and the deepest real path wins.
The path is compared lexically after ``normpath`` and never resolved, so a
symlinked subfolder keeps its listed spelling. Containment folds case
where the OS does. *source* is the real path. None when no source
contains the path.
"""
path = os.path.normpath(str(absolute_path))
folded = os.path.normcase(path)
best = None
for source in sources:
real = os.path.normpath(os.path.realpath(str(source)))
for spelling in dict.fromkeys((real, os.path.normpath(str(source)))):
prefix = os.path.join(os.path.normcase(spelling), "")
if not folded.startswith(prefix):
continue
if best is None or _source_depth(real) > _source_depth(best[0]):
best = (real, spelling)
break
if best is None:
return None
relative = os.path.relpath(path, best[1])
return best[0], relative.replace(os.sep, "/")
def image_category_key(sources, absolute_path):
"""The file key of an image's own category (#332), or None.
``(root key, relative key)``: the deepest output source that contains the
file and the path below it, keyed with the favorites helpers. Without
nested sources it equals the file's favorite key. A path no source
contains has no key (and so no own category).
"""
found = deepest_source(sources, absolute_path)
if found is None:
return None
source, relative = found
return _favorite_root_key(source), _favorite_relative_key(relative)
def _name_suffix(name):
"""``Path(name).suffix`` as Python 3.10-3.13 compute it.
@@ -1127,9 +1056,8 @@ def _workspace_filters(query: Mapping) -> dict:
the linked positive's text, title and tag names. ``category`` is exact,
``tags`` is a comma list that must all be present, ``sort=rating`` ranks by
the linked positive, and ``favorites=1`` keeps only favorites (the
Favorites page, #249 D51). ``collection=<id>`` (1 to 9 digits) keeps only
that collection's members (#331); it wins over ``favorites`` (a page never
sets both). Raises ValueError for any unknown or oversized value.
Favorites page, #249 D51). Raises ValueError for any unknown or oversized
value.
"""
text = query.get("q", "").strip()
folded = text.casefold()
@@ -1143,10 +1071,8 @@ def _workspace_filters(query: Mapping) -> dict:
{tag.strip() for tag in query.get("tags", "").split(",") if tag.strip()}
)
favorites = query.get("favorites", "0")
collection = query.get("collection", "")
if (
favorites not in ("0", "1")
or (collection != "" and _COLLECTION_ID.fullmatch(collection) is None)
or len(text) > 500
or len(folded) > 500
or media not in ("all", "image", "video")
@@ -1169,8 +1095,7 @@ def _workspace_filters(query: Mapping) -> dict:
"path": exact.casefold(),
"category": category,
"tags": tags,
"favorites": favorites == "1" and not collection,
"collection": int(collection) if collection else None,
"favorites": favorites == "1",
}
@@ -1188,8 +1113,6 @@ LISTING_FILTER_KEYS = frozenset(
"media",
"sort",
"favorites",
# A collection page's filter (#331): bulk actions act on its members.
"collection",
}
)
MAX_LISTING_FILTER_VALUE = 2048
@@ -1246,29 +1169,22 @@ def _admitted_entries(
entries: list,
filters: dict,
links: dict | None,
members: frozenset | None = None,
favorites: frozenset | None = None,
image_tags: dict | None = None,
own_categories: dict | None = None,
):
"""Yield ``(position, entry, link match)`` for each entry the filters admit.
*links* maps normalized paths to (hit, rating, positive id, inherited
category) for every unambiguously linked image (see
PromptDatabase.get_image_link_matches); a tag filter admits only those,
and ``q`` admits a filename match or a linked-positive hit. A category
admits a file whose effective category equals it (#332 MC5): its own
category (*own_categories*, ``{normalized path: category}``) when it has one,
else the inherited one, so an unlinked file with its own category is
admitted. *members*, when given, admits only those listed paths (the
favorites of ``favorites=1`` or a ``collection``'s members). *image_tags*
maps normalized paths to their _ImageTagMatch: a tag filter also admits a
file whose current image tags (with its linked positive's tags) carry every
tag, and ``q`` also admits an image tag name hit.
*links* maps normalized paths to (hit, rating, positive id) for every
unambiguously linked image (see PromptDatabase.get_image_link_matches); a
category or tag filter admits only those, and ``q`` admits a filename match
or a linked-positive hit. *favorites*, when given, admits only those listed
paths (``favorites=1``). *image_tags* maps normalized paths to their
_ImageTagMatch: a tag filter also admits a file whose current image tags
(with its linked positive's tags) carry every tag, and ``q`` also admits an
image tag name hit.
"""
query, media, folder = filters["query"], filters["media"], filters["folder"]
category = filters["category"]
members_only = bool(filters["tags"])
own = own_categories or {}
members_only = filters["category"] is not None or bool(filters["tags"])
native_folder = folder.replace("/", os.sep)
nested = native_folder + os.sep
exact = filters.get("path", "")
@@ -1281,18 +1197,12 @@ def _admitted_entries(
continue
if media != "all" and (media == "video") != entry.video:
continue
if members is not None and entry.path not in members:
if favorites is not None and entry.path not in favorites:
continue
match = None if links is None else links.get(entry.normalized)
image = None if image_tags is None else image_tags.get(entry.normalized)
if members_only and match is None and not (image and image.tags):
continue
if category is not None:
effective = own.get(entry.normalized) if own else None
if effective is None and match is not None:
effective = match[3]
if effective != category:
continue
if (
query
and query not in entry.folded
@@ -1309,9 +1219,8 @@ def _filtered_page(
size: int,
filters: dict,
links: dict | None,
members: frozenset | None = None,
favorites: frozenset | None = None,
image_tags: dict | None = None,
own_categories: dict | None = None,
) -> tuple[int, list, int | None]:
"""Count the *entries* the filters admit and take *size* from *start* on.
@@ -1321,7 +1230,7 @@ def _filtered_page(
"""
total, page, positives = 0, [], set()
for position, entry, match in _admitted_entries(
entries, filters, links, members, image_tags, own_categories
entries, filters, links, favorites, image_tags
):
total += 1
if match is not None:
@@ -2025,7 +1934,6 @@ class ImageRoutesMixin:
filters["category"],
filters["tags"],
filters["favorites"],
filters["collection"],
]
).encode()
).hexdigest()
@@ -2036,19 +1944,17 @@ class ImageRoutesMixin:
except ValueError as exc:
return bad_request(str(exc))
listings = [
(index, root, await self._get_gallery_files(root))
for index, root in enumerate(output_dirs)
if root_index is None or index == root_index
]
try:
if filters["collection"] is not None and not await self._run_in_executor(
self.db.collection_exists, filters["collection"]
):
raise CollectionMissing("No such collection")
listings = await self._workspace_listings(output_dirs, root_index)
page, total, has_more, next_cursor, positive_count = (
await self._run_in_executor(
self._select_workspace_page, listings, filters, after, limit, scope
)
)
except CollectionMissing:
return collection_not_found()
except Exception:
# A filter that needs the prompt links or the image tags cannot be
# served without them. Never echo the exception: it can carry SQL
@@ -2264,10 +2170,9 @@ class ImageRoutesMixin:
stat'ed (never every listed file), and a candidate's tags count only
while a byte record is verified at its present ``(size, mtime_ns)``
(spec amendment S1). Under a Tags filter the candidates' linked
positives are read too, so their tags combine; the category is checked
once, on the effective category, by _admitted_entries (#332).
positives are read too, so their tags and category combine.
"""
tags, query = filters["tags"], filters["query"]
tags, query, category = filters["tags"], filters["query"], filters["category"]
wanted = set(tags)
located = self._workspace_locators(listings)
# The tag index answers which files carry a relevant name (#379); no
@@ -2297,9 +2202,11 @@ class ImageRoutesMixin:
link = facts.get(entry.normalized)
carried = set(names).union(link[4] if link else ())
matches[entry.normalized] = _ImageTagMatch(
bool(wanted) and wanted <= carried,
bool(wanted)
and wanted <= carried
and (category is None or (link is not None and link[3] == category)),
bool(query) and any(query in name.casefold() for name in names),
tuple(link[:4]) if link else None,
link[:3] if link else None,
)
return matches
@@ -2308,16 +2215,10 @@ class ImageRoutesMixin:
) -> tuple:
"""The ordered entries and what admits them (blocking; run in executor).
Returns ``(entries, key, links, image_tags, members, own)``: the
listing in the filters' order, its keyset key, and the link, image-tag,
member and own-category maps _admitted_entries reads (members: a
collection's, else the favorites, else None). A collection deleted since
the route checked it raises CollectionMissing. One SELECT
(get_image_link_matches) reads every linked positive when *read_links*;
under a category it keeps only links whose raw recipe or positive
category is that value (the inherited one is decided in Python), and
the links of files whose own category is that value are read by path,
so their rating and positive still count (#332). A failed read raises when the
Returns ``(entries, key, links, image_tags, favorites)``: the listing in
the filters' order, its keyset key, and the link, image-tag and favorite
maps _admitted_entries reads. One SELECT (get_image_link_matches) reads
every linked positive when *read_links*. A failed read raises when the
filters need it (category, tags, the rating sort) or *strict*; else the
listing stays usable without links (a search then matches filenames
only). A failed image-tag read raises under a Tags filter; a search then
@@ -2329,10 +2230,10 @@ class ImageRoutesMixin:
try:
links = self.db.get_image_link_matches(
filters["text"],
category,
tags,
every_link=True,
wanted=self._workspace_wanted(listings),
category=category,
)
except Exception:
if strict or category is not None or tags or order == "rating":
@@ -2346,13 +2247,6 @@ class ImageRoutesMixin:
),
exc_info=True,
)
own = None
if category is not None:
# A failed read raises: an unfiltered page must never pass for the
# category (DC6).
own = self._own_category_map(self._get_all_output_dirs(), listings)
if links is not None:
links = self._own_category_links(links, own, filters)
image_tags = None
if filters["query"] or tags:
try:
@@ -2396,37 +2290,10 @@ class ImageRoutesMixin:
def key(entry):
return _workspace_key(entry, order)
return (
entries,
key,
links,
image_tags,
self._workspace_members(listings, filters),
own,
favorites = (
self._workspace_favorite_paths(listings) if filters["favorites"] else None
)
def _own_category_links(self, links: dict, own: dict, filters: dict) -> dict:
"""*links* plus the links of files whose own category is the filter's.
Their inherited category differs, so the category's link read left
them out; read by path (get_image_link_facts), kept only when their
positive has every filter tag, as the link read would.
"""
category, wanted = filters["category"], set(filters["tags"])
missing = [
path
for path, value in own.items()
if value == category and path not in links
]
if not missing:
return links
facts = self.db.get_image_link_facts(missing, filters["text"])
extra = {
path: tuple(fact[:4])
for path, fact in facts.items()
if wanted <= set(fact[4])
}
return {**links, **extra} if extra else links
return entries, key, links, image_tags, favorites
def _select_workspace_page(
self,
@@ -2455,7 +2322,7 @@ class ImageRoutesMixin:
or bool(filters["tags"])
or order == "rating"
)
entries, key, links, image_tags, members, own = self._workspace_admission(
entries, key, links, image_tags, favorites = self._workspace_admission(
listings, filters, read_links=needed or after is None
)
start = 0
@@ -2466,7 +2333,7 @@ class ImageRoutesMixin:
else:
start = bisect_right(entries, after, key=key)
total, page, positive_count = _filtered_page(
entries, start, limit + 1, filters, links, members, image_tags, own
entries, start, limit + 1, filters, links, favorites, image_tags
)
if not needed and after is not None:
positive_count = _COUNT_OMITTED
@@ -2488,12 +2355,12 @@ class ImageRoutesMixin:
The links are read strictly: the positives must be the listing's own.
"""
entries, _key, links, image_tags, members, own = self._workspace_admission(
entries, _key, links, image_tags, favorites = self._workspace_admission(
listings, filters, read_links=True, strict=True
)
targets, positives, unlinked, scopes = [], {}, 0, {}
for _position, entry, match in _admitted_entries(
entries, filters, links, members, image_tags, own
entries, filters, links, favorites, image_tags
):
if len(targets) == cap:
return ListingSelection(targets, list(positives), unlinked, True)
@@ -2515,11 +2382,9 @@ class ImageRoutesMixin:
) -> ListingSelection:
"""The images a parse_listing_filter result admits right now (#392).
Raises ValueError for a root outside the current roots,
Raises ValueError for a root outside the current roots, and
ListingSourceChanged when the filter's ``root_scope`` no longer names
its root (the sources were renumbered since the listing), and
CollectionMissing when its ``collection`` no longer exists (checked
before any scan and again after the members are read).
its root (the sources were renumbered since the listing).
"""
output_dirs = list(self._get_all_output_dirs())
root_index = _workspace_root(params, output_dirs)
@@ -2529,11 +2394,11 @@ class ImageRoutesMixin:
raise ListingSourceChanged()
raise ValueError("Invalid gallery root scope")
filters = _workspace_filters(params)
if filters["collection"] is not None and not await self._run_in_executor(
self.db.collection_exists, filters["collection"]
):
raise CollectionMissing("No such collection")
listings = await self._workspace_listings(output_dirs, root_index)
listings = [
(index, root, await self._get_gallery_files(root))
for index, root in enumerate(output_dirs)
if root_index is None or index == root_index
]
return await self._run_in_executor(
self._select_workspace_matches, listings, filters, cap
)
@@ -3718,58 +3583,25 @@ class ImageRoutesMixin:
self._favorite_listing_cache[cache_key] = (files, listing)
return listing
def _workspace_members(self, listings: list[tuple], filters: dict):
"""The member set the filters ask for, or None (blocking)."""
collection = filters.get("collection")
if collection is not None:
members = self._workspace_member_paths(
listings,
lambda root_key: self.db.collection_members(collection, root_key),
)
# Checked after the reads: a collection deleted before or while
# they ran is a 404, never an empty page passing for it.
if not self.db.collection_exists(collection):
raise CollectionMissing("No such collection")
return members
if filters["favorites"]:
return self._workspace_favorite_paths(listings)
return None
def _workspace_favorite_paths(self, listings: list[tuple]) -> frozenset:
"""The listed paths of every favorite in the selected roots (blocking)."""
return self._workspace_member_paths(listings, self.db.get_root_favorites)
"""The listed paths of every favorite in the selected roots (blocking).
def _workspace_member_paths(
self, listings: list[tuple], keys_for_source: Callable[[str], Iterable[str]]
) -> frozenset:
"""The listed paths of the stored keys of each selected root (blocking).
One read per root (``keys_for_source(root key)``: the favorites' or a
collection's primary-key range), matched against the root's cached
``{relative key: listing spelling}``, so the work follows the members
rather than the files, and a key folds case exactly as it was stored
(Windows). A member whose file is gone matches nothing. A failed read
raises: an unfiltered page must never pass for the members.
One primary-key read per root (get_root_favorites), matched against the
root's cached ``{relative key: listing spelling}``, so the work follows
the favorites rather than the files, and a key folds case exactly as
the toggle stored it (Windows). A favorite whose file is gone matches
nothing. A failed read raises: an unfiltered page must never pass for
the favorites.
"""
paths = set()
for _index, root, files in listings:
listing = self._favorite_listing(root, files)
for key in keys_for_source(_favorite_root_key(root)):
for key in self.db.get_root_favorites(_favorite_root_key(root)):
listed = listing.get(key)
if listed is not None:
paths.add(root / listed)
return frozenset(paths)
async def _workspace_listings(
self, output_dirs: list[Path], root_index: int | None
) -> list[tuple]:
"""``(index, root, scan)`` of every selected root (cached scans)."""
return [
(index, root, await self._get_gallery_files(root))
for index, root in enumerate(output_dirs)
if root_index is None or index == root_index
]
def _workspace_favorites(self, page: list[tuple]) -> set[tuple[int, str]]:
"""``(root index, listing path)`` of the favorites among *page* entries."""
by_root: dict[tuple[int, Path], dict[str, str]] = {}
@@ -3847,25 +3679,17 @@ class ImageRoutesMixin:
except Exception:
# The link stays known; only its title is withheld.
self.logger.warning("Inspection link title could not be read")
# An output file's own category (#332); LoRA roots have none.
own = (
self._read_own_category(outputs, path)
if index < len(outputs)
else None
)
try:
observed = inspect_current_file(
path,
source_label=_public_path(roots[index]),
saved=saved,
library=apply_own_category(
self._inspection_library(saved, unavailable=unavailable),
own,
library=self._inspection_library(
saved, unavailable=unavailable
),
image_tags=self._inspection_image_tags(
roots[index], Path(relative).parts
),
recorded=recorded_output_lookup(self.db),
)
except CurrentFileChanged:
return None
@@ -3928,19 +3752,6 @@ class ImageRoutesMixin:
return read
def _read_own_category(self, sources, path):
"""One file's own category, None, or OWN_CATEGORY_UNREAD (blocking).
Keyed by ``image_category_key`` over the output sources, as Images
writes it. A failed read is logged and hides the category (DC5).
"""
try:
key = image_category_key(sources, path)
return None if key is None else self.db.image_category(*key)
except Exception:
self.logger.warning("Image category could not be read", exc_info=True)
return OWN_CATEGORY_UNREAD
def _inspection_library(self, saved: dict | None, *, unavailable: bool) -> dict:
"""The inspector's Library facts; the route sets the title afterwards."""
result = {
+3 -14
View File
@@ -3,7 +3,7 @@
import os
import stat
from pathlib import Path
from typing import Any, Callable
from typing import Callable
try:
from ...utils.prompt_inspector import (
@@ -100,23 +100,18 @@ def inspect_current_file(
saved: dict | None,
library: dict,
image_tags: Callable[[int, int], dict] | None = None,
recorded: Callable[[Path], Any] | None = None,
) -> dict:
"""Current-file evidence read between two equal stats of *path*.
*image_tags*, when given, is called once with that stable
``(st_size, st_mtime_ns)`` and its facet is returned as ``image_tags``
(spec amendment S1: tags of a byte record verified at exactly that stat).
*recorded*, when given, is asked within the same stat window for what
recorded run history knows about *path* (``RecordedOutput`` or None: the
save node that wrote it and the texts PromptManager recorded, #386).
"""
before = _current_file_facts(path)
try:
inspection = read_inspection_file(path)
except MetadataReadError:
inspection = None
run = recorded(path) if recorded is not None and inspection is not None else None
after = _current_file_facts(path)
fields = ("st_dev", "st_ino", "st_mode", "st_size", "st_mtime_ns", "st_ctime_ns")
if any(getattr(before, key) != getattr(after, key) for key in fields):
@@ -134,15 +129,9 @@ def inspect_current_file(
profile = (
PREVIEW_INSPECTION_PROFILE if _in_host_temp(path) else INSPECTION_PROFILE
)
# The file name anchors on the saver that wrote it (#381); recorded run
# history names that saver when the file alone cannot (#386).
# The file name anchors on the saver that wrote it (#381).
evidence = inspect_prompt_evidence(
inspection.metadata,
role_saved,
profile=profile,
file_name=path.name,
recorded_saver_id=run.saver_id if run is not None else None,
recorded_texts=run.texts if run is not None else None,
inspection.metadata, role_saved, profile=profile, file_name=path.name
)
return {
"file": {
-54
View File
@@ -1,54 +0,0 @@
"""The listing identity of one media file, as a request names it.
Shared by the image-annotation routes and the collection routes (#331): a
target is ``{root_index, root_scope, path}``, checked here without touching
the filesystem or the database. The scope is compared with the current roots
by the caller.
"""
import os
import re
from .images import _forbidden, _has_traversal
from .prompts import bad_request
try:
from ...utils.media_capture import CAPTURE_EXTENSIONS
except ImportError:
from utils.media_capture import CAPTURE_EXTENSIONS
MAX_TARGET_PATH = 1024
TARGET_FIELDS = frozenset({"root_index", "root_scope", "path"})
_HEX64 = re.compile(r"[0-9a-f]{64}")
TAGGING_MEDIA_MESSAGE = "Only media files can be tagged"
def media_target(data: object, *, media_message: str = TAGGING_MEDIA_MESSAGE):
"""``((index, scope, relative), None)`` for a target, else ``(None, error)``.
Only the listing identity is accepted: an int root index, the root's
64-hex scope and a bounded root-relative POSIX media path, never an
absolute, drive-qualified, backslash or ``..`` path (a 400 for a malformed
shape, a 403 for an unsafe or non-media path, carrying *media_message*).
"""
if not isinstance(data, dict) or set(data) != TARGET_FIELDS:
return None, bad_request("target must name root_index, root_scope and path")
index, scope, relative = data["root_index"], data["root_scope"], data["path"]
if type(index) is not int or index < 0:
return None, bad_request("Invalid gallery root")
if not isinstance(scope, str) or _HEX64.fullmatch(scope) is None:
return None, bad_request("Invalid gallery root scope")
if not isinstance(relative, str) or not relative or len(relative) > MAX_TARGET_PATH:
return None, bad_request("path must be a relative media path")
parts = relative.split("/")
if (
"\0" in relative
or "\\" in relative
or relative.startswith("/")
or _has_traversal(relative)
or any(part in ("", ".", "..") or ":" in part for part in parts)
):
return None, _forbidden()
if os.path.splitext(parts[-1])[1].lower() not in CAPTURE_EXTENSIONS:
return None, _forbidden(media_message)
return (index, scope, relative), None
-4
View File
@@ -1227,9 +1227,7 @@ class PromptRoutesMixin:
"""
from .images import (
MAX_LISTING_SELECTION,
CollectionMissing,
ListingSourceChanged,
collection_not_found,
parse_listing_filter,
)
@@ -1254,8 +1252,6 @@ class PromptRoutesMixin:
},
status=409,
)
except CollectionMissing:
return collection_not_found()
except ValueError as exc:
return refused(str(exc))
if selection.truncated:
+7 -121
View File
@@ -7,7 +7,6 @@ Refactored from standalone_tagger.py for integration with PromptManager API.
import errno
import gc
import importlib.util
import inspect
import os
import sys
@@ -156,91 +155,6 @@ MODELS = {
},
}
# The Python modules each backend imports when it loads, with their pip names
# (#424). None of them is a PromptManager requirement: they are optional, and
# llama-cpp-python needs a build for the user's GPU. Nothing is ever installed.
BACKEND_PACKAGES: Dict[str, Tuple[Tuple[str, str], ...]] = {
"gguf": (("llama_cpp", "llama-cpp-python"),),
# The default 8-bit load needs bitsandbytes, and device_map needs accelerate.
"hf": (
("torch", "torch"),
("transformers", "transformers"),
("accelerate", "accelerate"),
("bitsandbytes", "bitsandbytes"),
),
"wd14-swinv2": (("onnxruntime", "onnxruntime"),),
"wd14-vit": (("onnxruntime", "onnxruntime"),),
}
_INSTALL_NOTES = {
"gguf": (
'For a CUDA build: CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python'
),
"wd14-swinv2": "Use onnxruntime-gpu for CUDA.",
"wd14-vit": "Use onnxruntime-gpu for CUDA.",
}
_PIP_NAMES = {
module: pip for packages in BACKEND_PACKAGES.values() for module, pip in packages
}
def _importable(module: str) -> bool:
"""Whether *module* (a top-level name) can be imported; imports nothing."""
if sys.modules.get(module) is not None:
return True
try:
return importlib.util.find_spec(module) is not None
except (ImportError, ValueError, OSError):
return False
def missing_packages(model_type: str) -> List[str]:
"""The pip names of the packages *model_type* needs and cannot import."""
return [
pip
for module, pip in BACKEND_PACKAGES.get(model_type, ())
if not _importable(module)
]
def package_fields(model_type: str, missing: List[str]) -> Dict[str, Any]:
"""The status fields of *missing*: pip names, install command and note."""
missing = list(missing)
return {
"missing_packages": missing,
"install_command": "pip install " + " ".join(missing) if missing else None,
"install_note": _INSTALL_NOTES.get(model_type) if missing else None,
}
def _joined(names: List[str]) -> str:
if len(names) <= 1:
return "".join(names)
return ", ".join(names[:-1]) + " and " + names[-1]
def missing_packages_message(model_type: str, missing: List[str]) -> str:
"""The one sentence every refusal of a model with missing packages uses."""
label = MODELS.get(model_type, {}).get("label", model_type)
fields = package_fields(model_type, missing)
pronoun = "it" if len(missing) == 1 else "them"
message = (
f"{label} needs {_joined(missing)}. Install {pronoun} into ComfyUI's "
f"Python: {fields['install_command']}."
)
if fields["install_note"]:
message += " " + fields["install_note"]
return message
class MissingBackendPackages(RuntimeError):
"""A model whose backend packages cannot be imported (#424)."""
def __init__(self, model_type: str, missing: List[str]):
self.model_type = model_type
self.missing = list(missing)
super().__init__(missing_packages_message(model_type, self.missing))
# Default prompts
DEFAULT_PROMPT = DEFAULT_TAG_TEMPLATE
@@ -253,10 +167,6 @@ SYSTEM_PROMPT = (
)
def _none(_model_type: str) -> List[str]:
return []
def _load_rgb(image_path: str) -> Image.Image:
"""Decode the image as RGB and close the file before inference.
@@ -293,7 +203,6 @@ class AutoTagService:
self,
models_dir: Optional[Path] = None,
model_loader: Optional[ModelLoader] = None,
package_probe: Optional[Callable[[str], List[str]]] = None,
):
"""Initialize the AutoTag service.
@@ -304,16 +213,9 @@ class AutoTagService:
``(kind, backend)`` tuple used by ``generate_tags``.
Defaults to ``_load_model``, which imports the ML
backends lazily. Tests inject a fake here.
package_probe: Callable ``(model_type)`` returning the pip names
of the backend packages that cannot be imported.
Defaults to ``missing_packages`` with the default
loader; an injected loader brings its own backend.
"""
self.logger = get_logger("autotag.service")
self._model_loader: ModelLoader = model_loader or self._load_model
if package_probe is None:
package_probe = missing_packages if model_loader is None else _none
self._package_probe = package_probe
# Determine models directory
if models_dir:
@@ -418,9 +320,6 @@ class AutoTagService:
# Get the actual path (local or cache)
model_status["model_path"] = str(self._get_hf_model_path())
model_status.update(
package_fields(model_type, self._package_probe(model_type))
)
status[model_type] = model_status
return status
@@ -556,9 +455,13 @@ class AutoTagService:
def _load_wd14_tagger(self, model_type: str, use_gpu: bool = True):
"""Load WD14 ONNX model and tag list."""
# A missing onnxruntime is refused by load_model before this runs
# (#424); a ModuleNotFoundError here becomes the same refusal.
import onnxruntime as ort
try:
import onnxruntime as ort
except ImportError:
raise RuntimeError(
"onnxruntime is required for WD14 models. "
"Install with: pip install onnxruntime (or onnxruntime-gpu for CUDA)"
)
import csv
config = MODELS[model_type]
@@ -791,7 +694,6 @@ class AutoTagService:
Raises:
ValueError: If model_type is invalid
RuntimeError: If model not downloaded or loading fails
MissingBackendPackages: If its backend packages cannot be imported
"""
if model_type not in MODELS:
raise ValueError(f"Invalid model type: {model_type}")
@@ -803,11 +705,6 @@ class AutoTagService:
status = self.get_models_status()
if not status[model_type]["downloaded"]:
raise RuntimeError(f"Model {model_type} not downloaded")
missing = status[model_type].get("missing_packages") or []
if missing:
refusal = MissingBackendPackages(model_type, missing)
self.logger.warning(str(refusal))
raise refusal
try:
self._tagger = self._model_loader(model_type, use_gpu)
@@ -815,17 +712,6 @@ class AutoTagService:
self.logger.info(f"Model {model_type} loaded successfully")
return True
except ModuleNotFoundError as e:
# A package find_spec found but that does not import (a broken
# install), or a dependency of one: the same refusal, no traceback.
self._tagger = None
self._current_model_type = None
module = (e.name or "").split(".")[0] or str(e)
refusal = MissingBackendPackages(
model_type, [_PIP_NAMES.get(module, module)]
)
self.logger.warning(str(refusal))
raise refusal from None
except Exception as e:
self.logger.error(f"Failed to load model {model_type}: {e}")
self._tagger = None
+1 -9
View File
@@ -19,7 +19,7 @@ test.beforeEach(async ({page, browserName}) => {
await import('/extensions/promptmanager/prompt_manager.js');
class Node {
constructor(){
this.widgets=[...['text','category','tags','search_text'].map(name=>({name,value:'',inputEl:document.createElement('textarea')})),{name:'save_to_library',type:'toggle',value:true}];
this.widgets=['text','category','tags','search_text'].map(name=>({name,value:'',inputEl:document.createElement('textarea')}));
this.widgets_values=[]; this.inputs=[];
}
addDOMWidget(name,type,element){document.body.append(element);return {element};}
@@ -36,14 +36,6 @@ test.afterEach(async ({page,browserName}) => {
expect(page.classicErrors).toEqual([]);
});
test('Classic loads an older workflow with Save to library on and keeps a saved off (#90)',async({page})=>{
// A workflow saved before the switch held the search UI's slot ('') at the switch's index
const loaded=await page.evaluate(()=>{const widget=node.widgets.find(w=>w.name==='save_to_library');widget.value='';node.configure({});return widget.value;});
expect(loaded).toBe(true);
const off=await page.evaluate(()=>{const widget=node.widgets.find(w=>w.name==='save_to_library');widget.value=false;node.configure({});return [widget.value,node.serialize().widgets_values[4]];});
expect(off).toEqual([false,false]);
});
test('Classic keeps immediate selection, search query and metadata serialization',async({page})=>{
await page.route('**/prompt_manager/search?**', route=>route.fulfill({json:{success:true,results:[{text:'A new portrait',category:'Portrait',tags:['film grain'],rating:5}]}}));
await page.evaluate(()=>{node.widgets.find(w=>w.name==='search_text').value='tag:"film grain" portrait';});
+1 -31
View File
@@ -187,33 +187,6 @@ export async function mutate(page,{css=null,remove=[],clone=[]}={}) {
},token);
}
/**
* #332 D146: the category chip names its origin after the category, in a muted `span.chip-source` that never splits
* (workspace.css `.chip .chip-source`). The concept's chip shows the bare name (app.js a402), so the recorded rule and
* the span are added to the concept page at runtime and the browser lays the chip out: the first `.tags` row's
* non-badge chip under `root`. `category` also replaces the chip's text (an own category the concept never shows).
* Returns the undo.
*/
export const D146_CHIP_SOURCE='.chip .chip-source { color: var(--ca-muted); white-space: nowrap; }';
export async function withChipSource(page,root,origin,category=null) {
const restoreRule=await mutate(page,{css:D146_CHIP_SOURCE});
const token=await page.evaluate(({root,origin,category})=>{
const chip=document.querySelector(root)?.querySelector('.tags')?.querySelector(':scope > .chip:not(.blue)');
if(!chip)throw new Error(`withChipSource: no category chip under ${root}`);
const text=chip.textContent,source=document.createElement('span');
source.className='chip-source';source.textContent=` · ${origin}`;
if(category!==null)chip.textContent=category;
chip.append(source);
const token=`chip-source-${Math.random().toString(36).slice(2)}`;
(window.__chipSources??=new Map()).set(token,()=>{chip.textContent=text;});
return token;
},{root,origin,category});
return async()=>{
await page.evaluate(token=>{window.__chipSources.get(token)();window.__chipSources.delete(token);},token);
await restoreRule();
};
}
/**
* Inventory sections: {chrome, heading, toolbar, filterRow, bulkbar, caption, grid, cards, continuation, panel, library,
* history, noPageScroll}. A section is null when its element is absent. The Images sections (toolbar … panel, #249 C7)
@@ -642,12 +615,9 @@ export function measureInspector(selector) {
primary:node.classList.contains('ca-primary'),...box(node),...font(node),...edges(node),...layout(node),
minHeight:style.minHeight,minWidth:style.minWidth,icon:iconOf(node)};
}
// #332 D146: a category chip's visible origin (`span.chip-source`), null on every other chip.
function chipOf(node) {
const source=node.querySelector(':scope > .chip-source');
return {tag:node.tagName.toLowerCase(),className:classes(node),text:textOf(node),hidden:hiddenOf(node),...box(node),...font(node),...edges(node),
minHeight:getComputedStyle(node).minHeight,
source:when(source,()=>({text:source.textContent,...font(source),...css(source,'whiteSpace'),...box(source)}))};
minHeight:getComputedStyle(node).minHeight};
}
function rowOf(row) {
return {...layout(row),...edges(row),...box(row),chips:[...row.children].map(chipOf)};
+2 -28
View File
@@ -26,7 +26,7 @@ test.beforeEach(async({page,browserName})=>{
window.fixtureApp={registerExtension(extension){this.extension=extension;},ui:{settings:new EventTarget()},graph:{setDirtyCanvas(){}},canvas:{}};
await import('/extensions/promptmanager/prompt_manager.js');
class Node {
constructor(){this.widgets=[...['text','category','tags','search_text'].map(name=>({name,type:'text',value:'',options:{},inputEl:document.createElement('textarea')})),{name:'save_to_library',type:'toggle',value:true,options:{}}];this.inputs=[...['clip','prepend_text','append_text'].map(name=>({name,type:name==='clip'?'CLIP':'STRING',link:null})),...['text','category','tags','search_text'].map(name=>({name,type:'STRING',link:null,widget:{name}})),{name:'save_to_library',type:'BOOLEAN',link:null,widget:{name:'save_to_library'}}];this.properties={};this.size=[550,700];}
constructor(){this.widgets=['text','category','tags','search_text'].map(name=>({name,type:'text',value:'',options:{},inputEl:document.createElement('textarea')}));this.inputs=[...['clip','prepend_text','append_text'].map(name=>({name,type:name==='clip'?'CLIP':'STRING',link:null})),...['text','category','tags','search_text'].map(name=>({name,type:'STRING',link:null,widget:{name}}))];this.properties={};this.size=[550,700];}
findInputByType(type){const index=this.inputs.findIndex(input=>input.type===type&&input.link==null);return index<0?undefined:{index,slot:this.inputs[index]};}
addWidget(type,name,value,callback,options){const widget={type,name,value,callback,options};this.widgets.push(widget);return widget;}
addDOMWidget(name,type,element){const host=document.createElement('div');host.style.height='650px';host.className='fixture-dom-widget';host.append(element);document.body.append(host);return {element};}
@@ -41,32 +41,6 @@ test.beforeEach(async({page,browserName})=>{
});
test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.focusedErrors).toEqual([]);});
test(`${variant} Save to library switch saves its value and loads older workflows on (#90)`, async ({page}) => {
const toggle=page.getByRole('checkbox',{name:'Save to library',exact:true});
await expect(toggle).toBeChecked();
expect(await page.evaluate(()=>node.widgets.map(widget=>widget.name))).toEqual(['text','category','tags','search_text','prepend_text','append_text','save_to_library']);
// The native switch is drawn in the toolbar only: the host keeps it hidden and zero-height, like the text fallbacks
expect(await page.evaluate(()=>{const widget=node.widgets.find(item=>item.name==='save_to_library');return {type:widget.type,hidden:widget.hidden,size:widget.computeSize()};})).toEqual({type:'hidden',hidden:true,size:[0,-4]});
expect(await page.evaluate(()=>node.widgets.find(item=>item.name==='text').computeSize())).toEqual([0,-4]);
await toggle.uncheck();
await expect(page.locator('.pm-focused > .pm-editor-status[role=status]')).toHaveText('Save to library is off: runs of this node are not added to the library.');
const saved=await page.evaluate(()=>node.serialize());
expect(saved.widgets_values[6]).toBe(false);
await page.evaluate(saved=>node.configure(saved),saved);
await expect(toggle).not.toBeChecked();
await page.getByLabel('Main prompt',{exact:true}).fill('Typed while off');
await expect(page.locator('.pm-focused > .pm-editor-status[role=status]')).toHaveText('Save to library is off: runs of this node are not added to the library.');
// A workflow saved before the switch held the editor's own slot ('') at its index
await page.evaluate(()=>node.configure({widgets_values:['Old text','','','','','',''],properties:{}}));
await expect(toggle).toBeChecked();
expect(await page.evaluate(()=>node.widgets.find(widget=>widget.name==='save_to_library').value)).toBe(true);
await toggle.check();
await page.getByLabel('Main prompt',{exact:true}).fill('Typed while on');
await expect(page.locator('.pm-focused > .pm-editor-status[role=status]')).toHaveText('Main prompt is saved when this node runs.');
await page.evaluate(()=>{node.inputs.find(input=>input.name==='save_to_library').link=9;node.onConnectionsChange?.();});
await expect(toggle).toBeDisabled();
});
test(`${variant} library autocompletes filters and searches without changing editor metadata`, async ({page}) => {
const requests=[];
await page.route('**/prompt_manager/categories',route=>route.fulfill({json:{success:true,categories:['Portrait']}}));
@@ -283,7 +257,7 @@ test(`${variant} restores workflow values and disclosures and protects connected
test(`${variant} offers text as a link socket, keeps typed text while linked and through a reload (#343)`,async({page})=>{
const sockets=()=>page.evaluate(()=>Object.fromEntries(node.inputs.map(input=>[input.name,input.widget?.name??null])));
// The hidden text widget no longer owns the input, so the host draws it as a socket; the other widget inputs are unchanged.
expect(await sockets()).toEqual({clip:null,prepend_text:null,append_text:null,text:null,category:'category',tags:'tags',search_text:'search_text',save_to_library:'save_to_library'});
expect(await sockets()).toEqual({clip:null,prepend_text:null,append_text:null,text:null,category:'category',tags:'tags',search_text:'search_text'});
expect(await page.evaluate(()=>node.findInputByType('STRING').slot.name)).toBe('text');
expect(await page.evaluate(()=>node.findInputByType('CLIP').slot.name)).toBe('clip');
const main=page.getByRole('textbox',{name:'Main prompt',exact:true});
+2 -53
View File
@@ -136,8 +136,7 @@ test('panel matches the concept order and copy',async({page,request})=>{
await expect(panel.locator('.detail-inner > p.small')).toHaveText('complete.png');
const badges=panel.locator('.detail-inner > .tags').first();
await expect(badges.locator('span.chip.blue')).toHaveText('Workflow inferred');
// #332 MC8: the chip names where its value came from, in visible text after the category.
await expect(badges.locator('span.chip:not(.blue)')).toHaveText('Recipe category: Inspection · recipe');
await expect(badges.locator('span.chip:not(.blue)')).toHaveText('Recipe category: Inspection');
await expect(badges.locator('span.chip:not(.blue) .ca-visually-hidden')).toHaveText('Recipe category: ');
// Recipe tags are not the positive tags the Images Tags filter matches, so they stay plain chips (D21 ruling).
await expect(panel.locator('.detail-inner > .tags').last().locator('span.chip')).toHaveText(['fixture','geometric']);
@@ -186,53 +185,6 @@ test('panel matches the concept order and copy',async({page,request})=>{
await expect(panel.locator('.detail-inner > button')).toHaveText(['Retry prompt details']);
});
// #332 CUJ 4, CUJ 5 (MC8, D146): every category chip names its origin: the image's own, the prompt's or the recipe's.
const originChip=scope=>scope.locator('span.chip:has(> span.chip-source)');
async function expectOrigin(scope,{prefix,category,origin}) {
await expect(originChip(scope)).toHaveText(`${prefix}${category} · ${origin}`);
await expect(originChip(scope).locator('.ca-visually-hidden')).toHaveText(prefix);
expect(await originChip(scope).locator('.chip-source').evaluate(node=>node.textContent)).toBe(` · ${origin}`);
}
async function putOwnCategory(request,paths,category) {
const {images}=await(await request.get('/prompt_manager/images/output?mode=workspace&subfolder=inspector&limit=40')).json();
const items=paths.map(path=>images.find(image=>image.root_index===0&&image.path===path)).map(({root_index,root_scope,path})=>({root_index,root_scope,path}));
const response=await request.put('/prompt_manager/images/category',{data:{items,category}});await expect(response).toBeOK();
}
test('the category chip names its origin in the side panel and the viewer (#332 CUJ 4, CUJ 5)',async({page,request})=>{
await page.setViewportSize({width:1600,height:1100});
await expect(await request.post('/__test__/reset',{data:{full_inspector:true}})).toBeOK();
// The positive's own category, set through the legacy prompt route; complete.png's recipe keeps "Inspection".
const list=await(await request.get('/prompt_manager/library')).json();const id=list.items.find(item=>item.text.startsWith('Inspector portrait')).id;
await expect(await request.post('/prompt_manager/bulk/category',{data:{prompt_ids:[id],category:'Landscapes'}})).toBeOK();
await page.goto('/prompt_manager/workspace#/images');const panel=inspector(page);
await selectFile(page,'complete.png');await expect(panel.locator('.panel-heading h2')).toHaveText(POSITIVE_TITLE);
await expectOrigin(panel,{prefix:'Recipe category: ',category:'Inspection',origin:'recipe'});
await selectFile(page,'workflow-only.png');await expect(panel.locator('.detail-inner > p.small')).toHaveText('workflow-only.png');
await expectOrigin(panel,{prefix:'Positive prompt category: ',category:'Landscapes',origin:'prompt'});
// An unlinked file shows no category until it has its own.
await selectFile(page,'empty.png');await expect(panel.locator('.panel-heading h2')).toHaveText('empty.png');
await expect(originChip(panel)).toHaveCount(0);
// Own categories: one overrides the prompt's on a linked file, one sits on an unlinked file.
await putOwnCategory(request,['inspector/workflow-only.png'],'Portraits');await putOwnCategory(request,['inspector/empty.png'],'Scans');
await page.reload();
await selectFile(page,'workflow-only.png');await expect(panel.locator('.detail-inner > p.small')).toHaveText('workflow-only.png');
await expectOrigin(panel,{prefix:'Image category: ',category:'Portraits',origin:'this image'});
await selectFile(page,'empty.png');await expect(panel.locator('.panel-heading h2')).toHaveText('empty.png');
await expectOrigin(panel,{prefix:'Image category: ',category:'Scans',origin:'this image'});
await selectFile(page,'complete.png');await expect(panel.locator('.panel-heading h2')).toHaveText(POSITIVE_TITLE);
await expectOrigin(panel,{prefix:'Recipe category: ',category:'Inspection',origin:'recipe'});
// The viewer's full details show the same chips.
let details=await viewerDetails(page,'workflow-only.png');
await expectOrigin(details,{prefix:'Image category: ',category:'Portraits',origin:'this image'});
await page.keyboard.press('Escape');await expect(page.locator('dialog#viewer-dialog[open]')).toHaveCount(0);
details=await viewerDetails(page,'complete.png');
await expectOrigin(details,{prefix:'Recipe category: ',category:'Inspection',origin:'recipe'});
await page.keyboard.press('Escape');await expect(page.locator('dialog#viewer-dialog[open]')).toHaveCount(0);
details=await viewerDetails(page,'empty.png');
await expectOrigin(details,{prefix:'Image category: ',category:'Scans',origin:'this image'});
});
test('tag chip filters Images by that tag',async({page,request})=>{
await expect(await request.post('/__test__/reset',{data:{full_inspector:true}})).toBeOK();
const list=await(await request.get('/prompt_manager/library')).json();const id=list.items.find(item=>item.text.startsWith('Inspector portrait')).id;
@@ -724,9 +676,6 @@ test('collection viewer uses real public transforms and resets on navigation',as
async function slideshowFixture(page,request,preferences={videoAutoplay:false,videoMute:true,videoLoop:false}) {
await expect(await request.post('/__test__/reset',{data:{full_inspector:true,include_video:true}})).toBeOK();
await page.goto('/prompt_manager/workspace#/images');
// Install the fake clock before the fixture wraps setTimeout: the wrapped timer (which the viewer session captures)
// must be the fake one, so page.clock.runFor alone decides when the 3500 ms slideshow deadline fires.
await page.clock.install();
await page.evaluate(async preferences=>{
if(preferences) localStorage.setItem('gallerySettings',JSON.stringify(preferences));
const [{createMediaViewer},{createInspectionController},{createScope},{createPromptSender}]=await Promise.all([
@@ -746,7 +695,7 @@ async function slideshowFixture(page,request,preferences={videoAutoplay:false,vi
},preferences);
await expect(mediaDialog(page).getByRole('button',{name:'Zoom in',exact:true})).toHaveAttribute('aria-disabled','false');
await expect(mediaDialog(page).locator('#viewer-details > div > p.small')).toHaveText('complete.png');
await page.clock.pauseAt(new Date(Date.now()+1000));
await page.clock.install();await page.clock.pauseAt(new Date(Date.now()+1000));
}
const slideshowButton=page=>mediaDialog(page).locator('#viewer-toolbar').getByRole('button',{name:/^(Play|Pause|Stop) slideshow$/});
const timerCount=page=>page.evaluate(()=>window.slideshowTimers.size);
+1 -57
View File
@@ -1,7 +1,7 @@
import {writeFile} from 'node:fs/promises';
import {test,expect} from './fixtures.mjs';
import {captureCanonicalState} from './history-canonical-comparison.mjs';
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact,measureInspector,withChipSource} from './concept-inventory.mjs';
import {SIZES,THEMES,MOTIONS,seedConcept,openConcept,openProduction,inventoryOf,diffInventory,mutate,stripExact} from './concept-inventory.mjs';
// Task F5 of #249: the production Prompt history (#/library/<id>) against the immutable concept's (#prompt/alpine,
// app.js renderHistory a587–622, the dialogs a1282–1304, style.css 1059–1179 and 1546–1567), on the concept-shaped real
@@ -272,62 +272,6 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`history geomet
} finally {await host.evaluate(()=>window.conceptHost.dispose());await host.close();}
});
/**
* D145 (#332): the History media inspector (a recorded run's current file, `recorded-run.mjs`) shows the file's own
* category with the chip rule of the Images side panel and viewer (D146). The concept has no recorded-run inspector, so
* its chip is the concept viewer's own category chip (app.js a402, imageDetails(i, true)) with the recorded D146 origin
* and the own category added at runtime (`withChipSource`). The chip's own place depends on what precedes it in either
* inspector, so its box is compared by size only.
*/
const OWN_CATEGORY='Portraits';
const chipOnly=chip=>{const {left,top,source,...rest}=chip;return {...rest,source:source&&(({left,top,...own})=>own)(source)};};
const categoryChipOf=async(page,root)=>(await page.evaluate(measureInspector,root)).badges.chips.find(chip=>chip.source);
async function conceptViewerChip(page,theme) {
await openConcept(page,'images',theme);await page.mouse.move(0,0);
await page.locator('.gallery-body > .image-grid > .media-card').first().locator('.image-open').click();
await expect(page.locator('#viewer-dialog[open] #viewer-details .panel-heading h2')).toHaveText(TITLE);
const restore=await withChipSource(page,'#viewer-details','this image',OWN_CATEGORY);await settle(page);
const chip=await categoryChipOf(page,'#viewer-details');
await restore();
return chip;
}
/** Recorded run for the baseline's newest run, its one reported output opened in the viewer. */
async function productionRunChip(page,theme,positiveId) {
await openProductionHistory(page,theme,'no-preference',positiveId);
await page.getByRole('button',{name:'Show repeat run for Quality baseline',exact:true}).click();
const run=page.getByRole('dialog',{name:'Recorded run',exact:true});
await expect(run).toBeVisible();
const view=run.getByRole('button',{name:'View current file',exact:true});
await expect(view).toBeEnabled();await view.click();
const details=page.locator('#viewer-dialog[open] #viewer-details');
await expect(details.locator('.chip-source')).toHaveText(' · this image');
await page.mouse.move(0,0);await settle(page);
return categoryChipOf(page,'#viewer-dialog[open] #viewer-details');
}
async function setOwnCategory(request,path,category) {
const listing=await (await request.get('/prompt_manager/images/output?mode=workspace&limit=40')).json();
const root_scope=listing.images.find(item=>item.root_index===0).root_scope;
const response=await request.put('/prompt_manager/images/category',{data:{items:[{root_index:0,root_scope,path}],category}});
await expect(response).toBeOK();
}
for(const [width,height] of SIZES)for(const theme of THEMES)test(`history media inspector category chip matches the concept ${width}x${height} ${theme} (D145)`,async({page,request},testInfo)=>{
await page.setViewportSize({width,height});
const {manifest}=testInfo;
// The baseline's newest image: the output of the run its dot opens (the fixture records one run per image).
const baseline=manifest.images.find(image=>image.prompt==='alpine'&&image.variant===0);
await setOwnCategory(request,baseline.path,OWN_CATEGORY);
const reference=await conceptViewerChip(page,theme);
const actual=await productionRunChip(page,theme,manifest.prompts[0].positive_id);
await captureCanonicalState(page,testInfo,`production-history-run-chip-${width}-${theme}`,'#viewer-dialog[open]');
expect.soft(reference,'concept chip').toMatchObject({tag:'span',className:'chip',text:`${OWN_CATEGORY} · this image`,hidden:null});
// D21: the hidden prefix names the scope; D146 as above.
expect.soft(diffInventory({...chipOnly(reference),hidden:'Image category: '},chipOnly(actual)),'chip differences after D21, D145, D146').toEqual([]);
});
// The comparison is only worth something if it notices a change (spec §7 mutation check): one CSS mutation (the variant
// dot's width) and one DOM mutation (Compare negatives removed), each applied after an exact match.
test('history comparison detects mutations',async({page},testInfo)=>{
+4 -117
View File
@@ -85,17 +85,15 @@ function listInfo(card,actionsWidth) {
/** Recorded Images deviations only (spec §8); everything else in the concept inventory must equal production. */
const nocase=(a,b)=>{const x=a.toLowerCase(),y=b.toLowerCase();return x<y?-1:x>y?1:a<b?-1:a>b?1:0;};
export function mapImagesConcept(reference,{manifest,now=new Date(manifest.now*1000),imageCategories=[]}) {
export function mapImagesConcept(reference,{manifest,now=new Date(manifest.now*1000)}) {
const paths=new Map(manifest.images.map(image=>[image.filename,image.path]));
// D14: the sort appends production's two sorts. Save view is the concept's own (saved views, #249 D16 retired).
const toolbar={...reference.toolbar};
toolbar.items=toolbar.items.map(item=>item.id==='sort'?{...item,options:[...item.options,'Oldest first','Name Z–A']}:item);
// D36: the category list is the Library's real list, sorted alphabetically (database/library.py `ORDER BY category
// COLLATE NOCASE, category`); the concept lists its samples in prompt order (app.js 348). Only the order is mapped.
// D141 (#332): the list also holds the listed images' own categories (`GET /prompt_manager/images/categories`),
// sorted in with the Library's.
toolbar.items=toolbar.items.map(item=>item.id==='category-filter'
?{...item,options:[item.options[0],...[...item.options.slice(1),...imageCategories].sort(nocase)]}:item);
?{...item,options:[item.options[0],...item.options.slice(1).sort(nocase)]}:item);
// D29: no "Illustrative previews" sample tag.
const filterRow=withoutItems(reference.filterRow,item=>String(item.className).includes('sample-tag'));
filterRow.items=filterRow.items.map(item=>{
@@ -165,9 +163,9 @@ const pick=inventory=>stripExact(inventory);
// The concept's own sample categories, in its first-appearance order (app.js 348): what D36 reorders.
const CONCEPT_CATEGORIES=['All categories','Landscapes','Architecture','Environments','Illustration'];
function compare(reference,actual,manifest,sections=SECTIONS,imageCategories=[]) {
function compare(reference,actual,manifest,sections=SECTIONS) {
const only=inventory=>Object.fromEntries(sections.map(key=>[key,inventory[key]]));
return diffInventory(pick(only(mapImagesConcept(reference,{manifest,imageCategories}))),pick(only(actual)));
return diffInventory(pick(only(mapImagesConcept(reference,{manifest}))),pick(only(actual)));
}
/** The comparison as a gate: throws, naming every differing path, unless production equals the mapped concept. */
@@ -472,117 +470,6 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`images selecti
expect(compare(oracle,await inventoryOf(page,'#workspace-main'),manifest,['bulkbar']).map(item=>item.path)).toContain('bulkbar.bar.paddingTop');
});
/** #332: an own category on one listed image, set through the real route (PUT /prompt_manager/images/category). */
const IMAGE_CATEGORY='Portraits';
async function setImageCategory(request,path,category) {
const listing=await (await request.get('/prompt_manager/images/output?mode=workspace&limit=40')).json();
const root_scope=listing.images.find(item=>item.root_index===0).root_scope;
const response=await request.put('/prompt_manager/images/category',{data:{items:[{root_index:0,root_scope,path}],category}});
await expect(response).toBeOK();
expect(await response.json()).toMatchObject({success:true,category,updated:1});
}
/**
* Runs in the page: the open Set category dialog (the concept's `simpleInput`, app.js 1384-1390 and 1138): its title,
* the body's tree, the field, the input and its suggestions (`datalist`), and the actions with every button's box,
* all relative to the dialog, with the concept's geometry and tokens.
*/
function measureCategoryDialog() {
const dialog=document.querySelector('dialog#modal[open]');
const css=(node,...keys)=>node?Object.fromEntries(keys.map(key=>[key,getComputedStyle(node)[key]])):null;
const name=node=>[node.localName,...[...node.classList].sort()].join('.');
const text=node=>node?.textContent.trim()??null;
const origin=dialog.getBoundingClientRect();
const box=node=>{const rect=node.getBoundingClientRect();return {left:Math.round(rect.left-origin.left),top:Math.round(rect.top-origin.top),
width:Math.round(rect.width),height:Math.round(rect.height)};};
const body=dialog.querySelector('#modal-body');
const field=body.querySelector(':scope > .field');
const label=field?.querySelector(':scope > label'),input=field?.querySelector(':scope > input');
const list=body.querySelector(':scope > datalist');
const actions=body.querySelector(':scope > .dialog-actions');
return {
title:text(dialog.querySelector('#modal-title')),
tree:[...body.children].map(name),
field:field&&{tree:[...field.children].map(name),...css(field,'display','flexDirection','rowGap','marginBottom'),...box(field)},
label:label&&{text:text(label),for:label.htmlFor,...css(label,'fontSize','fontWeight','color')},
input:input&&{id:input.id,type:input.type,value:input.value,list:input.getAttribute('list'),
...css(input,'height','paddingLeft','fontSize','color','backgroundColor','borderTopColor','borderTopLeftRadius'),...box(input)},
suggestions:list&&{id:list.id,shown:list.getClientRects().length>0,options:[...list.options].map(option=>option.value)},
actions:{tree:[...actions.children].map(name),...css(actions,'display','justifyContent','columnGap','rowGap','flexWrap','marginTop'),...box(actions),
buttons:[...actions.children].map(button=>({text:text(button),className:button.className,
...css(button,'backgroundColor','color','fontSize','paddingTop','paddingLeft','borderTopColor','minHeight'),...box(button)}))},
width:Math.round(origin.width),height:Math.round(origin.height),
};
}
/**
* The concept's own open dialog as production records it, changed at runtime so the browser lays it out: D140 (#332)
* empties the field (the concept pre-fills its demo "Landscapes"), lists the toolbar's categories as the field's
* suggestions (`datalist#workspace-input-suggestions`) and adds "Use prompt category", a plain button of the concept's
* own Cancel, between Cancel and the primary action; D8 names the primary action "Apply" ("Apply in prototype").
*/
async function categoryDialogAsRecorded(page,suggestions) {
await mutate(page,{clone:[{selector:'dialog#modal[open] .dialog-actions > button:first-child',text:'Use prompt category'}]});
await page.evaluate(suggestions=>{
const body=document.querySelector('dialog#modal[open] #modal-body');
const input=body.querySelector('#action-input');input.value='';input.setAttribute('list','workspace-input-suggestions');
const list=document.createElement('datalist');list.id='workspace-input-suggestions';
for(const value of suggestions){const option=document.createElement('option');option.value=value;list.append(option);}
body.querySelector(':scope > .field').after(list);
body.querySelector('.dialog-actions > .ca-primary').textContent='Apply';
},suggestions);
await settle(page);
return page.evaluate(measureCategoryDialog);
}
async function openCategoryDialog(page,scope) {
await page.locator(scope).locator(CARDS).first().locator('.image-overlays > label > input[type="checkbox"]').check();
await expect(page.locator('#bulkbar > .bulkbar > strong')).toHaveText('1 selected');
await page.locator('#bulkbar').getByRole('button',{name:'Set category',exact:true}).click();
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Categorize selected images');
await page.mouse.move(0,0);await settle(page);
return page.evaluate(measureCategoryDialog);
}
// D140 and D141 (#332): with one image's own category set, the toolbar's category options and the Set category dialog
// against the concept's (app.js 348 and 1384-1390). The dialog is compared with the concept page itself changed at
// runtime as recorded (`categoryDialogAsRecorded`); unmapped, it differs only in the recorded parts.
for(const [width,height] of SIZES)for(const theme of THEMES)test(`images category options and Set category dialog match the concept ${width}x${height} ${theme}`,async({page,request},testInfo)=>{
await page.setViewportSize({width,height});
const manifest=testInfo.manifest;
await setImageCategory(request,manifest.images.at(-1).path,IMAGE_CATEGORY);
await openConceptLoaded(page,theme,'no-preference');
const reference=await inventoryOf(page,'#main');
const referenceDialog=await openCategoryDialog(page,'#main');
await captureCanonicalState(page,testInfo,`concept-images-category-${width}-${theme}`,'dialog#modal');
const suggestions=[...CONCEPT_CATEGORIES.slice(1),IMAGE_CATEGORY].sort(nocase);
const oracle=await categoryDialogAsRecorded(page,suggestions);
await page.keyboard.press('Escape');await expect(page.locator('dialog#modal[open]')).toHaveCount(0);
await openProductionLoaded(page,theme,'no-preference');
await expect(page.locator('#category-filter option')).toHaveText(['All categories',...suggestions]);
const actual=await inventoryOf(page,'#workspace-main');
const actualDialog=await openCategoryDialog(page,'#workspace-main');
await captureCanonicalState(page,testInfo,`production-images-category-${width}-${theme}`,'dialog#modal');
const path=testInfo.outputPath('images-category-geometry.json');
await writeFile(path,JSON.stringify({referenceDialog,oracle,actualDialog,reference:reference.toolbar,actual:actual.toolbar},null,2));
await testInfo.attach('images-category-geometry',{path,contentType:'application/json'});
// The concept itself: its sample categories and its pre-filled simpleInput with Cancel / Apply in prototype.
expect.soft(reference.toolbar.items.find(item=>item.id==='category-filter').options,'concept categories').toEqual(CONCEPT_CATEGORIES);
expect.soft(referenceDialog,'concept dialog').toMatchObject({title:'Categorize selected images',label:{text:'Category'},
input:{id:'action-input',value:'Landscapes',list:null},suggestions:null});
expect.soft(referenceDialog.actions.buttons.map(button=>button.text),'concept actions').toEqual(['Cancel','Apply in prototype']);
// D141: the toolbar options add the image's own category, sorted in (D36).
expect.soft(compare(reference,actual,manifest,['toolbar'],[IMAGE_CATEGORY]),'toolbar differences after D14, D36, D141').toEqual([]);
// D140 and D8: the dialog equals the concept page laid out as recorded.
expect.soft(diffInventory(oracle,actualDialog),'dialog differences after D8, D140').toEqual([]);
// Unmapped, the concept differs only in the field's value and suggestions, the body's datalist and the actions.
const recorded=/^(input\.(value|list)|tree\.|suggestions|actions\.)/;
expect.soft(diffInventory(referenceDialog,actualDialog).map(item=>item.path).filter(item=>!recorded.test(item)),'unrecorded dialog differences').toEqual([]);
expect.soft(actualDialog.suggestions,'suggestions').toEqual({id:'workspace-input-suggestions',shown:false,options:suggestions});
});
// The comparison is only worth something if it notices a change (spec §7 mutation check): one CSS mutation (the
// `.media-info` padding) and one DOM mutation (the Inspect buttons removed), each applied after an exact match.
test('images comparison detects mutations',async({page},testInfo)=>{
+6 -27
View File
@@ -1,7 +1,7 @@
import {writeFile} from 'node:fs/promises';
import {test,expect} from './fixtures.mjs';
import {captureCanonicalState} from './history-canonical-comparison.mjs';
import {SIZES,THEMES,seedConcept,openConcept,openProduction,inspectorOf,diffInventory,mutate,countCopy,saveImageTags,withChipSource} from './concept-inventory.mjs';
import {SIZES,THEMES,seedConcept,openConcept,openProduction,inspectorOf,diffInventory,mutate,countCopy,saveImageTags} from './concept-inventory.mjs';
// Task D4 of #249: the Images side panel (app.js imageDetails a400–414) and the viewer's full details (imageDetails(i,
// true)) against the immutable concept, on the concept-shaped real fixture, at 1600x1100 and 390x844 in both themes.
@@ -13,8 +13,6 @@ test.use({locale:'en-US',timezoneId:'UTC'});
const CARDS='.gallery-body > .image-grid > .media-card';
const TITLE='Alpine observatory';
// D146 (#332): alpine_00001.png's recipe has no category, so its chip shows the positive's, " · prompt" (D39).
const ORIGIN='prompt';
const FIXTURE_IMAGES=24;
// The concept's sticky panel never scrolls; production's scrolls inside the window above 700px (D20/D21 side panel).
const PANEL_INSET=40;
@@ -40,18 +38,14 @@ function withFocusablePrompts(inventory) {
return {...inventory,controls,sections:inventory.sections.map(section=>({...section,text:{...section.text,tabIndex:0}}))};
}
/**
* D21: the category chip names its scope in a visually hidden prefix; here the linked positive's (D39 fallback). Its
* visible origin (D146, #332: " · prompt") is the concept page's own chip with `span.chip-source` added at runtime
* (`withChipSource`), so `reference.panel` and `viewerPlain` already carry it.
*/
/** D21: the category chip names its scope in a visually hidden prefix; here the linked positive's (D39 fallback). */
function withCategoryScope(inventory) {
const [badge,category,...rest]=inventory.badges.chips;
return {...inventory,badges:{...inventory.badges,chips:[badge,{...category,hidden:'Positive prompt category: '},...rest]}};
}
/**
* Recorded inspector deviations only (spec §8, TESTING.md D3, D20–D22, D34, D38, D117 and D146); everything else must
* Recorded inspector deviations only (spec §8, TESTING.md D3, D20–D22, D34, D38 and D117); everything else must
* equal production. `reference.viewerPlain` is the concept's own full details
* with its tag chips made plain (D20), D117's summary button added and, under a coarse pointer, D34's targets, all at
* runtime: the browser lays out what follows them, as the Images spec's removed toolbar items do.
@@ -118,7 +112,6 @@ async function openProductionPanel(page,theme) {
*/
async function plainViewerChips(page) {
const restoreTargets=await mutate(page,{css:D34_COARSE_TARGETS});
const restoreSource=await withChipSource(page,'#viewer-details',ORIGIN);
await page.evaluate(()=>{
window.__plainChips=[...document.querySelectorAll('#viewer-details .tags > button.chip')].map(button=>{
const span=document.createElement('span');span.className='chip';span.textContent=button.textContent;
@@ -133,7 +126,7 @@ async function plainViewerChips(page) {
return async()=>{
await page.evaluate(()=>{for(const [span,button] of window.__plainChips)span.replaceWith(button);delete window.__plainChips;
window.__summaryButton.remove();delete window.__summaryButton;});
await restoreSource();await restoreTargets();
await restoreTargets();
};
}
@@ -169,11 +162,7 @@ async function viewerOf(page,concept,mutation=null) {
return {viewerDetails,viewerPlain,fileControls};
}
async function inventory(page,concept=false) {
// D146: the concept panel's category chip with its visible origin, laid out by the concept page itself.
const restoreSource=concept?await withChipSource(page,'#image-inspector',ORIGIN):null;
if(concept)await settle(page);
const {panel}=await inspectorOf(page);
await restoreSource?.();
const {viewerDetails,viewerPlain,fileControls}=await viewerOf(page,concept);
return concept?{panel,viewerDetails,viewerPlain,fileControls}:{panel,viewerDetails,fileControls};
}
@@ -299,8 +288,7 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`inspector pane
expect.soft(panel.sections.map(section=>section.label.color)[0],'concept role colors').not.toBe(panel.sections[1].label.color);
expect.soft(panel.facts,'concept facts').toMatchObject({columns:2,columnGap:'12px',rowGap:'12px',fontSize:width>700?'11px':'12px'});
expect.soft(panel.footer,'concept footer').toMatchObject({paddingTop:'13px',paddingRight:'17px',paddingBottom:'13px',paddingLeft:'17px'});
// The concept's bare "Landscapes" with the D146 origin added at runtime (`withChipSource`).
expect.soft(panel.badges.chips.map(chip=>chip.text),'concept badges').toEqual(['Execution recorded','Landscapes · prompt']);
expect.soft(panel.badges.chips.map(chip=>chip.text),'concept badges').toEqual(['Execution recorded','Landscapes']);
expect.soft(reference.viewerDetails.structure.inner.slice(-2),'concept full details').toEqual(['details','details']);
expect.soft(diff,'differences after the recorded deviations').toEqual([]);
@@ -334,10 +322,8 @@ test('a recipe-linked image without recipe metadata shows its positive prompt me
await expect(panelOf(page).locator('.panel-heading h2')).toHaveText('Quiet geometry');
const rows=panelOf(page).locator('.detail-inner > .tags');
await expect(rows).toHaveCount(2);
await expect(rows.first().locator('.chip')).toHaveText(['Execution recorded','Positive prompt category: Architecture · prompt']);
await expect(rows.first().locator('.chip')).toHaveText(['Execution recorded','Positive prompt category: Architecture']);
await expect(rows.first().locator('.chip .ca-visually-hidden')).toHaveText('Positive prompt category: ');
// D146 (#332): the visible origin names where the value came from.
await expect(rows.first().locator('.chip .chip-source')).toHaveText(' · prompt');
await expect(rows.nth(1).locator('button.chip')).toHaveText(['minimal','warm_light','editorial']);
await expect(rows.nth(1).locator('span.chip')).toHaveCount(0);
await expect(panelOf(page).locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Prompt history']);
@@ -414,13 +400,6 @@ test('inspector comparison detects mutations',async({page})=>{
expect(()=>assertInspectorMatch(reference,withoutSummary,viewport)).toThrow(/viewerDetails\.structure\.inner/);
assertInspectorMatch(reference,await viewerMutated(),viewport);
// D146: the visible origin is part of the mapped concept, so losing it is a difference.
const restoreSource=await mutate(page,{remove:['#image-inspector .chip-source']});
const sourceMutated=await measure();
expect(()=>assertInspectorMatch(reference,sourceMutated,viewport)).toThrow(/panel\.badges\.chips\.1\.source/);
await restoreSource();await settle(page);
assertInspectorMatch(reference,await measure(),viewport);
await mutate(page,{remove:['#image-inspector > .detail-footer']});
const domMutated=await measure();
expect(()=>assertInspectorMatch(reference,domMutated,viewport)).toThrow(/panel\.footer/);
+79 -3
View File
@@ -1,9 +1,11 @@
// #328: the legacy admin and gallery auto-tag batch and review keep working beside System › Auto-tagging. Since #330
// the pages download and unload models only through System (legacy-system-links.spec.mjs); a batch never downloads
// one. The fixture's fake download (`/__test__/tagging/download`) is held so nothing is ever fetched.
// #328: the legacy admin and gallery auto-tag tools keep working beside System › Auto-tagging. The model download is
// the `model_download` Activity job; the pages' stream attaches to it. No model is ever downloaded: the fixture's
// fake download waits until the test releases it (`/__test__/tagging/download`).
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
import { test, expect } from './fixtures.mjs';
const TITLE = 'Tagging model download · WD14 ViT';
async function post(request, url, data) {
await expect(await request.post(url, data === undefined ? {} : { data })).toBeOK();
}
@@ -31,6 +33,61 @@ test.afterEach(async ({ page, request, browserName }) => {
await post(request, '/__test__/jobs/restart');
});
async function startDownload(page, request) {
await page.locator('#autoTagBtn').click();
const status = page.locator('#wd14VitModelStatus');
await status.getByRole('button', { name: 'Download', exact: true }).click();
const modal = page.locator('#autoTagDownloadModal');
await expect(modal).toBeVisible();
await expect(page.locator('#downloadStatus')).toHaveText('Downloading…');
await expect(page.locator('#downloadProgressBar')).toHaveClass(/animate-pulse/);
await expect(modal).toContainText('The download continues in Activity.');
await expect(page.locator('#cancelDownloadBtn')).toHaveText('Close');
const running = (await downloads(request)).filter(job => job.state === 'running');
expect(running.map(job => [job.title, job.cancellable])).toEqual([[TITLE, false]]);
return { modal, status };
}
test('admin download modal closes on complete and Activity lists the job without Cancel', async ({ page, request }) => {
await page.goto('/prompt_manager/admin');
const { modal, status } = await startDownload(page, request);
await page.goto('/prompt_manager/workspace#/activity');
const item = page.locator('#workspace-main section.surface > .activity-item')
.filter({ has: page.getByRole('link', { name: new RegExp(`${TITLE}$`) }) });
await expect(item.locator('.chip')).toHaveText('Running');
await expect(item.locator('progress')).not.toHaveAttribute('value');
await expect(item.getByRole('button', { name: 'Cancel job', exact: true })).toHaveCount(0);
// Back on the admin page a second download of the same model attaches to the running job.
await page.goto('/prompt_manager/admin');
await startDownload(page, request);
await post(request, '/__test__/tagging/download', { mode: 'succeed' });
await expect(modal).toBeHidden();
await expect(status).toContainText('Downloaded');
expect((await downloads(request)).map(job => [job.title, job.state])).toEqual([[TITLE, 'finished']]);
});
test('gallery download modal closes on complete', async ({ page, request }) => {
await page.goto('/prompt_manager/gallery');
const { modal, status } = await startDownload(page, request);
await post(request, '/__test__/tagging/download', { mode: 'succeed' });
await expect(modal).toBeHidden();
await expect(page.locator('#autoTagModal')).toBeVisible();
await expect(status).toContainText('Downloaded');
expect((await downloads(request)).map(job => job.state)).toEqual(['finished']);
});
test('a failed download closes the modal and says so', async ({ page, request }) => {
await page.goto('/prompt_manager/gallery');
const { modal, status } = await startDownload(page, request);
await post(request, '/__test__/tagging/download', { mode: 'fail' });
await expect(modal).toBeHidden();
await expect(page.getByText('Download failed: Download failed')).toBeVisible();
await expect(status.getByRole('button', { name: 'Download', exact: true })).toBeVisible();
expect((await downloads(request)).map(job => job.state)).toEqual(['failed']);
});
// (wd14-vit is not downloaded in these tests: beforeEach marks it so the download tests can offer it.)
// The gallery's batch reader keyed on `data.status` values the server never sends, so its modal never moved or
// closed (#328 Problem 5). It now reads `type` frames as the admin page does.
@@ -114,6 +171,25 @@ test('admin review opens the first image', async ({ page }) => {
await expect(page.getByText('Failed to start review mode')).toHaveCount(0);
});
test('gallery unload works and is refused while the gate is held', async ({ page, request }) => {
await page.goto('/prompt_manager/gallery');
await runGalleryBatch(page); // keeps the model in memory
await page.locator('#autoTagBtn').click();
const unload = page.locator('#unloadModelBtn');
await expect(unload).toBeVisible();
// A held download owns the model gate: Unload is refused with the server's error.
const held = await request.post('/prompt_manager/jobs/model-download', { data: { model_type: 'gguf' } });
expect(held.status()).toBe(202);
await unload.click();
await expect(page.getByText(/^Failed to unload model: A tagging model is busy/)).toBeVisible();
await post(request, '/__test__/tagging/download', { mode: 'succeed' });
await expect.poll(async () => (await downloads(request)).map(job => job.state)).toEqual(['finished']);
await unload.click();
await expect(page.getByText('WD14-SWINV2 model unloaded successfully')).toBeVisible();
});
/** The admin review lists every row, videos too: Skip any image that is not one until the confirmation shows. */
async function untilAutoTagged(page, confirm) {
// Each image's skip check answers before its confirmation or its error shows.
+18 -141
View File
@@ -15,40 +15,9 @@ const LABELS = Object.freeze({
diagnostics: 'Diagnostics & logs',
});
async function openPageOptions(page) {
await page.locator('#pageOptionsBtn').click();
await expect(page.locator('#settingsModal')).toBeVisible();
}
/** The auto-tag modal with its model rows read from /autotag/models. */
async function openAutoTag(page) {
const read = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/models'));
await page.locator('#autoTagBtn').click();
await read;
await expect(page.locator('#autoTagModal')).toBeVisible();
}
const notDownloaded = (page, request) => post(request, '/__test__/tagging/models', { unavailable: ['gguf'] });
const modelLoaded = (page, request) => post(request, '/__test__/tagging/loaded', { model: 'wd14-vit' });
// One row per System link of the design's §2 map: the page, the control's label, how to find it, the section it opens
// and what a test must arrange before the control shows.
const LINKS = Object.freeze([
{ page: 'admin', label: 'Settings', locate: page => page.locator('#settingsBtn'), section: 'overview' },
{ page: 'admin', label: 'Maintenance', locate: page => page.locator('#maintenanceBtn'), section: 'maintenance' },
{ page: 'admin', label: 'Backup DB', locate: page => page.locator('#backupBtn'), section: 'backups' },
{ page: 'admin', label: 'Restore DB', locate: page => page.locator('#restoreBtn'), section: 'backups' },
{ page: 'gallery', label: 'Settings', locate: page => page.locator('#settingsBtn'), section: 'appearance' },
{ page: 'admin', label: 'Manage in System (model row)', arrange: notDownloaded, reveal: openAutoTag,
locate: page => page.locator('#ggufModelStatus a'), section: 'tagging' },
{ page: 'gallery', label: 'Manage in System (model row)', arrange: notDownloaded, reveal: openAutoTag,
locate: page => page.locator('#ggufModelStatus a'), section: 'tagging' },
{ page: 'gallery', label: 'Manage in System (Unload Model)', arrange: modelLoaded, reveal: openAutoTag,
locate: page => page.locator('#unloadModelBtn'), section: 'tagging' },
{ page: 'gallery', label: 'Page options Maintenance', reveal: openPageOptions,
locate: page => page.locator('#pageOptionsMaintenanceLink'), section: 'maintenance' },
{ page: 'gallery', label: 'Page options Sources & scans', reveal: openPageOptions,
locate: page => page.locator('#pageOptionsSourcesLink'), section: 'sources' },
{ page: 'admin', label: 'Diagnostics', locate: page => page.locator('#diagnosticsBtn'), section: 'diagnostics' },
{ page: 'admin', label: 'View Logs', locate: page => page.locator('#logsBtn'), section: 'diagnostics' },
]);
@@ -85,31 +54,17 @@ async function openLink(page, request, link) {
return control;
}
// Each row opens its section in a new tab and sends no request from the legacy page (the design's `a System link sends
// no request from the legacy page`, checked per row so one slow page never times out the whole table).
for (const link of LINKS) {
test(`legacy ${link.page} ${link.label} opens System ${link.section}`, async ({ page, context, request }) => {
// The gallery's thumbnail startup check (K9) reads its status on a timer of its own; it is off here.
await page.addInitScript(() => {
localStorage.setItem('gallerySettings', JSON.stringify({ checkThumbnailsAtStartup: false }));
});
const control = await openLink(page, request, link);
const legacyUrl = page.url();
await expect(control).toHaveAttribute('href', `workspace#/system/${link.section}`);
await expect(control).toHaveAttribute('target', '_blank');
await expect(control).toHaveAttribute('rel', 'noopener');
// The page's own start-up reads (list, film strips, settings) finish first; then the click must add none.
// Lazy thumbnails may still load whenever the layout changes; they are not the click's.
await page.waitForLoadState('networkidle');
const sent = [];
const record = item => { if (item.resourceType() !== 'image') sent.push(`${item.method()} ${item.url()}`); };
page.on('request', record);
const opened = context.waitForEvent('page');
await control.click();
const system = await opened;
await expect(system).toHaveURL(new RegExp(`/prompt_manager/workspace#/system/${link.section}$`));
page.off('request', record);
expect(sent, 'a System link sends no request from the legacy page').toEqual([]);
// System's page heading is "System"; the open section is the pressed button of its section switcher.
const main = system.locator('#workspace-main');
await expect(main.getByRole('heading', { level: 1, name: 'System', exact: true })).toBeVisible();
@@ -120,6 +75,24 @@ for (const link of LINKS) {
});
}
test('a System link sends no request from the legacy page', async ({ page, context, request }) => {
for (const link of LINKS) {
const control = await openLink(page, request, link);
// The page's own start-up reads (list, film strips, settings) finish first; then the click must add none.
await page.waitForLoadState('networkidle');
const sent = [];
const record = item => sent.push(`${item.method()} ${item.url()}`);
page.on('request', record);
const opened = context.waitForEvent('page');
await control.click();
const system = await opened;
await system.waitForLoadState('domcontentloaded');
page.off('request', record);
expect(sent, `${link.page} ${link.label}`).toEqual([]);
await system.close();
}
});
test('kept admin flows still work', async ({ page, context }) => {
await page.goto(PAGES.admin);
const texts = page.locator('#resultsList .prompt-text');
@@ -188,99 +161,3 @@ test('kept admin flows still work', async ({ page, context }) => {
await tab.close();
}
});
test('Page options keeps the page-only settings admin', async ({ page, request }) => {
const before = (await (await request.get('/prompt_manager/settings')).json()).settings;
await page.goto(PAGES.admin);
await expect(page.locator('#resultsList .prompt-text').first()).toBeVisible();
await page.locator('#pageOptionsBtn').click();
const dialog = page.locator('#settingsModal');
await expect(dialog).toBeVisible();
await expect(dialog.locator('h3')).toHaveText('Page options');
// Exactly the page-only setting (G5): the list's Infinite Scrolling default.
expect(await dialog.locator('input, select, textarea').evaluateAll(fields => fields.map(field => field.id)))
.toEqual(['infiniteScrollDefault']);
await expect(page.locator('#infiniteScrollDefault')).not.toBeChecked();
await page.locator('#infiniteScrollDefault').check();
const saved = page.waitForRequest(item => item.method() === 'POST' && item.url().endsWith('/prompt_manager/settings'));
await page.locator('#saveSettings').click();
expect((await saved).postDataJSON()).toEqual({ infinite_scroll: true });
await expect(dialog).toBeHidden();
// The list header follows the saved default at once; nothing else changed.
await expect(page.locator('#infiniteScrollSession')).toBeChecked();
const after = (await (await request.get('/prompt_manager/settings')).json()).settings;
expect(after).toEqual({ ...before, infinite_scroll: true });
await page.reload();
// The dialog shows the saved default once the page has read it (the header follows it first).
await expect(page.locator('#infiniteScrollSession')).toBeChecked();
await page.locator('#pageOptionsBtn').click();
await expect(page.locator('#infiniteScrollDefault')).toBeChecked();
});
test('Page options keeps the page-only settings gallery', async ({ page }) => {
// Native Appearance keys and the startup prompt's own keys are in the shared object before the page opens.
const stored = { imageQuality: 'high', showImageInfo: false, videoMute: false,
autoLoadMetadata: false, workspaceLayout: 'masonry', thumbnailPromptDismissed: 'sig', thumbnailsGenerated: true,
checkThumbnailsAtStartup: false, defaultViewMode: 'list' };
await page.addInitScript(value => {
if (!sessionStorage.getItem('seeded')) {
sessionStorage.setItem('seeded', '1');
localStorage.setItem('gallerySettings', value);
}
}, JSON.stringify(stored));
const listed = page.waitForResponse(item => item.url().includes('/prompt_manager/images/output?'));
await page.goto(PAGES.gallery);
await listed;
// With no Page options control, a stored default view still opens the page in it.
await expect(page.locator('#listViewBtn')).toHaveClass(/bg-pm-accent/);
await openPageOptions(page);
const dialog = page.locator('#settingsModal');
await expect(dialog.locator('h3')).toHaveText('Page options');
// Exactly the page-only settings (G5); the default view is System › Appearance's Default layout.
expect(await dialog.locator('input, select, textarea').evaluateAll(fields => fields.map(field => field.id))).toEqual([
'checkThumbnailsAtStartup', 'lazyLoadingToggle', 'defaultLimitSelect', 'gridColumnsSelect',
'infiniteScrollToggle', 'debugModeToggle', 'apiTimeoutInput']);
await expect(dialog.getByText('Default View Mode')).toHaveCount(0);
await page.locator('#defaultLimitSelect').selectOption('50');
await page.locator('#debugModeToggle').check();
await page.locator('#saveSettingsBtn').click();
await expect(dialog).toBeHidden();
const read = () => page.evaluate(() => JSON.parse(localStorage.getItem('gallerySettings')));
let saved = await read();
expect(saved).toMatchObject({ ...stored, defaultLimit: 50, debugMode: true });
// Reset puts back only the page options' defaults; the Appearance, startup-prompt and default-view keys stay.
await openPageOptions(page);
page.once('dialog', dialog => dialog.accept());
await page.locator('#resetSettingsBtn').click();
await expect(page.getByText('Page options reset to defaults')).toBeVisible();
await expect(page.locator('#defaultLimitSelect')).toHaveValue('100');
saved = await read();
expect(saved).toMatchObject({ defaultViewMode: 'list', imageQuality: 'high', showImageInfo: false, videoMute: false,
autoLoadMetadata: false, workspaceLayout: 'masonry', thumbnailPromptDismissed: 'sig', thumbnailsGenerated: true,
defaultLimit: 100, debugMode: false, checkThumbnailsAtStartup: true, lazyLoading: true, gridColumns: 8,
infiniteScroll: false, apiTimeout: 30 });
});
test('the legacy auto-tag modal stays open after Manage in System', async ({ page, context, request }) => {
for (const legacy of ['admin', 'gallery']) {
await post(request, '/__test__/tagging/models', { unavailable: ['gguf'] });
await page.goto(PAGES[legacy]);
await openAutoTag(page);
const link = page.locator('#ggufModelStatus').getByRole('link', { name: 'Manage in System', exact: true });
await expect(link).toHaveAttribute('title', 'Opens System › Auto-tagging in a new tab');
// A downloaded model keeps its status text.
await expect(page.locator('#wd14VitModelStatus')).toHaveText('\u2713 Downloaded');
const opened = context.waitForEvent('page');
await link.click();
const system = await opened;
await expect(system).toHaveURL(/#\/system\/tagging$/);
await expect(page.locator('#autoTagModal')).toBeVisible();
await system.close();
// Downloaded in System meanwhile: reopening the modal reads the models again (CUJ 4).
await post(request, '/__test__/tagging/models', { unavailable: [] });
await page.goto(PAGES[legacy]);
await openAutoTag(page);
await expect(page.locator('#ggufModelStatus')).toHaveText('\u2713 Downloaded');
}
});
+1 -13
View File
@@ -17,7 +17,7 @@ test.beforeEach(async({page,browserName})=>{
window.fixtureApp={registerExtension(extension){this.extension=extension;},ui:{settings},graph:{_nodes:[],setDirtyCanvas(){}},canvas:{},graphToPrompt:async()=>({})};
await import('/extensions/promptmanager/prompt_manager.js');
class Node {
constructor(){this.id=1;this.type='PromptManagerPair';this.widgets=[...['text','negative_text','category','tags','search_text'].map(name=>({name,type:'text',value:'',options:{},inputEl:document.createElement('textarea')})),{name:'save_to_library',type:'toggle',value:true,options:{}}];this.inputs=[...['clip','prepend_text','append_text'].map(name=>({name,type:name==='clip'?'CLIP':'STRING',link:null})),...['text','negative_text','category','tags','search_text'].map(name=>({name,type:'STRING',link:null,widget:{name}})),{name:'save_to_library',type:'BOOLEAN',link:null,widget:{name:'save_to_library'}}];this.properties={};this.size=[1100,700];}
constructor(){this.id=1;this.type='PromptManagerPair';this.widgets=['text','negative_text','category','tags','search_text'].map(name=>({name,type:'text',value:'',options:{},inputEl:document.createElement('textarea')}));this.inputs=[...['clip','prepend_text','append_text'].map(name=>({name,type:name==='clip'?'CLIP':'STRING',link:null})),...['text','negative_text','category','tags','search_text'].map(name=>({name,type:'STRING',link:null,widget:{name}}))];this.properties={};this.size=[1100,700];}
findInputByType(type){const index=this.inputs.findIndex(input=>input.type===type&&input.link==null);return index<0?undefined:{index,slot:this.inputs[index]};}
addWidget(type,name,value,callback,options){const widget={type,name,value,callback,options};this.widgets.push(widget);return widget;}
addDOMWidget(name,type,element){const host=document.createElement('div');host.style.cssText='height:700px;width:1100px;max-width:100%';host.className='fixture-dom-widget';host.append(element);document.body.append(host);return {element};}
@@ -34,18 +34,6 @@ test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);
async function selectPair(page){await page.getByRole('button',{name:'Recent',exact:true}).click();await page.locator('.pm-editor-result').click();}
async function draft(page,positive='Draft positive',negative='Draft negative'){await page.getByRole('textbox',{name:'Positive prompt',exact:true}).fill(positive);await page.getByRole('textbox',{name:'Negative prompt',exact:true}).fill(negative);}
test('paired editor Save to library switch keeps the widget order and loads older workflows on (#90)',async({page})=>{
await expect(page.locator('.pm-pair')).toBeVisible();
const toggle=page.getByRole('checkbox',{name:'Save to library',exact:true});
await expect(toggle).toBeChecked();
expect(await page.evaluate(()=>node.widgets.map(widget=>widget.name))).toEqual(['text','negative_text','category','tags','search_text','prepend_text','append_text','save_to_library']);
await toggle.uncheck();
expect(await page.evaluate(()=>node.serialize().widgets_values[7])).toBe(false);
await expect(page.locator('.pm-pair > .pm-editor-status[role=status]').last()).toHaveText('Save to library is off: runs of this node are not added to the library.');
await page.evaluate(()=>node.configure({widgets_values:['P','N','','','','','',''],properties:{}}));
await expect(toggle).toBeChecked();
});
test('paired editor previews both texts before explicitly replacing both',async({page})=>{
await expect(page.locator('.pm-pair')).toBeVisible();
await draft(page);
-8
View File
@@ -75,8 +75,6 @@ class TestModel:
loaded = None
# Models the Tag image dialog workflows mark as not downloaded.
unavailable: set = set()
# Pip names per model whose backend packages "cannot be imported" (#424).
missing_packages: dict = {}
# Scored names the concept comparisons answer instead of the fixed list
# (``/__test__/tagging/suggestions``); None keeps the fixed list.
suggestions: list | None = None
@@ -84,13 +82,10 @@ class TestModel:
models_config: dict = {}
def get_models_status(self) -> dict:
from py.autotag import package_fields
return {
name: {
"downloaded": name not in self.unavailable,
"loaded": self.loaded == name,
**package_fields(name, self.missing_packages.get(name, [])),
}
for name in self.models_config
}
@@ -267,15 +262,12 @@ def create_app(root: Path) -> web.Application:
with target.db.model.get_connection() as conn:
for table in (
"saved_views",
"image_collection_items",
"image_collections",
"image_favorites",
"media_review_decisions",
"media_review_items",
"media_image_tags",
"media_annotation_bytes",
"media_annotation_files",
"image_categories",
"image_prompt_pairs",
"prompt_pair_tags",
"prompt_pairs",
@@ -1,7 +1,6 @@
// Concept-vs-production inventory for the native Tags & categories screen (concept renderTags, app.js:719-729).
// pageInventoryOf runs in the page; dialogs reuse the editor's inventoryOf. The concept is never rewritten:
// mapVocabularyConcept / mapVocabularyDialog apply only the recorded deviations, each named with its ruling.
import {diffInventory} from './library-editor-canonical-comparison.mjs';
export {inventoryOf, diffInventory} from './library-editor-canonical-comparison.mjs';
export function pageInventoryOf(main) {
@@ -62,51 +61,23 @@ export function pageInventoryOf(main) {
};
}
const countOf=(count,noun)=>`${count} ${noun}${count===1?'':'s'}`;
/**
* D143 (#332): a category row counts the images whose own category it is: "N positive prompts · M images" when
* prompts hold it too, "M images" for a value only images hold. Image-only rows follow every prompt-backed row, before
* the closing Manage category row, each drawn as the concept's last category row with its own text (vocabulary-visual
* checks this against the concept page with those rows added at runtime). `images` is `{value: count}`; `only` lists
* the image-only values in their order.
*/
export function withImageCounts(operations,{images={},only=[]}={}) {
const rows=operations.map(row=>images[row.h3.text]&&!only.includes(row.h3.text)
?{...row,p:{...row.p,text:`${row.p.text} · ${countOf(images[row.h3.text],'image')}`}}:row);
const manage=rows.at(-1),last=rows.at(-2);
const added=only.map(value=>({...last,h3:{...last.h3,text:value},p:{...last.p,text:countOf(images[value],'image')}}));
return [...rows.slice(0,-1),...added,manage];
}
// Recorded deviations only (plan "Mock inventory" + decisions D1-D15 + ledger rulings). Everything else must be equal.
// `imageCategories` (#332 D143) is the fixture's own image categories, none by default.
export function mapVocabularyConcept(reference,imageCategories={}) {
const operations=reference.operations.map(row=>({...row,p:{...row.p,
// Ledger ruling: "1 positive prompts" -> singular, consistent with the shared plural helper (#238/#243).
text:row.p.text.replace(/^1 positive prompts$/,'1 positive prompt')}}));
export function mapVocabularyConcept(reference) {
return {...reference,
table:{...reference.table,rows:reference.table.rows.map(row=>({...row,
// Data, not a deviation: the concept tags each sample image with its prompt's tags; the vocabulary fixture
// tags no image, so every Images count is 0 (#249 C2 retired #245 D3).
images:'0'}))},
operations:withImageCounts(operations,imageCategories),
operations:reference.operations.map(row=>({...row,p:{...row.p,
// Ledger ruling: "1 positive prompts" -> singular, consistent with the shared plural helper (#238/#243).
text:row.p.text.replace(/^1 positive prompts$/,'1 positive prompt')}})),
};
}
/** The page comparison: concept rows are the concept's sample tags; production appends fixture-only tags (D4). */
export function compareVocabularyPage(reference,actual,imageCategories={}) {
const expected=mapVocabularyConcept(reference,imageCategories);
const shared=expected.table.rows.length;
return {diff:diffInventory(expected,{...actual,table:{...actual.table,rows:actual.table.rows.slice(0,shared)}}),
extra:actual.table.rows.slice(shared).map(row=>row.tag)};
}
// Concept dialogs are simpleInput (app.js:1138-1143). donors are concept inventories for controls the concept
// shows elsewhere: a `.field select` and a `p.small` (Filters dialog, app.js:1351) and a danger dialog action
// (Archive… in the editor, app.js:976), so added controls are held to concept tokens, never to production values.
// `preview` is the Manage categories sentence once Preview has been pressed (D9; D144, #332, adds its images).
export function mapVocabularyDialog(mode,reference,{filters,editor,categories,preview=null}) {
export function mapVocabularyDialog(mode,reference,{filters,editor,categories}) {
const fields=reference.fields.map(item=>({...item,
// Inventory rows 27-29: the name is `required` (blank Apply is blocked natively).
required:true}));
@@ -127,9 +98,9 @@ export function mapVocabularyDialog(mode,reference,{filters,editor,categories,pr
// D9 / Task 8: the proposed name offers a category datalist (no trailing commas: one value).
...fields.map(item=>({...item,listed:true,commaOptions:false}))],
// D9: the preview sentence is a `p.small`, empty and hidden until Preview.
small:[{...filters.small[0],text:preview??''}],
small:[{...filters.small[0],text:''}],
// D9: the primary is `Preview` until a preview is shown, then `Apply`.
actions:{...reference.actions,buttons:preview===null?buttons.map(item=>item.text==='Apply'?{...item,text:'Preview'}:item):buttons}};
actions:{...reference.actions,buttons:buttons.map(item=>item.text==='Apply'?{...item,text:'Preview'}:item)}};
}
return {...reference,fields,actions:{...reference.actions,buttons}};
}
+1 -88
View File
@@ -1,14 +1,11 @@
import {writeFile} from 'node:fs/promises';
import {test,expect} from './fixtures.mjs';
import {captureCanonicalState} from './history-canonical-comparison.mjs';
import {pageInventoryOf,inventoryOf,diffInventory,mapVocabularyConcept,mapVocabularyDialog,compareVocabularyPage} from './vocabulary-canonical-comparison.mjs';
// Artifact-only captures of Tags & categories states that have no concept counterpart: the Manage categories
// preview, loading and 1,000-category cap notes, tag conflict and restore notes, the Delete tag confirmation (D14),
// empty lists (D12), a failed read with Retry, and Load more (D5). The page and the three concept dialogs are captured
// as concept/production pairs by workspace-vocabulary.spec.mjs. There are deliberately no pixel assertions or
// baselines: the maintainer accepts these captures visually before any baseline is added. The last test (#332 D143,
// D144) compares the category rows with image counts and the Manage category preview against the concept.
// baselines: the maintainer accepts these captures visually before any baseline is added.
test.use({locale:'en-US',timezoneId:'UTC',reducedMotion:'reduce'});
const route='/prompt_manager/workspace#/tags';
const main=page=>page.locator('#workspace-main');
@@ -104,87 +101,3 @@ for(const [width,height] of [[1600,1100],[390,844]])for(const theme of ['dark','
await testInfo.attach(`production-load-more-${width}-${theme}`,{path,contentType:'image/png'});
await expect(rows(page)).toHaveCount(100);
});
// D143 and D144 (#332): with own image categories seeded (`image_categories`: "Landscapes" on two images, "Portraits"
// on one image only), the Categories panel and the Manage category preview against the concept (renderTags, app.js
// 719-729; the Manage dialog is its simpleInput, 1138-1143). The page is compared with D143 mapped, a model checked
// against the concept page with those rows added at runtime; the preview with D9's sentence and D144's images.
const concept='/__test__/canonical/workspace';
const IMAGE_CATEGORIES=Object.freeze({images:{Landscapes:2,Portraits:1},only:['Portraits']});
// D9's sentence; D144 adds the images holding the value ("and 2 images").
const PREVIEW='“Landscapes” is used by 3 positive prompts and 2 images. Apply renames it to “Scenery”.';
const pageOf=(page,production)=>page.locator(production?'#workspace-main':'#main').evaluate(pageInventoryOf);
async function openConceptRoute(page,route,theme) {
await page.goto('about:blank');await page.goto(`${concept}#${route}`);
await page.getByRole('combobox',{name:'Theme',exact:true}).selectOption(theme);
}
/** The concept page as D143 records it: image counts after the prompt counts, image-only rows before Manage category. */
function conceptWithImageCounts(page) {
return page.evaluate(({images,only})=>{
const surface=document.querySelectorAll('#main .two-col > section.surface')[1];
const rows=[...surface.querySelectorAll('.operation')];
const count=value=>`${images[value]} image${images[value]===1?'':'s'}`;
for(const row of rows) {
const value=row.querySelector('h3').textContent;
if(images[value]&&!only.includes(value))row.querySelector('p').textContent+=` · ${count(value)}`;
}
const manage=rows.at(-1),last=rows.at(-2);
for(const value of only) {
const row=last.cloneNode(true);row.querySelector('h3').textContent=value;row.querySelector('p').textContent=count(value);manage.before(row);
}
},IMAGE_CATEGORIES);
}
/** Concept controls the Manage dialog adds (as in workspace-vocabulary.spec.mjs): the Filters field select and p.small. */
async function conceptDonors(page,theme,categories) {
await openConceptRoute(page,'images',theme);
await page.locator('#main').getByRole('button',{name:'Filters',exact:true}).click();await expect(page.locator('#modal')).toBeVisible();
const filters=await page.locator('#modal').evaluate(inventoryOf);
await openConceptRoute(page,'prompt/alpine',theme);
await page.getByRole('button',{name:'Edit metadata',exact:true}).click();await expect(page.locator('#modal')).toBeVisible();
const editor=await page.locator('#modal').evaluate(inventoryOf);
return {filters,editor,categories};
}
for(const [width,height] of [[1600,1100],[390,844]])for(const theme of ['dark','light'])test(`category rows and the Manage category preview match the concept with image categories ${width}x${height} ${theme} (D143, D144)`,async({page,request},testInfo)=>{
await page.setViewportSize({width,height});
const manifest=await seed(request,{image_categories:true});
expect(manifest.image_categories).toEqual(IMAGE_CATEGORIES.images);
await openConceptRoute(page,'tags',theme);await expect(page.locator('#main h1')).toHaveText('Tags & categories');
await page.mouse.move(0,0);
const reference=await pageOf(page,false);
await conceptWithImageCounts(page);
const oracle=await pageOf(page,false);
await captureCanonicalState(page,testInfo,`concept-image-categories-${width}-${theme}`,'#main');
const donors=await conceptDonors(page,theme,['Landscapes','Architecture','Environments','Illustration','Portraits']);
await openConceptRoute(page,'tags',theme);
await page.locator('#main').getByRole('button',{name:'Review',exact:true}).click();await expect(page.locator('#modal')).toBeVisible();
const referenceDialog=await page.locator('#modal').evaluate(inventoryOf);
await open(page,theme,manifest.tags.length);
await expect(categoryOperation(page,'Portraits')).toHaveCount(1);
// The pointer still rests where the concept's Review was pressed: park it so no row is hovered.
await page.mouse.move(0,0);
const actual=await pageOf(page,true);
await captureCanonicalState(page,testInfo,`production-image-categories-${width}-${theme}`,'#workspace-main');
const box=await openDialog(page,categoryOperation(page,'Manage category').getByRole('button',{name:'Review',exact:true}));
await expect(box.getByLabel('Category',{exact:true})).toHaveValue('Landscapes');
await box.getByLabel('Proposed new category name',{exact:true}).fill('Scenery');await button(box,'Preview').click();
await expect(box.locator('p.small')).toHaveText(PREVIEW);
// The pressed primary keeps the pointer and focus; the concept is measured as it opens (focus on ×, no hover),
// which workspace-vocabulary.spec.mjs compares, so this state is restored before measuring the preview.
await page.mouse.move(0,0);await box.locator('.dialog-header button').focus();
const actualDialog=await box.evaluate(inventoryOf);
await captureCanonicalState(page,testInfo,`production-image-categories-preview-${width}-${theme}`);
const path=testInfo.outputPath('vocabulary-image-categories.json');
await writeFile(path,JSON.stringify({reference,oracle,actual,referenceDialog,actualDialog},null,2));
await testInfo.attach('vocabulary-image-categories',{path,contentType:'application/json'});
// The D143 model reproduces the concept page with the rows added at runtime.
expect.soft(diffInventory(mapVocabularyConcept(reference,IMAGE_CATEGORIES).operations,mapVocabularyConcept(oracle).operations),'D143 model').toEqual([]);
const {diff,extra}=compareVocabularyPage(reference,actual,IMAGE_CATEGORIES);
expect.soft(actual.operations.map(row=>[row.h3.text,row.p.text]).slice(0,-1),'category rows').toEqual([['Landscapes','3 positive prompts · 2 images'],
['Architecture','1 positive prompt'],['Environments','1 positive prompt'],['Illustration','1 positive prompt'],['Portraits','1 image']]);
expect.soft(diff,'page differences after the recorded deviations and D143').toEqual([]);
expect.soft(extra,'fixture-only rows').toEqual([manifest.pair_only_tag,manifest.archived_only_tag,manifest.negative_only_tag,manifest.unused_tag]);
expect.soft(diffInventory(mapVocabularyDialog('manage',referenceDialog,{...donors,preview:PREVIEW}),actualDialog),'preview differences after D9 and D144').toEqual([]);
});
+236 -1
View File
@@ -104,6 +104,33 @@ for (const leave of ['beforeunload', 'pagehide']) {
});
}
// #222: the gallery's Clear cache clears the server's thumbnails by its own request and keeps this browser's
// preferences and other tools' storage; nothing of Prompt Manager's is cached in a browser, and the toast says so.
test('the gallery cache clear keeps preferences and other tools caches', async ({ page }) => {
await page.addInitScript(() => {
if (sessionStorage.getItem('seeded')) return;
sessionStorage.setItem('seeded', '1');
localStorage.setItem('pm_workspace_theme', 'light');
localStorage.setItem('other-tool-setting', 'keep');
localStorage.setItem('pm_other_tool', 'keep');
});
await page.goto('/prompt_manager/gallery');
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2);
await page.evaluate(async () => { await (await caches.open('other-tool-gallery-thumbnails')).put('/x', new Response('x')); });
const clears = [];
page.on('request', req => { if (req.url().includes('/clear-thumbnails')) clears.push(req.method()); });
page.once('dialog', dialog => dialog.accept());
await page.locator('#settingsBtn').click();
await page.locator('#clearCacheBtn').click();
await expect(page.getByText("Thumbnails cleared; original images untouched. Nothing of Prompt Manager's is cached in this browser."))
.toBeVisible();
expect(clears).toEqual(['POST']);
expect(await page.evaluate(async () => ({
theme: localStorage.getItem('pm_workspace_theme'), other: localStorage.getItem('other-tool-setting'),
prefixed: localStorage.getItem('pm_other_tool'), notice: localStorage.getItem('pm_update_notice_dismissed'),
caches: await caches.keys(),
}))).toEqual({ theme: 'light', other: 'keep', prefixed: 'keep', notice: '3.2.0', caches: ['other-tool-gallery-thumbnails'] });
});
// #379: the reviewed tags are that image's own tags; no prompt changes.
test('actual gallery review generates and persists tags for the selected root\'s image', async ({ page, request }) => {
@@ -199,7 +226,7 @@ test('gallery infinite scroll appends a real backend page while retaining existi
await expect(await request.post('/__test__/reset', { data: { extra_images: 108 } })).toBeOK();
await page.goto('/prompt_manager/gallery');
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(100);
await page.locator('#pageOptionsBtn').click();
await page.locator('#settingsBtn').click();
await page.locator('#infiniteScrollToggle').check();
await page.locator('#saveSettingsBtn').click();
await expect(page.locator('#settingsModal')).toBeHidden();
@@ -219,6 +246,37 @@ async function promptSnapshot(request) {
})).sort((a, b) => a.id - b.id);
}
async function downloadBackup(page) {
const downloading = page.waitForEvent('download');
await page.locator('#backupBtn').click();
const download = await downloading;
expect(download.suggestedFilename()).toMatch(/^prompts_backup_.*\.db$/);
expect(await download.failure()).toBeNull();
const chunks = [];
for await (const chunk of await download.createReadStream()) chunks.push(chunk);
const buffer = Buffer.concat(chunks);
expect(buffer.subarray(0, 16).toString()).toBe('SQLite format 3\0');
return { name: download.suggestedFilename(), mimeType: 'application/octet-stream', buffer };
}
async function chooseRestore(page, file) {
await page.locator('#restoreBtn').click();
await expect(page.locator('#restoreModal')).toBeVisible();
await page.locator('#restoreFileInput').setInputFiles(file);
}
// Confirms the chosen restore and waits for the server's answer, not for a timer: a restore is disk-bound and can take
// longer than an expect timeout on a slow runner (#359). The modal closes only after that answer.
async function confirmRestoreAndWait(page) {
const restored = page.waitForResponse(response => response.url().endsWith('/prompt_manager/restore'));
await page.locator('#confirmRestore').click();
const response = await restored;
const body = await response.text();
expect(response.status(), body).toBe(200);
expect(JSON.parse(body)).toMatchObject({ success: true, prompt_count: 2, backup_created: expect.any(String) });
await expect(page.locator('#restoreModal')).toBeHidden();
}
function confirmNextDialog(page, accept, message) {
page.once('dialog', async dialog => {
expect(dialog.type()).toBe('confirm');
@@ -234,6 +292,88 @@ async function deletePromptButton(page, text) {
return page.locator(`[data-action="delete"][data-prompt-id="${id}"]`);
}
test('downloaded backup restores deleted prompts, tags, and image links through the actual upload form', async ({ page, request }) => {
await page.goto('/prompt_manager/admin');
await expect(page.locator('.prompt-text')).toHaveCount(2);
const before = await promptSnapshot(request);
expect(before.map(prompt => prompt.tags)).toEqual([['cats'], ['dogs']]);
expect(before.every(prompt => prompt.images.length === 1)).toBe(true);
const backup = await downloadBackup(page);
const button = await deletePromptButton(page, 'Cat portrait');
confirmNextDialog(page, true, 'delete this prompt');
await button.click();
await expect(page.locator('.prompt-text')).toHaveCount(1);
await expect(await request.post('/prompt_manager/save', { data: { text: 'Added after backup', tags: ['temporary'] } })).toBeOK();
expect(await promptSnapshot(request)).not.toEqual(before);
await chooseRestore(page, backup);
confirmNextDialog(page, true, backup.name);
await confirmRestoreAndWait(page);
await expect(page.locator('.prompt-text')).toHaveCount(2);
await page.reload();
await expect(page.locator('.prompt-text').filter({ hasText: 'Cat portrait' })).toBeVisible();
await expect(page.locator('.prompt-text').filter({ hasText: 'Added after backup' })).toHaveCount(0);
expect(await promptSnapshot(request)).toEqual(before);
await page.goto('/prompt_manager/admin#/tags');
await page.locator('[data-tag-name="cats"]').click();
await expect(page.locator('#tagPromptsGrid')).toContainText('Cat portrait');
await page.locator('#tagPromptsGrid img').first().click();
await expect(page.locator('.viewer-canvas img')).toBeVisible();
await expect.poll(() => page.locator('.viewer-canvas img').evaluate(image => image.complete && image.naturalWidth > 0)).toBe(true);
});
test('restore validation and cancellation preserve live data and allow a valid retry', async ({ page, request }) => {
await page.goto('/prompt_manager/admin');
const before = await promptSnapshot(request);
const backup = await downloadBackup(page);
let uploads = 0;
page.on('request', request => { if (request.url().endsWith('/prompt_manager/restore')) uploads++; });
await page.locator('#restoreBtn').click();
await page.locator('#confirmRestore').click();
await expect(page.getByText('Please select a database file to restore', { exact: true })).toBeVisible();
await page.locator('#restoreFileInput').setInputFiles({ ...backup, name: 'backup.txt' });
await page.locator('#confirmRestore').click();
await expect(page.getByText('Please select a valid .db file', { exact: true })).toBeVisible();
expect(uploads).toBe(0);
await page.locator('#restoreFileInput').setInputFiles(backup);
confirmNextDialog(page, false, 'replace your current database');
await page.locator('#confirmRestore').click();
await expect(page.locator('#restoreModal')).toBeVisible();
expect(uploads).toBe(0);
await page.locator('#restoreFileInput').setInputFiles({ name: 'corrupt.db', mimeType: 'application/octet-stream', buffer: Buffer.from('not a SQLite database') });
confirmNextDialog(page, true, 'corrupt.db');
const rejected = page.waitForResponse(response => response.url().endsWith('/prompt_manager/restore'));
await page.locator('#confirmRestore').click();
expect((await rejected).status()).toBe(400);
await expect(page.getByText(/Restore failed:/)).toBeVisible();
await expect(page.locator('#restoreModal')).toBeVisible();
await expect(page.locator('#confirmRestore')).toBeEnabled();
expect(await promptSnapshot(request)).toEqual(before);
await page.locator('#restoreFileInput').setInputFiles(backup);
confirmNextDialog(page, true, backup.name);
await confirmRestoreAndWait(page);
expect(await promptSnapshot(request)).toEqual(before);
});
test('backup and restore recover from interrupted network requests without losing data', async ({ page, request }) => {
await page.goto('/prompt_manager/admin');
const before = await promptSnapshot(request);
await page.route('**/prompt_manager/backup', route => route.abort('failed'), { times: 1 });
await page.locator('#backupBtn').click();
await expect(page.getByText(/Backup failed:/)).toBeVisible();
const backup = await downloadBackup(page);
await chooseRestore(page, backup);
await page.route('**/prompt_manager/restore', route => route.abort('failed'), { times: 1 });
confirmNextDialog(page, true, backup.name);
await page.locator('#confirmRestore').click();
await expect(page.getByText(/Restore failed:/)).toBeVisible();
await expect(page.locator('#confirmRestore')).toBeEnabled();
await expect(page.locator('#restoreModal')).toBeVisible();
expect(await promptSnapshot(request)).toEqual(before);
confirmNextDialog(page, true, backup.name);
await confirmRestoreAndWait(page);
expect(await promptSnapshot(request)).toEqual(before);
});
test('prompt deletion requires confirmation and can retry after a network failure', async ({ page, request }) => {
await page.goto('/prompt_manager/admin');
const before = await promptSnapshot(request);
@@ -261,3 +401,98 @@ test('prompt deletion requires confirmation and can retry after a network failur
await expect(await request.get(url)).toBeOK();
}
});
// The duplicate scan reads the output root only, but the fixture's input root holds a different
// shared/same.png. A bare /images/serve/<path> URL tries every gallery root in turn, so once the
// output copy is deleted it answers 200 with the input copy: check each file in its own root.
const serveIn = (root, path) => `/prompt_manager/images/serve/${path.split(/[\\/]/).map(encodeURIComponent).join('/')}?root=${root}`;
const rootOf = url => new URL(url, 'http://gallery.test').searchParams.get('root');
// Which copy the scan keeps depends on file times: the fixture stamps shared/same.png with the
// wall clock, and the kernel can stamp its bulk copies a fraction of a millisecond earlier.
// listOrder rewrites the finished scan's groups so a test can pick the order it needs.
async function openDuplicateScan(page, request, listOrder = null) {
await expect(await request.post('/__test__/reset', { data: { extra_images: 2 } })).toBeOK();
if (listOrder) {
await page.route('**/prompt_manager/scan_duplicates/stream', async route => {
const response = await route.fetch();
const body = (await response.text()).replace(/^data: (.*)$/gm, (line, json) => {
const event = JSON.parse(json);
if (event.type !== 'complete') return line;
for (const group of event.duplicates) group.images = listOrder(group.images);
return `data: ${JSON.stringify(event)}`;
});
await route.fulfill({ response, body });
});
}
await page.goto('/prompt_manager/gallery');
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(4);
const originals = await page.locator('#galleryGrid img[data-original]').evaluateAll(images => images.map(image => image.dataset.original));
const output = rootOf(originals.find(url => url.includes('/bulk/')));
const input = rootOf(originals.find(url => url.includes('/shared/same.png') && rootOf(url) !== output));
expect(output).not.toBeNull();
expect(input).not.toBeNull();
await page.locator('#settingsBtn').click();
await page.locator('#scanDuplicatesBtn').click();
await expect(page.locator('#duplicatesCount')).toHaveText('1');
const candidates = page.locator('.duplicate-checkbox');
await expect(candidates).toHaveCount(2);
return { originals, output, input, candidates };
}
async function expectOnlySelectedGone(page, request, { originals, output, input }, selected, retained) {
expect((await request.get(serveIn(output, selected))).status()).toBe(404);
await expect(await request.get(serveIn(output, retained))).toBeOK();
await expect(await request.get(serveIn(input, 'shared/same.png'))).toBeOK();
await page.reload();
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(3);
const remaining = await page.locator('#galleryGrid img[data-original]').evaluateAll(images => images.map(image => image.dataset.original));
expect(remaining).toHaveLength(originals.length - 1);
for (const url of remaining) {
expect(originals).toContain(url);
await expect(await request.get(url)).toBeOK();
}
}
test('duplicate cleanup cancels safely and deletes only the selected file after a retry', async ({ page, request }) => {
const scan = await openDuplicateScan(page, request);
const { output, candidates } = scan;
const selected = await candidates.first().getAttribute('data-image-path');
const retained = await candidates.last().getAttribute('data-image-path');
await candidates.first().check();
let deletes = 0;
page.on('request', request => { if (request.url().endsWith('/delete_duplicate_images')) deletes++; });
confirmNextDialog(page, false, 'permanently delete 1 duplicate image files');
await page.locator('#removeDuplicatesBtn').click();
expect(deletes).toBe(0);
await expect(await request.get(serveIn(output, selected))).toBeOK();
await page.route('**/prompt_manager/delete_duplicate_images', route => route.abort('failed'), { times: 1 });
confirmNextDialog(page, true, 'permanently delete 1 duplicate image files');
await page.locator('#removeDuplicatesBtn').click();
await expect(page.getByText('Failed to delete duplicate images', { exact: true })).toBeVisible();
await expect(page.locator('#removeDuplicatesBtn')).toBeEnabled();
await expect(await request.get(serveIn(output, selected))).toBeOK();
confirmNextDialog(page, true, 'permanently delete 1 duplicate image files');
const deleted = page.waitForResponse(response => response.url().endsWith('/delete_duplicate_images'));
await page.locator('#removeDuplicatesBtn').click();
expect((await (await deleted).json())).toMatchObject({ success: true, deleted_count: 1, failed_count: 0 });
await expect(page.locator('#duplicatesModal')).toBeHidden();
await expectOnlySelectedGone(page, request, scan, selected, retained);
});
test('deleting a duplicate whose path another gallery root also holds removes only the scanned copy', async ({ page, request }) => {
// The order CI saw: a bulk copy stamped first, so shared/same.png is the first Delete candidate.
const ciOrder = ['bulk/0000.png', 'shared/same.png', 'bulk/0001.png'];
const rank = image => ciOrder.indexOf(image.path.replace(/\\/g, '/'));
const scan = await openDuplicateScan(page, request, images => [...images].sort((a, b) => rank(a) - rank(b)));
const selected = await scan.candidates.first().getAttribute('data-image-path');
const retained = await scan.candidates.last().getAttribute('data-image-path');
expect(selected.replace(/\\/g, '/')).toBe('shared/same.png');
await scan.candidates.first().check();
confirmNextDialog(page, true, 'permanently delete 1 duplicate image files');
const deleted = page.waitForResponse(response => response.url().endsWith('/delete_duplicate_images'));
await page.locator('#removeDuplicatesBtn').click();
expect((await (await deleted).json())).toMatchObject({ success: true, deleted_count: 1, failed_count: 0 });
await expect(page.locator('#duplicatesModal')).toBeHidden();
await expectOnlySelectedGone(page, request, scan, selected, retained);
});
-24
View File
@@ -495,30 +495,6 @@ test('the inline current file names its prompts apart from the frozen run fields
expect(await exact(dialog(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'))).toBe(before);
});
// #332 CUJ 5 (MC8, §5.3): History's current-file inspector reads the file's own category with the Images key rule, so
// the inline current file and its viewer show "Portraits · this image"; current.png has two saved links, so it inherits
// nothing and shows no chip before.
test('the current file shows its own category with its origin in the recorded run and its viewer',async({page,request})=>{
const chip=scope=>scope.locator('span.chip:has(> span.chip-source)');
const current=dialog(page).getByRole('region',{name:'Current reported file',exact:true});
const viewCurrent=async()=>{await dialog(page).locator('[data-receipt]').first().getByRole('button',{name:'View current file',exact:true}).click();
const details=page.locator('dialog#viewer-dialog #viewer-details');await expect(details.locator('.panel-heading')).toBeVisible();return details;};
await open(page,page.fixture.detail_id);await marker(page,page.fixture.runs.shared).click();await expect(dialog(page).locator('[data-receipt]')).toHaveCount(5);
let details=await viewCurrent();await expect(details.locator('.panel-heading h2')).toBeVisible();await expect(chip(details)).toHaveCount(0);
await page.keyboard.press('Escape');await expect(current).toBeVisible();await expect(chip(current)).toHaveCount(0);
await dialog(page).getByRole('button',{name:'Close recorded run',exact:true}).click();
const {images}=await(await request.get('/prompt_manager/images/output?mode=workspace&subfolder=history&limit=40')).json();
const file=images.find(image=>image.root_index===0&&image.path==='history/current.png');
const put=await request.put('/prompt_manager/images/category',{data:{items:[{root_index:0,root_scope:file.root_scope,path:file.path}],category:'Portraits'}});
await expect(put).toBeOK();expect((await put.json()).updated).toBe(1);
await open(page,page.fixture.detail_id);await marker(page,page.fixture.runs.shared).click();await expect(dialog(page).locator('[data-receipt]')).toHaveCount(5);
details=await viewCurrent();
await expect(chip(details)).toHaveText('Image category: Portraits · this image');
await page.keyboard.press('Escape');await expect(current).toBeVisible();
await expect(chip(current)).toHaveText('Image category: Portraits · this image');
expect(await chip(current).locator('.chip-source').evaluate(node=>node.textContent)).toBe(' · this image');
});
// LI-4 (D117): the recorded history's current-file viewer is the shared full inspector, so it copies the legacy summary.
test('the current file viewer copies its settings summary',async({page})=>{
await page.addInitScript(()=>{window.__copied=[];Object.defineProperty(navigator,'clipboard',{configurable:true,value:{writeText:async text=>{window.__copied.push(text);}}});});
+74 -286
View File
@@ -1988,286 +1988,70 @@ test('Export metadata downloads nothing when no image can be read', async ({ pag
await expect(bulkbar(page).locator('.bulkbar > strong')).toHaveText('2 selected');
});
// #332 (D140, MC6, MC7): Set category writes each selected image's own category (PUT /prompt_manager/images/category);
// "Use prompt category" removes it after a confirmation (DELETE). It never writes a prompt: Images no longer posts
// /prompt_manager/bulk/category (#259's prompt action is retired from Images).
// #249 D8 follow-up (image selection, Task 3): Set category sets the linked positives' category (app.js simpleInput).
const categoryButton = page => bulkbar(page).getByRole('button', { name: 'Set category', exact: true });
const categoryDialog = page => page.getByRole('dialog', { name: 'Categorize selected images', exact: true });
const removeDialog = page => page.getByRole('dialog', { name: 'Use prompt category', exact: true });
const categoryFilter = page => page.getByRole('combobox', { name: 'Filter by category', exact: true });
const categoryRoute = '**/prompt_manager/images/category';
const IMAGE_CATEGORY = '/prompt_manager/images/category';
const cardOf = (page, key) => page.locator(`article[data-image-key="${key}"]`);
const keyOf = image => `${image.root_index}:${image.path}`;
const cardKeys = page => cards(page).evaluateAll(nodes => nodes.map(node => node.dataset.imageKey).sort());
// The four outputs of "Cat portrait" (CUJ 1's batch); the input root's same.png is linked to "Dog landscape".
const BATCH = ['0:bulk/0000.png', '0:bulk/0001.png', '0:bulk/0002.png', '0:shared/same.png'];
/** Every category write the page sends, Images' own route and #259's prompt route alike. */
function categoryWrites(page) {
const writes = [];
page.on('request', item => {
const path = new URL(item.url()).pathname;
if (item.method() === 'GET' || !['/prompt_manager/images/category', '/prompt_manager/bulk/category'].includes(path)) return;
writes.push({ method: item.method(), path, body: item.postDataJSON() });
});
return writes;
}
async function listedImages(request, query = '') {
const response = await request.get(`/prompt_manager/images/output?mode=workspace&limit=40${query}`);
await expect(response).toBeOK();
return (await response.json()).images;
}
async function targetsOf(request, keys) {
const images = await listedImages(request);
return keys.map(key => {
const image = images.find(item => keyOf(item) === key);
return { root_index: image.root_index, root_scope: image.root_scope, path: image.path };
});
}
async function putOwnCategory(request, keys, category) {
const response = await request.put(IMAGE_CATEGORY, { data: { items: await targetsOf(request, keys), category } });
await expect(response).toBeOK();
}
const filteredKeys = async (request, category) => (await listedImages(request, `&category=${encodeURIComponent(category)}`)).map(keyOf).sort();
async function positiveOf(request, id) {
const response = await request.get(`/prompt_manager/library/${id}`);
await expect(response).toBeOK();
const { positive } = await response.json();
return { id: positive.id, text: positive.text, category: positive.category };
}
/** CUJ 1's Given: four images linked to the positive "Cat portrait", whose category is "Landscapes". */
async function seedBatch(request) {
await expect(await request.post('/__test__/reset', { data: { extra_images: 3 } })).toBeOK();
const response = await request.post('/prompt_manager/save', { data: { text: 'Cat portrait', category: 'Landscapes' } });
await expect(response).toBeOK();
const id = (await response.json()).prompt_id;
expect(await filteredKeys(request, 'Landscapes')).toEqual(BATCH);
return id;
}
async function openSetCategory(page) {
await categoryButton(page).click();
await expect(categoryDialog(page)).toBeVisible();
return categoryDialog(page);
}
async function applyCategory(page, category) {
const dialog = await openSetCategory(page);
await dialog.getByLabel('Category', { exact: true }).fill(category);
await dialog.getByRole('button', { name: 'Apply', exact: true }).click();
await expect(dialog).toBeHidden();
const categoryRoute = '**/prompt_manager/bulk/category';
/** Unlink the input root's card by deleting its prompt; returns the output card's positive id. */
async function unlinkInputCard(request) {
const { images } = await (await request.get('/prompt_manager/images/output?mode=workspace')).json();
const byKey = Object.fromEntries(images.map(image => [`${image.root_index}:${image.path}`, image]));
const [output, input] = [byKey['0:shared/same.png'], byKey['1:shared/same.png']];
expect([output.linked.status, input.linked.status]).toEqual(['linked', 'linked']);
await expect(await request.delete(`/prompt_manager/delete/${input.linked.positive_id}`)).toBeOK();
return output.linked.positive_id;
}
test('Set category overrides one image of a batch and never writes its prompt (#332 CUJ 1)', async ({ page, request }) => {
const positiveId = await seedBatch(request);
const before = await positiveOf(request, positiveId);
const writes = categoryWrites(page);
test('Set category applies the category to the linked positives and the category filter then matches', async ({ page, request }) => {
const outputPositive = await unlinkInputCard(request);
await page.goto(workspace);
await expect(cards(page)).toHaveCount(5);
await checkbox(cardOf(page, '0:bulk/0001.png'), '0001.png').check();
const dialog = await openSetCategory(page);
// The concept's simpleInput plus "Use prompt category" between Cancel and Apply (primary).
await checkBoth(page);
const opener = categoryButton(page);
await opener.click();
const dialog = categoryDialog(page);
await expect(dialog).toBeVisible();
// The concept's simpleInput: one field and Cancel / Apply (primary).
await expect(dialog.locator('#modal-body > .field > label[for="action-input"]')).toHaveText('Category');
await expect(dialog.locator('#modal-body > .field > input#action-input')).toHaveValue('');
await expect(dialog.locator('#modal-body > .dialog-actions > button')).toHaveText(['Cancel', 'Use prompt category', 'Apply']);
await expect(dialog.locator('#modal-body > .dialog-actions > button')).toHaveText(['Cancel', 'Apply']);
await expect(dialog.getByRole('button', { name: 'Apply', exact: true })).toHaveClass(/ca-primary/);
await expect(dialog.getByLabel('Category', { exact: true })).toBeFocused();
await expect(dialog.getByLabel('Category', { exact: true })).toHaveAttribute('maxlength', '100');
// An empty value applies nothing and keeps the dialog open.
const posts = [];
page.on('request', item => { if (item.method() === 'POST' && item.url().endsWith('/prompt_manager/bulk/category')) posts.push(item.postDataJSON()); });
await dialog.getByRole('button', { name: 'Apply', exact: true }).click();
await expect(dialog).toBeVisible();
await dialog.getByLabel('Category', { exact: true }).fill(' Portraits ');
await dialog.getByLabel('Category', { exact: true }).fill(' Landscapes ');
const listings = [];
page.on('request', item => { if (imagePages(new URL(item.url()))) listings.push(item.url()); });
await dialog.getByRole('button', { name: 'Apply', exact: true }).click();
await expect(dialog).toBeHidden();
await expect(toast(page)).toHaveText('Category “Portraits” set on 1 image.');
expect(writes).toEqual([{ method: 'PUT', path: IMAGE_CATEGORY,
body: { items: await targetsOf(request, ['0:bulk/0001.png']), category: 'Portraits' } }]);
await expect(toast(page)).toHaveText('Category set on 1 prompt. 1 selected image had no linked prompt and was skipped.');
expect(posts).toEqual([{ prompt_ids: [outputPositive], category: 'Landscapes' }]);
// The listing and the category options reload; the reload clears the selection.
await expect.poll(() => listings.length).toBeGreaterThan(0);
await expect(bulkbar(page)).toBeEmpty();
await expect(categoryFilter(page).locator('option')).toHaveText(['All categories', 'Landscapes', 'Portraits']);
await categoryFilter(page).selectOption('Portraits');
await expect(cards(page)).toHaveCount(1);
await expect(cards(page)).toHaveAttribute('data-image-key', '0:bulk/0001.png');
await categoryFilter(page).selectOption('Landscapes');
await expect(cards(page)).toHaveCount(3);
expect(await cardKeys(page)).toEqual(BATCH.filter(key => key !== '0:bulk/0001.png'));
// The prompt keeps its category: only the image carries "Portraits".
expect(await positiveOf(request, positiveId)).toEqual(before);
expect(before.category).toBe('Landscapes');
expect(writes.filter(write => write.path === '/prompt_manager/bulk/category')).toEqual([]);
});
test('Set category categorizes unlinked images and lists the value in the toolbar and the suggestions (#332 CUJ 2)', async ({ page, request }) => {
// Unlink both images by deleting their prompts.
for (const image of await listedImages(request)) {
await expect(await request.delete(`/prompt_manager/delete/${image.linked.positive_id}`)).toBeOK();
}
expect((await listedImages(request)).filter(image => image.linked.status === 'linked')).toEqual([]);
const writes = categoryWrites(page);
await page.goto(workspace);
await checkBoth(page);
await applyCategory(page, 'Scans');
await expect(toast(page)).toHaveText('Category “Scans” set on 2 images.');
expect(writes).toEqual([{ method: 'PUT', path: IMAGE_CATEGORY,
body: { items: await targetsOf(request, ['0:shared/same.png', '1:shared/same.png']), category: 'Scans' } }]);
await expect(categoryFilter(page).locator('option')).toHaveText(['All categories', 'Scans']);
await categoryFilter(page).selectOption('Scans');
await expect(cards(page)).toHaveCount(2);
expect(await cardKeys(page)).toEqual(['0:shared/same.png', '1:shared/same.png']);
// The field suggests the toolbar's categories through its own datalist.
await checkbox(cards(page).first(), 'same.png').check();
const dialog = await openSetCategory(page);
await expect(dialog.getByLabel('Category', { exact: true })).toHaveAttribute('list', 'workspace-input-suggestions');
expect(await dialog.locator('datalist#workspace-input-suggestions option').evaluateAll(nodes => nodes.map(node => node.value))).toEqual(['Scans']);
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
await expect(page.locator('dialog[open]')).toHaveCount(0);
expect(writes).toHaveLength(1);
});
test('Use prompt category removes the own category only after Remove in its confirmation (#332 CUJ 3)', async ({ page, request }) => {
const positiveId = await seedBatch(request);
await putOwnCategory(request, ['0:bulk/0001.png'], 'Portraits');
const writes = categoryWrites(page);
await page.goto(workspace);
await expect(cards(page)).toHaveCount(5);
await expect(categoryFilter(page).locator('option')).toHaveText(['All categories', 'Landscapes', 'Portraits']);
await checkbox(cardOf(page, '0:bulk/0001.png'), '0001.png').check();
// Cancel in the confirmation sends nothing and keeps the selection.
await (await openSetCategory(page)).getByRole('button', { name: 'Use prompt category', exact: true }).click();
await expect(categoryDialog(page)).toHaveCount(0);
const confirm = removeDialog(page);
await expect(confirm).toBeVisible();
await expect(confirm.locator('p').first()).toHaveText("Remove the image category from the 1 selected image? Each one then shows its prompt's category, or none if it has no linked prompt. This cannot be undone.");
await expect(confirm.locator('.dialog-actions > button')).toHaveText(['Cancel', 'Remove']);
await expect(confirm.getByRole('button', { name: 'Remove', exact: true })).toHaveClass(/danger/);
await expect(confirm.getByRole('button', { name: 'Cancel', exact: true })).toBeFocused();
await confirm.getByRole('button', { name: 'Cancel', exact: true }).click();
await expect(page.locator('dialog[open]')).toHaveCount(0);
await expect(selectedCount(page)).toHaveText('1 selected');
expect(writes).toEqual([]);
expect(await filteredKeys(request, 'Portraits')).toEqual(['0:bulk/0001.png']);
// Remove sends exactly one DELETE naming the selected image.
await (await openSetCategory(page)).getByRole('button', { name: 'Use prompt category', exact: true }).click();
await removeDialog(page).getByRole('button', { name: 'Remove', exact: true }).click();
await expect(toast(page)).toHaveText('Removed the image category from 1 image.');
expect(writes).toEqual([{ method: 'DELETE', path: IMAGE_CATEGORY, body: { items: await targetsOf(request, ['0:bulk/0001.png']) } }]);
await expect(bulkbar(page)).toBeEmpty();
// The image shows its prompt's category again: "Landscapes" lists all four, and "Portraits" is gone.
await expect(categoryFilter(page).locator('option')).toHaveText(['All categories', 'Landscapes']);
await categoryFilter(page).selectOption('Landscapes');
await expect(cards(page)).toHaveCount(4);
expect(await cardKeys(page)).toEqual(BATCH);
expect((await positiveOf(request, positiveId)).category).toBe('Landscapes');
});
test('a saved view filtered by a category lists an image categorized after it was saved (#332 CUJ 8)', async ({ page, request }) => {
await seedBatch(request);
await putOwnCategory(request, ['0:bulk/0001.png'], 'Portraits');
await page.goto(workspace);
await categoryFilter(page).selectOption('Portraits');
await expect(cards(page)).toHaveCount(1);
await page.locator('#workspace-main search.toolbar').getByRole('button', { name: 'Save view', exact: true }).click();
const save = page.getByRole('dialog', { name: 'Save this view', exact: true });
await save.getByRole('textbox', { name: 'View name', exact: true }).fill('Portraits');
await save.getByRole('button', { name: 'Save', exact: true }).click();
await expect(toast(page)).toHaveText('Saved view added to the sidebar.');
// Another image gets "Portraits" through Set category.
await categoryFilter(page).selectOption('');
await expect(cards(page)).toHaveCount(5);
await checkbox(cardOf(page, '0:bulk/0002.png'), '0002.png').check();
await applyCategory(page, 'Portraits');
await expect(toast(page)).toHaveText('Category “Portraits” set on 1 image.');
// Reopening the view lists both.
await page.getByRole('navigation', { name: 'Saved views', exact: true }).getByRole('link', { name: 'Portraits', exact: true }).click();
await expect(page.locator('#workspace-main h1')).toHaveText('Portraits');
await expect(categoryFilter(page)).toHaveValue('Portraits');
await expect(cards(page)).toHaveCount(2);
expect(await cardKeys(page)).toEqual(['0:bulk/0001.png', '0:bulk/0002.png']);
});
test('at 390px in light theme the Set category dialog and the category chip fit (#332 CUJ 10)', async ({ page, request }) => {
const LONG = 'Portraits taken on a phone at the end of a long walk through the old harbour at dusk';
await page.addInitScript(() => { try { localStorage.setItem('pm_workspace_theme', 'light'); } catch { /* the default theme */ } });
await page.setViewportSize({ width: 390, height: 844 });
await seedBatch(request);
await putOwnCategory(request, ['0:bulk/0001.png'], LONG);
await page.goto(workspace);
await expect(cards(page)).toHaveCount(5);
await expect(page.locator('html')).toHaveCSS('color-scheme', 'light');
const card = cardOf(page, '0:bulk/0001.png');
// The chip wraps inside the side panel; its origin never splits.
await card.getByRole('button', { name: 'Inspect 0001.png', exact: true }).click();
const chip = details(page).locator('span.chip', { has: page.locator('span.chip-source') });
await expect(chip).toHaveText(`Image category: ${LONG} · this image`);
const fit = await chip.evaluate(node => {
const panel = node.closest('aside, [role="complementary"]').getBoundingClientRect(), box = node.getBoundingClientRect();
const source = node.querySelector('.chip-source');
const line = parseFloat(getComputedStyle(source).lineHeight) || source.getBoundingClientRect().height;
return { inside: box.left >= panel.left - 0.5 && box.right <= panel.right + 0.5, sourceLines: source.getClientRects().length,
wrapped: box.height > line * 1.5, pageScroll: document.documentElement.scrollWidth - window.innerWidth };
});
expect(fit).toEqual({ inside: true, sourceLines: 1, wrapped: true, pageScroll: 0 });
// The dialog fits the width with no horizontal scroll, and its three buttons wrap without overlap.
await checkbox(card, '0001.png').check();
const dialog = await openSetCategory(page);
const geometry = await dialog.evaluate(node => {
const box = node.getBoundingClientRect();
const buttons = [...node.querySelectorAll('.dialog-actions > button')].map(button => button.getBoundingClientRect())
.map(({ left, right, top, bottom }) => ({ left, right, top, bottom }));
const overlaps = buttons.some((a, i) => buttons.some((b, j) => j > i && a.left < b.right && b.left < a.right && a.top < b.bottom && b.top < a.bottom));
const scrollers = [node, ...node.querySelectorAll('*')].filter(item => item.scrollWidth > item.clientWidth + 1 && getComputedStyle(item).overflowX !== 'visible');
return { fits: box.left >= 0 && box.right <= window.innerWidth, scrollers: scrollers.length, count: buttons.length, overlaps,
inside: buttons.every(b => b.left >= box.left && b.right <= box.right), pageScroll: document.documentElement.scrollWidth - window.innerWidth };
});
expect(geometry).toEqual({ fits: true, scrollers: 0, count: 3, overlaps: false, inside: true, pageScroll: 0 });
});
test('Set category of more than 500 images says so before any dialog or request (#332)', async ({ page, request }) => {
await page.route(url => url.pathname === '/prompt_manager/images/output' && url.searchParams.get('limit') !== '1', async route => {
const response = await route.fetch();
const body = await response.json();
await route.fulfill({ response, json: { ...body, total: 501 } });
});
await bulkReset(page, request);
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
await page.keyboard.press('ControlOrMeta+a');
await bulkbar(page).getByRole('button', { name: 'Select all 501 matching', exact: true }).click();
await expect(selectedCount(page)).toHaveText('All 501 matching selected');
const writes = categoryWrites(page);
await categoryButton(page).click();
await expect(toast(page)).toHaveText('Select at most 500 images.');
await expect(page.locator('dialog[open]')).toHaveCount(0);
expect(writes).toEqual([]);
await expect(selectedCount(page)).toHaveText('All 501 matching selected');
});
test('a changed source says so and reloads the listing (#332)', async ({ page }) => {
await page.goto(workspace);
await checkBoth(page);
await page.route(categoryRoute, route => route.fulfill({ status: 409,
json: { success: false, code: 'source-changed', error: 'The output sources changed. Refresh the images and try again.' } }));
const reloaded = page.waitForRequest(item => imagePages(new URL(item.url())));
await applyCategory(page, 'Landscapes');
await expect(toast(page)).toHaveText('The sources changed. Refresh the images and try again.');
await reloaded;
await expect(cards(page)).toHaveCount(2);
// The reload hides the bar with Set category in it: focus moves to the page heading, as after a success.
await expect(page.locator('#workspace-main h1')).toBeFocused();
await expect(cards(page).getByRole('checkbox', { checked: true })).toHaveCount(0);
const filter = page.getByRole('combobox', { name: 'Filter by category', exact: true });
await expect(filter.locator('option')).toContainText(['Landscapes']);
await filter.selectOption('Landscapes');
await expect(cards(page)).toHaveCount(1);
await expect(cards(page)).toHaveAttribute('data-image-key', '0:shared/same.png');
});
test('a recipe-linked image is listed under its recipe\'s category (#332 D142)', async ({ page, request }) => {
await expect(await request.post('/__test__/reset', { data: { native_library: true } })).toBeOK();
test('Set category without a linked image posts nothing and opens no dialog', async ({ page, request }) => {
await unlinkInputCard(request);
await page.goto(workspace);
await expect(cards(page)).toHaveCount(3);
await expect(categoryFilter(page).locator('option')).toHaveText(['All categories', 'Animals', 'Experiments', 'Portraits']);
// The output same.png is linked to recipe "Portrait variant 1" (Portraits), the input copy to variant 2 (Experiments);
// only the image without a recipe shows its positive's "Animals".
for (const [category, keys] of [['Portraits', ['0:shared/same.png']], ['Experiments', ['1:shared/same.png']], ['Animals', ['0:legacy-unassigned.png']]]) {
await categoryFilter(page).selectOption(category);
await expect(cards(page)).toHaveCount(keys.length);
expect(await cardKeys(page)).toEqual(keys);
}
await expect(cards(page)).toHaveCount(2);
let posted = 0;
await page.route(categoryRoute, route => { posted += 1; return route.continue(); });
await checkbox(cards(page).last(), 'same.png').check();
await categoryButton(page).click();
await expect(toast(page)).toHaveText("None of the selected images is linked to a saved prompt, so there is no prompt category to set.");
await expect(page.locator('dialog[open]')).toHaveCount(0);
expect(posted).toBe(0);
await expect(bulkbar(page).locator('.bulkbar > strong')).toHaveText('1 selected');
});
test('Set category reports a failure, keeps the selection and ignores a second press while it runs', async ({ page }) => {
@@ -2275,7 +2059,9 @@ test('Set category reports a failure, keeps the selection and ignores a second p
await checkBoth(page);
const held = [];
await page.route(categoryRoute, route => { held.push(route); });
await applyCategory(page, 'Landscapes');
await categoryButton(page).click();
await categoryDialog(page).getByLabel('Category', { exact: true }).fill('Landscapes');
await categoryDialog(page).getByRole('button', { name: 'Apply', exact: true }).click();
await expect.poll(() => held.length).toBe(1);
await expect(categoryButton(page)).toHaveAttribute('aria-disabled', 'true');
// Playwright waits on an aria-disabled control, so the second press is a dispatched click.
@@ -2312,12 +2098,14 @@ test('leaving Images during Set category finishes the write, and the return relo
await checkBoth(page);
const held = [];
await page.route(categoryRoute, route => { held.push(route); });
await applyCategory(page, 'Landscapes');
await categoryButton(page).click();
await categoryDialog(page).getByLabel('Category', { exact: true }).fill('Landscapes');
await categoryDialog(page).getByRole('button', { name: 'Apply', exact: true }).click();
await expect.poll(() => held.length).toBe(1);
await page.evaluate(() => { window.location.hash = '/overview'; });
await expect(page.locator('#workspace-main h1')).toHaveText('Overview');
// The write is not aborted with the page: the server still receives and answers it.
const answered = page.waitForResponse(response => new URL(response.url()).pathname === IMAGE_CATEGORY);
const answered = page.waitForResponse(response => response.url().endsWith('/prompt_manager/bulk/category'));
await held[0].continue();
expect((await answered).ok()).toBe(true);
await page.evaluate(() => new Promise(resolve => setTimeout(resolve, 150)));
@@ -2328,25 +2116,23 @@ test('leaving Images during Set category finishes the write, and the return relo
await expect(cards(page)).toHaveCount(2);
await expect(bulkbar(page)).toBeEmpty();
await expect(cards(page).getByRole('checkbox', { checked: true })).toHaveCount(0);
await expect(categoryFilter(page).locator('option')).toContainText(['Landscapes']);
await expect(page.getByRole('combobox', { name: 'Filter by category', exact: true }).locator('option')).toContainText(['Landscapes']);
});
test('Set category shows the server counts and coded refusals, and a rejection keeps the selection', async ({ page }) => {
test('Set category limits the field to the server length, shows the server count, and a rejection keeps the selection', async ({ page }) => {
await page.goto(workspace);
await checkBoth(page);
const answers = [
{ status: 400, json: { success: false, error: 'Category cannot exceed 100 characters' } },
{ status: 404, json: { success: false, code: 'files-not-found', error: 'None of the images could be found.' } },
{ status: 200, json: { success: true, category: 'Landscapes', updated: 1, unchanged: 0, not_found: 1 } },
{ status: 200, json: { success: true, updated_count: 0 } },
];
await page.route(categoryRoute, route => route.fulfill(answers.shift()));
for (const expected of ['Category could not be set. Try again.',
'None of the selected images could be found. Refresh the images and try again.',
'Category “Landscapes” set on 1 image. 1 could not be found.']) {
const dialog = await openSetCategory(page);
await expect(dialog.getByLabel('Category', { exact: true })).toHaveAttribute('maxlength', '100');
await dialog.getByLabel('Category', { exact: true }).fill('Landscapes');
await dialog.getByRole('button', { name: 'Apply', exact: true }).click();
for (const expected of ['Category could not be set. Try again.', 'Category set on 0 prompts.']) {
await categoryButton(page).click();
const field = categoryDialog(page).getByLabel('Category', { exact: true });
await expect(field).toHaveAttribute('maxlength', '100');
await field.fill('Landscapes');
await categoryDialog(page).getByRole('button', { name: 'Apply', exact: true }).click();
await expect(toast(page)).toHaveText(expected);
if (answers.length) await expect(bulkbar(page).locator('.bulkbar > strong')).toHaveText('2 selected');
}
@@ -2595,20 +2381,24 @@ test('all matching sets the category through the listing filter and survives a r
// Export metadata waits for loaded images: a press does nothing.
const exportButton = bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true });
await expect(exportButton).toHaveAttribute('title', 'Export metadata reads each image in the browser, so it works on loaded images only.');
// No match listed any more (a stubbed answer): the not-found sentence, and the selection stays.
await page.route(categoryRoute, route => route.fulfill({ status: 404, json: { success: false, code: 'files-not-found', error: 'None of the images could be found.' } }), { times: 1 });
await applyCategory(page, 'Bulk392');
await expect(toast(page)).toHaveText('None of the selected images could be found. Refresh the images and try again.');
// No match linked (a stubbed answer): the nothing-linked sentence, and the selection stays.
await page.route('**/prompt_manager/bulk/category', route => route.fulfill({ json: { success: true, updated_count: 0, prompt_count: 0, skipped: 202 } }), { times: 1 });
const categorize = page.getByRole('dialog', { name: 'Categorize selected images', exact: true });
await bulkbar(page).getByRole('button', { name: 'Set category', exact: true }).click();
await categorize.getByRole('textbox', { name: 'Category', exact: true }).fill('Bulk392');
await categorize.getByRole('button', { name: 'Apply', exact: true }).click();
await expect(toast(page)).toHaveText("None of the selected images is linked to a saved prompt, so there is no prompt category to set.");
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
// The real write: one PUT with the listing filter sets every matching image's own category; no prompt changes.
const writes = categoryWrites(page);
await applyCategory(page, 'Bulk392');
await expect(toast(page)).toHaveText('Category “Bulk392” set on 202 images.');
expect(writes).toEqual([{ method: 'PUT', path: IMAGE_CATEGORY, body: { filter: { sort: 'newest' }, category: 'Bulk392' } }]);
// The real write: one request with the filter, every matching image's positive changes.
await bulkbar(page).getByRole('button', { name: 'Set category', exact: true }).click();
await categorize.getByRole('textbox', { name: 'Category', exact: true }).fill('Bulk392');
const posted = page.waitForRequest(item => item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/bulk/category');
await categorize.getByRole('button', { name: 'Apply', exact: true }).click();
expect((await posted).postDataJSON()).toEqual({ images_filter: { sort: 'newest' }, category: 'Bulk392' });
await expect(toast(page)).toHaveText(/^Category set on \d+ prompts?\.$/);
await expect(bulkbar(page)).toBeEmpty();
const filtered = await (await request.get('/prompt_manager/images/output?mode=workspace&limit=1&category=Bulk392')).json();
expect(filtered.total).toBe(202);
expect((await (await request.get('/prompt_manager/library/options')).json()).categories).not.toContain('Bulk392');
// A return visit keeps "all matching" as the loaded images, and the offer shows again.
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
await page.keyboard.press('ControlOrMeta+a');
@@ -2636,12 +2426,10 @@ test('all matching over 10,000 images says so before any dialog or request', asy
await bulkbar(page).getByRole('button', { name: 'Select all 20,002 matching', exact: true }).click();
await expect(selectedCount(page)).toHaveText('All 20,002 matching selected');
let posted = 0;
page.on('request', item => { if (item.method() !== 'GET') posted += 1; });
// Set category names at most 500 images (#332); the tagging jobs take up to 10,000.
for (const [action, sentence] of [['Set category', 'Select at most 500 images.'], ['Add tags', 'Narrow the view to at most 10,000 images.'],
['Auto-tag selected', 'Narrow the view to at most 10,000 images.']]) {
page.on('request', item => { if (item.method() === 'POST') posted += 1; });
for (const action of ['Set category', 'Add tags', 'Auto-tag selected']) {
await bulkbar(page).getByRole('button', { name: action, exact: true }).click();
await expect(toast(page)).toHaveText(sentence);
await expect(toast(page)).toHaveText('Narrow the view to at most 10,000 images.');
await expect(page.locator('dialog[open]')).toHaveCount(0);
}
expect(posted).toBe(0);
@@ -395,15 +395,14 @@ test('a failed settings read leaves the policy disabled with the reason', async
await expect(row.locator('input')).toBeDisabled();
});
test('the legacy Backup and Restore routes keep working and the admin page links to System', async ({ page, request }) => {
test('the legacy admin Backup and Restore keep working', async ({ page, request }) => {
const bytes = await backupBytes(request);
const restored = await request.post('/prompt_manager/restore', { multipart: { database_file: {
name: 'prompts_backup.db', mimeType: 'application/octet-stream', buffer: bytes } } });
await expect(restored).toBeOK();
expect((await restored.json()).success).toBe(true);
// #330: the admin page's Backup DB and Restore DB open System › Backup & export.
await page.goto('/prompt_manager/admin');
for (const id of ['#backupBtn', '#restoreBtn']) await expect(page.locator(id)).toHaveAttribute('href', 'workspace#/system/backups');
await expect(page.locator('#restoreFileInput')).toHaveCount(1);
});
/** Start a fake running job (a metadata scan another tab started), which an exclusive restore must wait for. */
@@ -123,32 +123,6 @@ test('model options are the four real engines with size and the description show
await expect(actionOf(page, 'Unload')).toBeDisabled();
});
const NEEDS_LLAMA = 'GGUF JoyCaption needs llama-cpp-python. Install it into ComfyUI\'s Python: pip install llama-cpp-python. '
+ 'For a CUDA build: CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python';
test('a downloaded model whose backend package is missing says so and is not offered as ready (#424)', async ({ page, request }) => {
await expect(await request.post('/__test__/tagging/models', { data: { unavailable: ['wd14-vit'],
missing_packages: { gguf: ['llama-cpp-python'], 'wd14-vit': ['onnxruntime'] } } })).toBeOK();
await openTagging(page);
const line = modelRow(page).locator('p').first();
await page.locator('#tag-model').selectOption('gguf');
await expect(line).toHaveText('Downloaded. Needs llama-cpp-python: pip install llama-cpp-python. '
+ 'For a CUDA build: CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python');
// Files and packages are separate: Download (again) stays offered.
await expect(actionOf(page, 'Download again')).not.toHaveAttribute('aria-disabled', 'true');
// Review explains instead of reviewing; nothing is previewed.
await expect(reviewButton(page)).toHaveAttribute('aria-disabled', 'true');
await reviewButton(page).dispatchEvent('click');
await expect(toast(page)).toHaveText(NEEDS_LLAMA);
expect(page.modelCalls.filter(call => call.endsWith('/autotag/batch/preview'))).toEqual([]);
// Not downloaded reads differently and still names what it will need.
await page.locator('#tag-model').selectOption('wd14-vit');
await expect(line).toHaveText('Not downloaded. Download size ~400 MB. Also needs onnxruntime.');
await page.locator('#tag-model').selectOption('wd14-swinv2');
await expect(line).toHaveText('Downloaded. Not loaded.');
await expect(reviewButton(page)).not.toHaveAttribute('aria-disabled', 'true');
});
test('entering the section makes no model call besides status', async ({ page }) => {
await openTagging(page);
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Integrations', exact: true }).click();
@@ -186,26 +186,6 @@ test('a model that is not downloaded keeps Start disabled and says why', async (
expect(posted).toBe(0);
});
const NEEDS_LLAMA = 'GGUF JoyCaption needs llama-cpp-python. Install it into ComfyUI\'s Python: pip install llama-cpp-python. '
+ 'For a CUDA build: CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python';
test('a model missing its backend package keeps Start disabled and names the install command (#424)', async ({ page, request }) => {
await expect(await request.post('/__test__/tagging/models', { data: { unavailable: [],
missing_packages: { gguf: ['llama-cpp-python'] } } })).toBeOK();
await select(page, FIRST);
await barButton(page, 'Auto-tag selected').click();
const status = modal(page).locator('#tag-generation-status');
const start = actionButtons(page).last();
await modal(page).getByRole('combobox', { name: 'Tagging model', exact: true }).selectOption('gguf');
await expect(status).toHaveText(NEEDS_LLAMA);
await expect(start).toHaveAttribute('aria-disabled', 'true');
await modal(page).getByRole('combobox', { name: 'Tagging model', exact: true }).selectOption('wd14-vit');
await expect(status).toHaveText('');
await expect(start).not.toHaveAttribute('aria-disabled', 'true');
await actionButtons(page).first().click();
await expect(modal(page)).toHaveCount(0);
});
test('Suggestions field prefills from the setting and caps chips at the entered count', async ({ page, request }) => {
await expect(await request.post('/__test__/tagging/defaults', { data: { general: 0.5, character: 0.9, limit: 7 } })).toBeOK();
await select(page, FIRST);
-12
View File
@@ -260,18 +260,6 @@ test('unavailable model shows inline status and never downloads', async ({ page,
await expect(dialog(page).locator('.dialog-actions .ca-primary')).toBeFocused();
});
test('a missing backend package is the server\'s install sentence, no traceback page (#424)', async ({ page, request }) => {
await expect(await request.post('/__test__/tagging/models', { data: { unavailable: [],
missing_packages: { 'wd14-swinv2': ['onnxruntime'] } } })).toBeOK();
await openTag(page);
const answer = await generated(page);
expect(answer.status()).toBe(409);
expect(await answer.json()).toMatchObject({ success: false, code: 'model-unavailable' });
await expect(status(page)).toHaveText('WD14 SwinV2 needs onnxruntime. Install it into ComfyUI\'s Python: '
+ 'pip install onnxruntime. Use onnxruntime-gpu for CUDA.');
await expect(chips(page)).toHaveCount(0);
});
test('video tag is manual only', async ({ page, request }) => {
await openTag(page, VIDEO);
const preview = dialog(page).locator('video.tag-image-preview');
@@ -383,62 +383,6 @@ test('manage categories previews real impact before applying',async({page,reques
expect(after.prompts.filter(row=>row[2]==='Scenery')).toHaveLength(3);
});
// #332 MC9, MC10: image categories (their own, set on Images) count beside the prompts. The vocabulary fixture has no
// image files, so these tests add two committed files to the output folder and set their categories through the API.
async function imageCategories(request,category) {
for(const name of ['f02.png','f03.png']) await expect(await request.post('/__test__/add-local-inspect-file',{data:{name}})).toBeOK();
const listing=await(await request.get('/prompt_manager/images/output?mode=workspace&subfolder=local-inspect&limit=40')).json();
const items=listing.images.map(({root_index,root_scope,path})=>({root_index,root_scope,path}));expect(items).toHaveLength(2);
const response=await request.put('/prompt_manager/images/category',{data:{items,category}});await expect(response).toBeOK();
expect((await response.json()).updated).toBe(2);
}
const ownCategories=async request=>(await(await request.get('/prompt_manager/images/categories')).json()).categories;
const IMAGE_KEYS=['0:local-inspect/f02.png','0:local-inspect/f03.png'];
test('manage categories renames the images\' own categories too and the preview counts them (#332 CUJ 6)',async({page,request})=>{
await imageCategories(request,'Landscapes');
const writes=trackWrites(page);await openPage(page);
await expect(categoryRow(page,'Landscapes').locator('p')).toHaveText('3 positive prompts · 2 images');
const box=await openManage(page);
await expect(field(box,'Category')).toHaveValue('Landscapes');
await field(box,'Proposed new category name').fill('Landscape');await button(box,'Preview').click();
await expect(box.locator('p.small')).toHaveText('“Landscapes” is used by 3 positive prompts and 2 images. Apply renames it to “Landscape”.');
await button(box,'Apply').click();
await expect(box).toHaveCount(0);await expect(toast(page)).toHaveText('Category renamed.');
expect(writes.at(-1)).toMatchObject({url:'/prompt_manager/vocabulary/categories/apply',body:{source:'Landscapes',target:'Landscape'}});
await expect(categoryRow(page,'Landscapes')).toHaveCount(0);
await expect(categoryRow(page,'Landscape').locator('p')).toHaveText('3 positive prompts · 2 images');
// The prompts and both images carry the new name; nothing keeps the old one.
expect((await snapshot(request)).prompts.filter(row=>row[2]==='Landscape')).toHaveLength(3);
expect(await ownCategories(request)).toEqual(['Landscape']);
for(const [category,total] of [['Landscape',2],['Landscapes',0]]) {
const listing=await(await request.get(`/prompt_manager/images/output?mode=workspace&limit=40&category=${category}`)).json();
expect(listing.images.map(image=>`${image.root_index}:${image.path}`).sort()).toEqual(total?IMAGE_KEYS:[]);
}
});
test('a category only images hold is listed after the prompt-backed ones and Browse opens Images (#332 CUJ 7)',async({page,request})=>{
await imageCategories(request,'Scans');
await openPage(page);
const operations=surfaces(page).nth(1).locator('.operation');
await expect(operations.locator('h3')).toHaveText([...CATEGORIES.map(([name])=>name),'Scans','Manage category']);
await expect(categoryRow(page,'Scans').locator('p')).toHaveText('2 images');
for(const [name,count] of CATEGORIES) await expect(categoryRow(page,name).locator('p')).toHaveText(`${count} positive prompt${count===1?'':'s'}`);
// Manage category can rename it: its select lists it last.
const box=await openManage(page);
await expect(field(box,'Category').locator('option')).toHaveText([...CATEGORIES.map(([name])=>name),'Scans']);
await button(box,'Cancel').click();await expect(box).toHaveCount(0);
// Browse opens Images filtered by it (its Library would be empty).
await categoryRow(page,'Scans').getByRole('button',{name:'Browse',exact:true}).click();
await expect(page).toHaveURL(/#\/images$/);
await expect(main(page).getByRole('heading',{level:1})).toHaveText('Images');
await expect(page.getByRole('combobox',{name:'Filter by category',exact:true})).toHaveValue('Scans');
const cards=page.locator('article[data-image-key]');await expect(cards).toHaveCount(2);
expect((await cards.evaluateAll(nodes=>nodes.map(node=>node.dataset.imageKey))).sort()).toEqual(IMAGE_KEYS);
await expect(page.getByRole('searchbox',{name:'Search workspace',exact:true})).toHaveValue('');
await expect(page.locator('#workspace-main search.toolbar').getByRole('button',{name:'Filters',exact:true})).toBeVisible();
});
test('an edit during a pending preview needs a new Preview; Apply never sends the old values',async({page,request})=>{
const writes=trackWrites(page);await openPage(page);const before=await snapshot(request);
let release;const held=new Promise(resolve=>{release=resolve;});
@@ -181,7 +181,6 @@ _TABLES = (
"media_image_tags",
"media_annotation_bytes",
"media_annotation_files",
"image_categories",
"image_prompt_pairs",
"prompt_pair_tags",
"prompt_pairs",
@@ -33,7 +33,6 @@ _TABLES = (
"media_image_tags",
"media_annotation_bytes",
"media_annotation_files",
"image_categories",
"image_prompt_pairs",
"prompt_pair_tags",
"prompt_pairs",
@@ -85,7 +85,6 @@ _TABLES = (
"media_image_tags",
"media_annotation_bytes",
"media_annotation_files",
"image_categories",
"image_prompt_pairs",
"prompt_pair_tags",
"prompt_pairs",
+2 -40
View File
@@ -41,18 +41,6 @@ FILES = (
("loose/unlinked.png", "unlinked"),
)
MODELS = ("wd14-swinv2", "wd14-vit", "gguf", "hf")
def _valid_missing(value) -> bool:
"""``{"<model>": ["<pip name>", ...]}`` with known models and string names."""
return isinstance(value, dict) and all(
name in MODELS
and isinstance(names, list)
and all(isinstance(pip, str) and pip for pip in names)
for name, names in value.items()
)
# Unlinked copies enough to virtualize the Images grid (it renders all up to 120).
MAX_EXTRA_IMAGES = 160
_TABLES = (
@@ -63,7 +51,6 @@ _TABLES = (
"media_image_tags",
"media_annotation_bytes",
"media_annotation_files",
"image_categories",
"image_prompt_pairs",
"prompt_pair_tags",
"prompt_pairs",
@@ -229,7 +216,6 @@ def register_tagging_commands(
if not 0 <= extra <= MAX_EXTRA_IMAGES:
raise web.HTTPBadRequest()
model.unavailable = set()
model.missing_packages = {}
model.suggestions = None
model.loaded = None
restore_download()
@@ -248,41 +234,17 @@ def register_tagging_commands(
@routes.post("/__test__/tagging/models")
async def models(request: web.Request) -> web.Response:
"""Mark fake models as not downloaded (``{"unavailable": [...]}``) and,
optionally, as missing backend packages (``"missing_packages":
{"<model>": ["<pip name>", ...]}``, #424)."""
"""Mark fake models as not downloaded (``{"unavailable": [...]}``)."""
data = await request.json() if request.can_read_body else None
if (
request.query
or not isinstance(data, dict)
or "unavailable" not in data
or set(data) - {"unavailable", "missing_packages"}
or set(data) != {"unavailable"}
or not isinstance(data["unavailable"], list)
or any(name not in MODELS for name in data["unavailable"])
or not _valid_missing(data.get("missing_packages", {}))
):
raise web.HTTPBadRequest()
model.unavailable = set(data["unavailable"])
model.missing_packages = {
name: list(names)
for name, names in data.get("missing_packages", {}).items()
}
return web.json_response({"success": True})
@routes.post("/__test__/tagging/loaded")
async def loaded(request: web.Request) -> web.Response:
"""Mark one fake model loaded (``{"model": "<key>"}``), as a batch that
kept it in memory leaves it; ``/autotag/models`` then reports
``model_loaded``. Reset unloads it."""
data = await request.json() if request.can_read_body else None
if (
request.query
or not isinstance(data, dict)
or set(data) != {"model"}
or data["model"] not in MODELS
):
raise web.HTTPBadRequest()
model.loaded = data["model"]
return web.json_response({"success": True})
@routes.post("/__test__/tagging/download")
@@ -119,7 +119,6 @@ def seed_workspace_visual(api: PromptManagerAPI, root: Path) -> None:
"media_image_tags",
"media_annotation_bytes",
"media_annotation_files",
"image_categories",
"image_prompt_pairs",
"prompt_pair_tags",
"prompt_pairs",
+3 -35
View File
@@ -72,10 +72,6 @@ CONCEPT_RECIPES = (
"negative": "oversaturated, flat lighting, watermark, text",
},
)
# #332 D143/D144 (opt-in, ``image_categories``): own image categories, one on
# images only and one shared with prompts. Stored rows count whether or not the
# files exist, so the keys name files the fixture never writes.
IMAGE_CATEGORY_ROWS = (("Landscapes", 2), ("Portraits", 1))
PAIR_ONLY_TAG = "pair_only_detail"
ARCHIVED_ONLY_TAG = "archived_pair_detail"
NEGATIVE_ONLY_TAG = "legacy_negative_detail"
@@ -90,7 +86,6 @@ _TABLES = (
"media_image_tags",
"media_annotation_bytes",
"media_annotation_files",
"image_categories",
"image_prompt_pairs",
"prompt_pair_tags",
"prompt_pairs",
@@ -116,26 +111,8 @@ _SNAPSHOT = {
}
def _seed_image_categories(api: PromptManagerAPI) -> dict:
"""Own image category rows keyed as Images keys them; ``{value: count}``."""
from py.api.images import image_category_key
sources = list(api._get_all_output_dirs())
for category, count in IMAGE_CATEGORY_ROWS:
keys = [
image_category_key(sources, str(Path(sources[0]) / f"{category}_{n}.png"))
for n in range(count)
]
api.db.set_image_categories(keys, category)
return dict(IMAGE_CATEGORY_ROWS)
def seed_vocabulary(
api: PromptManagerAPI,
root: Path,
extra_tags: int = 0,
extra_categories: int = 0,
image_categories: bool = False,
api: PromptManagerAPI, root: Path, extra_tags: int = 0, extra_categories: int = 0
) -> dict:
"""Replace this server's disposable rows with the concept vocabulary."""
db = api.db
@@ -189,7 +166,6 @@ def seed_vocabulary(
"category": f"bulk_category_{number:04}",
},
)
images = _seed_image_categories(api) if image_categories else {}
with db.model.get_connection() as conn:
tags = [row[0] for row in conn.execute("SELECT name FROM tags ORDER BY id")]
api.invalidate_gallery_cache()
@@ -203,7 +179,6 @@ def seed_vocabulary(
"archived_only_tag": ARCHIVED_ONLY_TAG,
"negative_only_tag": NEGATIVE_ONLY_TAG,
"unused_tag": UNUSED_TAG,
"image_categories": images,
}
@@ -224,14 +199,7 @@ def register_vocabulary_commands(
@routes.post("/__test__/vocabulary/reset")
async def reset(request: web.Request) -> web.Response:
data = await request.json() if request.can_read_body else {}
if not isinstance(data, dict) or set(data) - {
"extra_tags",
"extra_categories",
"image_categories",
}:
raise web.HTTPBadRequest()
images = data.get("image_categories", False)
if not isinstance(images, bool):
if not isinstance(data, dict) or set(data) - {"extra_tags", "extra_categories"}:
raise web.HTTPBadRequest()
extra = data.get("extra_tags", 0)
categories = data.get("extra_categories", 0)
@@ -246,7 +214,7 @@ def register_vocabulary_commands(
if request.query:
raise web.HTTPBadRequest()
manifest = await api._run_in_executor(
seed_vocabulary, api, root, extra, categories, images
seed_vocabulary, api, root, extra, categories
)
return web.json_response({"success": True, **manifest})
+6 -210
View File
@@ -11,13 +11,13 @@ import { installHostHooks } from './host-fixture.mjs';
// The worker's owned host must be up before the shared host hooks open the page.
test.beforeEach(async ({ execution }) => { expect(execution.databasePath).toBeTruthy(); });
installHostHooks(test, ['PromptManager', 'PromptManagerText', 'PromptManagerFocused', 'PromptManagerPair', 'PMHistoryInputs', 'PMCycledText']);
installHostHooks(test, ['PromptManager', 'PromptManagerText', 'PromptManagerFocused', 'PromptManagerPair', 'PMHistoryInputs']);
// `primitive`: a core string node the queue-time resolver follows. `legacy`: the frontend's virtual Primitive node,
// whose value is queued as literal text. `preview`: PrimitiveString -> PreviewAny, the shape of the maintainer's
// TextGenerate -> PreviewAny chain, whose text is only known when the node runs.
function build(page, { kind, positive, negative, prefix, saveToLibrary }) {
return page.evaluate(async ({ kind, positive, negative, prefix, saveToLibrary }) => {
function build(page, { kind, positive, negative, prefix }) {
return page.evaluate(async ({ kind, positive, negative, prefix }) => {
const { app } = await import('/scripts/app.js');
app.graph.clear();
const add = (type, x, y) => { const node = LiteGraph.createNode(type); app.graph.add(node); node.pos = [x, y]; return node; };
@@ -28,7 +28,6 @@ function build(page, { kind, positive, negative, prefix, saveToLibrary }) {
const set = (name, value) => { const widget = prompt.widgets.find(item => item.name === name); widget.value = value; if (widget.inputEl) widget.inputEl.value = value; };
set('text', positive.typed);
if (negative) set('negative_text', negative.typed);
if (saveToLibrary !== undefined) prompt.widgets.find(item => item.name === 'save_to_library').value = saveToLibrary;
const drive = (name, side, y) => {
if (!side.via) return;
if (side.via === 'legacy') {
@@ -38,7 +37,6 @@ function build(page, { kind, positive, negative, prefix, saveToLibrary }) {
primitive.widgets[0].value = side.text;
return;
}
if (side.via === 'cycled') { connect(add('PMCycledText', 0, y), 0, prompt, name); return; }
const text = add('PrimitiveString', 0, y);
text.widgets.find(widget => widget.name === 'value').value = side.text;
let source = text;
@@ -74,12 +72,12 @@ function build(page, { kind, positive, negative, prefix, saveToLibrary }) {
await prompt._pmTextSerializerReady;
const { output, workflow } = await app.graphToPrompt();
return { output, workflow, promptNode: String(prompt.id) };
}, { kind, positive, negative, prefix, saveToLibrary });
}, { kind, positive, negative, prefix });
}
async function run(execution, page, testCase, built) {
async function run(execution, page, testCase) {
const prefix = `textlink/${randomUUID()}`;
const { output, workflow, promptNode } = built ?? await build(page, { ...testCase, prefix });
const { output, workflow, promptNode } = await build(page, { ...testCase, prefix });
const id = randomUUID();
const response = await execution.request.post('/prompt', { data: { prompt: output, prompt_id: id, extra_data: { extra_pnginfo: { workflow } } } });
expect(response.ok(), await response.text()).toBe(true);
@@ -195,205 +193,3 @@ for (const [label, linked, negativeVia] of [['positive', ['text'], 'primitive'],
await expectImagesLinked(execution, evidence, sides.text.via === 'preview');
});
}
// #90: with Save to library off the run still produces its image, but PromptManager saves, counts and links nothing,
// and the run's history records no prompt text for it.
for (const kind of ['PromptManager', 'PromptManagerText', 'PromptManagerFocused', 'PromptManagerSplit', 'PromptManagerPair']) {
test(`${kind} with Save to library off saves, counts and links nothing (#90)`, async ({ execution, page }) => {
test.setTimeout(90000);
const text = unique(`Not saved ${kind}`);
const negative = kind === 'PromptManagerPair' ? { typed: unique('Not saved negative') } : undefined;
const prefix = `notsaved/${randomUUID()}`;
const { output, workflow, promptNode } = await build(page, { kind, positive: { typed: text }, negative, prefix, saveToLibrary: false });
expect(output[promptNode].inputs.save_to_library).toBe(false);
const id = randomUUID();
const response = await execution.request.post('/prompt', { data: { prompt: output, prompt_id: id, extra_data: { extra_pnginfo: { workflow } } } });
expect(response.ok(), await response.text()).toBe(true);
let item;
await expect.poll(async () => {
item = (await (await execution.request.get(`/history/${id}`)).json())[id];
return item?.status?.status_str ?? null;
}, { timeout: 20000 }).toBe('success');
let result;
await expect.poll(async () => {
result = await readExecutionRuns(execution.databasePath, id);
const run = result.rows[0];
return Boolean(run && result.captures[run.run_id]);
}, { timeout: 10000, message: 'the run reaches SQLite' }).toBe(true);
const run = result.rows[0];
for (const context of result.captures[run.run_id].contexts || []) {
expect(context.positive_prompt_id ?? null).toBeNull();
expect(context.pair_id ?? null).toBeNull();
}
expect((result.usage[run.run_id] || []).every(entry => entry.run_count === 0)).toBe(true);
const files = Object.values(item.outputs).flatMap(value => value.images || []).map(image => image.filename);
expect(files.length).toBeGreaterThan(0);
await scan(execution);
expect(Object.values(await linkedPrompts(execution, files))).toEqual(files.map(() => null));
for (const query of [text, negative?.typed].filter(Boolean)) {
const found = await (await execution.request.get(`/prompt_manager/search?text=${encodeURIComponent(query)}`)).json();
expect(found.results ?? [], query).toEqual([]);
}
});
}
// #347: a cached PromptManager node counts every job, as an executing one does. Jobs after the first change only
// the sampler seed, so ComfyUI serves the PromptManager node (and its PreviewAny source) from its output cache.
const COUNTS = `import json, sqlite3, sys, pathlib
db = sqlite3.connect(pathlib.Path(sys.argv[1]).resolve().as_uri() + "?mode=ro", uri=True)
one = lambda sql, key: (db.execute(sql, (int(key),)).fetchone() or [None])[0] if key != "-" else None
print(json.dumps({"positive": one("SELECT run_count FROM prompts WHERE id=?", sys.argv[2]),
"negative": one("SELECT run_count FROM prompts WHERE id=?", sys.argv[3]),
"pair": one("SELECT run_count FROM prompt_pairs WHERE id=?", sys.argv[4])}))`;
async function runCounts(execution, context) {
const key = value => value == null ? '-' : String(value);
const { stdout } = await promisify(execFile)(process.env.PM_TEST_PYTHON || 'python',
['-c', COUNTS, execution.databasePath, key(context.positive_prompt_id), key(context.negative_prompt_id), key(context.pair_id)], { timeout: 5000 });
return JSON.parse(stdout);
}
async function setCache(execution, mode) {
expect((await execution.request.post('/pm_execution_fixture/cache', { data: { mode } })).ok()).toBe(true);
}
// Queue an already built workflow again with another sampler seed; returns the job's history item and cached nodes.
async function queueAgain(execution, first, { seed, edit, targets, status = 'success' } = {}) {
const output = structuredClone(first.output);
const sampler = Object.values(output).find(node => node.class_type === 'KSampler');
sampler.inputs.seed = seed;
edit?.(output);
const id = randomUUID();
const data = { prompt: output, prompt_id: id, extra_data: { extra_pnginfo: { workflow: first.workflow } } };
if (targets) data.partial_execution_targets = targets;
const response = await execution.request.post('/prompt', { data });
expect(response.ok(), await response.text()).toBe(true);
let item;
await expect.poll(async () => {
item = (await (await execution.request.get(`/history/${id}`)).json())[id];
return item?.status?.status_str ?? null;
}, { timeout: 20000 }).toBe(status);
const cached = item.status.messages.find(([event]) => event === 'execution_cached')?.[1]?.nodes ?? [];
return { id, item, cached };
}
// The recorded context of one job, once its capture reached SQLite.
async function capturedContext(execution, id) {
let result;
await expect.poll(async () => {
result = await readExecutionRuns(execution.databasePath, id);
const row = result.rows[0];
return row ? result.captures[row.run_id].state : null;
}, { timeout: 10000 }).toMatch(/^(complete|partial)$/);
return result.captures[result.rows[0].run_id].contexts[0];
}
async function firstRun(execution, page, testCase) {
const prefix = `textlink/${randomUUID()}`;
const built = await build(page, { ...testCase, prefix });
const evidence = await run(execution, page, testCase, built);
return { ...evidence, workflow: built.workflow };
}
for (const kind of ['PromptManager', 'PromptManagerFocused', 'PromptManagerSplit', 'PromptManagerText', 'PromptManagerPair']) {
test(`${kind} counts each cached re-run of text linked from PreviewAny (#347)`, async ({ execution, page }) => {
test.setTimeout(90000);
await setCache(execution, 'RAM_PRESSURE');
const pair = kind === 'PromptManagerPair';
const testCase = { kind, positive: { typed: 'Typed under the link', text: unique(`Cached ${kind}`), via: 'preview' },
negative: pair ? { typed: unique('Typed negative') } : undefined };
const first = await firstRun(execution, page, testCase);
for (const seed of [2, 3]) {
const again = await queueAgain(execution, first, { seed });
expect(again.cached, 'ComfyUI served the PromptManager node from its cache').toContain(first.promptNode);
}
const counts = await runCounts(execution, first.context);
expect(counts.positive).toBe(3);
if (pair) expect(counts).toMatchObject({ negative: 0, pair: 3 });
});
}
test.describe('cached runs in one renderer (#347)', () => {
// Cache behavior does not depend on the renderer: one project runs these.
test.beforeEach(({}, testInfo) => { test.skip(testInfo.project.name !== 'classic-start', 'covered in classic-start'); });
const linked = () => ({ kind: 'PromptManager', positive: { typed: 'Typed under the link', text: unique('Cached'), via: 'preview' } });
for (const mode of ['CLASSIC', 'LRU', 'RAM_PRESSURE', 'NONE']) {
test(`counts 3 jobs under the ${mode} cache`, async ({ execution, page }) => {
test.setTimeout(90000);
await setCache(execution, mode);
try {
const first = await firstRun(execution, page, linked());
for (const seed of [2, 3]) {
const again = await queueAgain(execution, first, { seed });
if (mode === 'NONE') expect(again.cached).not.toContain(first.promptNode);
else expect(again.cached).toContain(first.promptNode);
}
expect((await runCounts(execution, first.context)).positive).toBe(3);
} finally { await setCache(execution, 'RAM_PRESSURE'); }
});
}
test('alternating upstream values A, B, A under LRU each count their own prompt', async ({ execution, page }) => {
test.setTimeout(90000);
await setCache(execution, 'LRU');
try {
const [a, b] = [unique('Cycled A'), unique('Cycled B')];
const cycle = async value => expect((await execution.request.post('/pm_execution_fixture/cycled', { data: { value } })).ok()).toBe(true);
await cycle(a);
const first = await firstRun(execution, page, { kind: 'PromptManager', positive: { typed: 'Typed under the link', text: a, via: 'cycled' } });
expect(first.context.positive.raw).toBe(a);
await cycle(b);
const second = await queueAgain(execution, first, { seed: 2 });
expect(second.cached).not.toContain(first.promptNode);
await cycle(a);
const third = await queueAgain(execution, first, { seed: 3 });
expect(third.cached, 'LRU kept A').toContain(first.promptNode);
expect((await runCounts(execution, first.context)).positive).toBe(2);
const bContext = await capturedContext(execution, second.id);
expect(bContext.positive.raw).toBe(b);
expect((await runCounts(execution, bContext)).positive).toBe(1);
} finally { await setCache(execution, 'RAM_PRESSURE'); }
});
test('typed text served from the cache still counts once per job', async ({ execution, page }) => {
test.setTimeout(90000);
await setCache(execution, 'RAM_PRESSURE');
const first = await firstRun(execution, page, { kind: 'PromptManager', positive: { typed: unique('Typed cached') } });
for (const seed of [2, 3]) expect((await queueAgain(execution, first, { seed })).cached).toContain(first.promptNode);
expect((await runCounts(execution, first.context)).positive).toBe(3);
});
test('a job failing downstream of a cached node counts it once; a job that does not need it, never', async ({ execution, page }) => {
test.setTimeout(90000);
await setCache(execution, 'RAM_PRESSURE');
const first = await firstRun(execution, page, linked());
const failing = await queueAgain(execution, first, { seed: 2, status: 'error', edit: output => {
output.pm_failure = { class_type: 'PMHistoryOutput', inputs: { value: [Object.keys(output).find(id => output[id].class_type === 'KSampler'), 0], mode: 'failure' } };
} });
expect(failing.cached).toContain(first.promptNode);
expect((await runCounts(execution, first.context)).positive).toBe(2);
// "Run selected output": only an unrelated output node; ComfyUI still reports the cached PromptManager node
const unrelated = await queueAgain(execution, first, { seed: 2, targets: ['pm_unrelated'], edit: output => {
output.pm_unrelated = { class_type: 'PMHistoryCached', inputs: { value: unique('unrelated') } };
} });
expect(unrelated.cached).toContain(first.promptNode);
expect((await runCounts(execution, first.context)).positive).toBe(2);
});
test('a cached node rewired to the negative input does not count the positive prompt', async ({ execution, page }) => {
test.setTimeout(90000);
await setCache(execution, 'RAM_PRESSURE');
const first = await firstRun(execution, page, linked());
const rewired = await queueAgain(execution, first, { seed: 2, edit: output => {
const sampler = Object.values(output).find(node => node.class_type === 'KSampler');
[sampler.inputs.positive, sampler.inputs.negative] = [sampler.inputs.negative, sampler.inputs.positive];
} });
expect(rewired.cached).toContain(first.promptNode);
expect((await runCounts(execution, first.context)).positive).toBe(1);
// The history capture of a cached job is unchanged: effective text from the cache, no stored-prompt hint
const cachedContext = await capturedContext(execution, (await queueAgain(execution, first, { seed: 3 })).id);
expect(cachedContext.positive.effective).toBe(first.context.positive.effective);
expect(cachedContext.positive_prompt_id).toBeNull();
expect((await runCounts(execution, first.context)).positive).toBe(2);
});
});
-33
View File
@@ -201,39 +201,6 @@ NODE_CLASS_MAPPINGS.update(
{"PMHistoryImages": PMHistoryImages, "PMHistoryImageOutput": PMHistoryImageOutput}
)
# #347: text whose value (and cache signature) follows a value the test sets, so
# one queued graph can serve A, then B, then A again (each cached under LRU).
_CYCLED = {"value": "cycled"}
class PMCycledText:
@classmethod
def INPUT_TYPES(cls) -> dict:
return {"required": {}}
RETURN_TYPES = ("STRING",)
FUNCTION = "execute"
CATEGORY = "testing/execution-history"
@classmethod
def IS_CHANGED(cls, **kwargs: object) -> str:
return _CYCLED["value"]
def execute(self) -> tuple:
return (_CYCLED["value"],)
NODE_CLASS_MAPPINGS["PMCycledText"] = PMCycledText
@PromptServer.instance.routes.post("/pm_execution_fixture/cycled")
async def cycled_text(request: web.Request) -> web.Response:
value = (await request.json())["value"]
if type(value) is not str:
raise web.HTTPBadRequest()
_CYCLED["value"] = value
return web.json_response({"value": value})
@PromptServer.instance.routes.post("/pm_execution_fixture/metadata")
async def metadata(request: web.Request) -> web.Response:
-23
View File
@@ -140,26 +140,3 @@ test('an output saved without metadata links once its recorded run names a store
expect((await request.post('/pm_execution_fixture/metadata', { data: { disabled: false } })).ok()).toBe(true);
}
});
// #332 on the real host: the migration ran at startup and the image category routes
// set, list and remove an own category on a generated output.
test('an output gets its own image category and loses it again', async ({ execution }, testInfo) => {
const request = execution.request;
const filename = await saved(request, graph({ text: `category host ${randomUUID()}`, prefix: 'pm332/category' }));
let image;
await expect.poll(async () => {
const listing = await (await request.get('/prompt_manager/images/output?mode=workspace&q=pm332&rescan=1')).json();
image = (listing.images || []).find(entry => entry.filename === filename || entry.path?.endsWith(`/${filename}`));
return Boolean(image);
}, { timeout: 20000, message: `${filename} is listed by Images` }).toBe(true);
const items = [{ root_index: image.root_index, root_scope: image.root_scope, path: image.path }];
const value = `PM332 host ${testInfo.project.name}`;
const set = await request.put('/prompt_manager/images/category', { data: { items, category: value } });
expect(set.status()).toBe(200);
expect(await set.json()).toEqual({ success: true, category: value, updated: 1, unchanged: 0, not_found: 0 });
expect((await (await request.get('/prompt_manager/images/categories')).json()).categories).toContain(value);
const cleared = await request.delete('/prompt_manager/images/category', { data: { items } });
expect(cleared.status()).toBe(200);
expect(await cleared.json()).toEqual({ success: true, cleared: 1 });
expect((await (await request.get('/prompt_manager/images/categories')).json()).categories).not.toContain(value);
});
+1 -10
View File
@@ -370,7 +370,6 @@ test('workflows saved before #343 load unchanged and their linked text keeps wor
const nodes = Object.fromEntries(app.graph._nodes.map(node => [node.id, node]));
const reloaded = app.graph.serialize();
return {
saves: Object.fromEntries(Object.entries(output).filter(([, node]) => node.class_type.startsWith('PromptManager')).map(([id, node]) => [id, node.inputs.save_to_library])),
values: Object.fromEntries(reloaded.nodes.map(node => [node.id, node.widgets_values])),
links: Object.fromEntries(Object.entries(output).map(([id, node]) => [id, { text: node.inputs.text, negative_text: node.inputs.negative_text }])),
widgetBound: Object.fromEntries(Object.values(nodes).filter(node => node.type.startsWith('PromptManager')).map(node => [node.id, node.inputs.filter(input => ['text', 'negative_text'].includes(input.name) && input.widget).map(input => input.name)])),
@@ -378,16 +377,8 @@ test('workflows saved before #343 load unchanged and their linked text keeps wor
}, workflow);
for (const node of workflow.nodes) {
// Every saved widget value comes back as it was saved, except the typed text main lost under a link (null).
// #90: the last saved value is the editor's own slot (''), where the Save to library switch is now: it loads on.
if (node.widgets_values?.[0] === null) continue;
if (!node.type.startsWith('PromptManager')) { expect(result.values[node.id], `${node.type} ${node.id}`).toEqual(node.widgets_values); continue; }
const saved = node.widgets_values.length - 1;
expect(result.values[node.id].slice(0, saved), `${node.type} ${node.id}`).toEqual(node.widgets_values.slice(0, saved));
expect(result.values[node.id][saved], `${node.type} ${node.id} save_to_library`).toBe(true);
if (node.widgets_values?.[0] !== null) expect(result.values[node.id], `${node.type} ${node.id}`).toEqual(node.widgets_values);
}
// Every PromptManager node of the old workflow queues with Save to library on.
for (const save of Object.values(result.saves)) expect(save).toBe(true);
expect(Object.keys(result.saves).length).toBeGreaterThan(0);
expect(result.links).toMatchObject({
1: { text: 'Saved PromptManager text' }, 2: { text: 'Saved PromptManagerText text' }, 3: { text: 'Saved PromptManagerFocused text' },
4: { text: 'Saved PromptManagerSplit text' }, 5: { text: 'Saved PromptManagerPair text', negative_text: 'Saved pair negative' },
-9
View File
@@ -1,10 +1,7 @@
{
"routes": [
"DELETE /prompt_manager/collections/items",
"DELETE /prompt_manager/collections/{collection_id}",
"DELETE /prompt_manager/delete/{prompt_id}",
"DELETE /prompt_manager/image-annotations/capture/{token}",
"DELETE /prompt_manager/images/category",
"DELETE /prompt_manager/images/favorite",
"DELETE /prompt_manager/images/{image_id}",
"DELETE /prompt_manager/pairs/{pair_id}",
@@ -17,8 +14,6 @@
"GET /prompt_manager/backup",
"GET /prompt_manager/backup/status",
"GET /prompt_manager/categories",
"GET /prompt_manager/collections",
"GET /prompt_manager/collections/membership",
"GET /prompt_manager/diagnostics",
"GET /prompt_manager/export",
"GET /prompt_manager/gallery",
@@ -31,7 +26,6 @@
"GET /prompt_manager/history/runs/{run_id}/outputs",
"GET /prompt_manager/image-annotations/capture/{token}",
"GET /prompt_manager/images/all",
"GET /prompt_manager/images/categories",
"GET /prompt_manager/images/inspect/{image_path}",
"GET /prompt_manager/images/output",
"GET /prompt_manager/images/prompt/{image_path}",
@@ -96,7 +90,6 @@
"GET /prompt_manager/vocabulary/tags/{tag_id}/impact",
"GET /prompt_manager/web",
"GET /prompt_manager/workspace",
"PATCH /prompt_manager/collections/{collection_id}",
"PATCH /prompt_manager/library/{positive_id}",
"PATCH /prompt_manager/pairs/{pair_id}",
"PATCH /prompt_manager/saved-views/{view_id}",
@@ -111,7 +104,6 @@
"POST /prompt_manager/bulk/delete",
"POST /prompt_manager/bulk/tags",
"POST /prompt_manager/cleanup",
"POST /prompt_manager/collections/items",
"POST /prompt_manager/delete_duplicate_images",
"POST /prompt_manager/diagnostics/test-link",
"POST /prompt_manager/gallery/rescan",
@@ -164,7 +156,6 @@
"POST /prompt_manager/vocabulary/tags",
"POST /prompt_manager/vocabulary/tags/{tag_id}/delete",
"PUT /prompt_manager/image-annotations/tags",
"PUT /prompt_manager/images/category",
"PUT /prompt_manager/images/favorite",
"PUT /prompt_manager/prompts/{prompt_id}",
"PUT /prompt_manager/prompts/{prompt_id}/rating",
+19 -85
View File
@@ -76,101 +76,35 @@ test("the workspace keys have their own defaults and validation", () => {
assert.equal(normalizeGallerySettings({ defaultViewMode: "masonry" }).defaultViewMode, "grid");
});
// #330 S13: the legacy gallery's Page options edit only the page-only keys (PAGE_OPTION_KEYS); System › Appearance owns the rest, so a Save
// or a Reset from the legacy page keeps every other stored key.
function legacyGallery(stored, controls, extra = {}) {
test("a legacy gallery Save keeps the workspace keys and every other stored key", () => {
const fs = require("node:fs");
const path = require("node:path");
const vm = require("node:vm");
const settings = require("../../web/js/gallery-settings.js");
const storage = new Map([["gallerySettings", JSON.stringify(stored)]]);
const storage = new Map([["gallerySettings", JSON.stringify({ workspaceLayout: "masonry", workspaceDensity: "compact", someOtherTool: 1 })]]);
const controls = {
lazyLoadingToggle: { checked: true }, imageQualitySelect: { value: "high" }, defaultViewModeSelect: { value: "list" },
defaultLimitSelect: { value: "200" }, gridColumnsSelect: { value: "6" }, showImageInfoToggle: { checked: false },
autoLoadMetadataToggle: { checked: true }, cacheMetadataToggle: { checked: true }, showFilePathsToggle: { checked: false },
sidebarCollapsedByDefaultToggle: { checked: true }, infiniteScrollToggle: { checked: false }, debugModeToggle: { checked: false },
apiTimeoutInput: { value: "30" }, checkThumbnailsAtStartup: { checked: true }, videoAutoplayToggle: { checked: true },
videoMuteToggle: { checked: false }, videoLoopToggle: { checked: true },
};
const context = vm.createContext({
document: { addEventListener() {}, getElementById: id => controls[id] },
window: {}, console: { debug() {}, warn() {}, error() {} }, GallerySettings: settings,
localStorage: { getItem: key => storage.get(key) ?? null, setItem: (key, value) => storage.set(key, value),
removeItem: key => storage.delete(key) },
...extra,
localStorage: { getItem: key => storage.get(key) ?? null, setItem: (key, value) => storage.set(key, value) },
});
const file = path.join(__dirname, "../../web/js/gallery.js");
const Manager = vm.runInContext(`${fs.readFileSync(file, "utf8")}\nGalleryManager;`, context, { filename: file });
const manager = Object.create(Manager.prototype);
const calls = [];
Object.assign(manager, { showNotification: (message, type) => calls.push([type, message]), hideSettings() {},
applySettings: applied => calls.push(["apply", applied.lazyLoading]) });
return { manager, storage, calls, saved: () => JSON.parse(storage.get("gallerySettings")) };
}
const PAGE_OPTION_CONTROLS = () => ({
checkThumbnailsAtStartup: { checked: false }, lazyLoadingToggle: { checked: false },
defaultLimitSelect: { value: "200" },
gridColumnsSelect: { value: "6" }, infiniteScrollToggle: { checked: true }, debugModeToggle: { checked: true },
apiTimeoutInput: { value: "45" },
});
test("a legacy gallery Save keeps the workspace keys and every other stored key", () => {
const { manager, saved } = legacyGallery({ workspaceLayout: "masonry", workspaceDensity: "compact", someOtherTool: 1,
imageQuality: "high", showImageInfo: false, videoMute: false, thumbnailPromptDismissed: "sig", defaultViewMode: "list" },
PAGE_OPTION_CONTROLS());
Object.assign(manager, { showNotification() {}, hideSettings() {}, applySettings() {} });
manager.saveSettings();
const result = saved();
assert.equal(result.workspaceLayout, "masonry");
assert.equal(result.workspaceDensity, "compact");
assert.equal(result.someOtherTool, 1);
assert.equal(result.imageQuality, "high");
assert.equal(result.showImageInfo, false);
assert.equal(result.videoMute, false);
assert.equal(result.thumbnailPromptDismissed, "sig");
// Page options no longer edit the default view (S13 G5), so a stored one stays.
assert.equal(result.defaultViewMode, "list");
// The page-only keys come from the dialog.
assert.equal(result.lazyLoading, false);
assert.equal(result.debugMode, true);
assert.equal(result.checkThumbnailsAtStartup, false);
assert.equal(result.defaultLimit, 200);
assert.equal(result.gridColumns, 6);
assert.equal(result.infiniteScroll, true);
assert.equal(result.apiTimeout, 45);
});
test("a legacy gallery Reset restores only the page options and keeps every other stored key", () => {
const controls = PAGE_OPTION_CONTROLS();
const { manager, saved, calls } = legacyGallery({ workspaceLayout: "masonry", imageQuality: "high", videoMute: false,
thumbnailPromptDismissed: "sig", thumbnailsGenerated: true, lazyLoading: false, debugMode: true, defaultLimit: 200,
gridColumns: 6, infiniteScroll: true, apiTimeout: 45, checkThumbnailsAtStartup: false, defaultViewMode: "list" }, controls,
{ confirm: message => (calls.push(["confirm", message]), true) });
manager.resetSettings();
const result = saved();
const { DEFAULT_SETTINGS } = require("../../web/js/gallery-settings.js");
for (const key of ["checkThumbnailsAtStartup", "lazyLoading", "defaultLimit", "gridColumns",
"infiniteScroll", "debugMode", "apiTimeout"]) {
assert.equal(result[key], DEFAULT_SETTINGS[key], key);
}
// The default view is not a page option (S13 G5): Reset keeps a stored one.
assert.equal(result.defaultViewMode, "list");
assert.equal(result.workspaceLayout, "masonry");
assert.equal(result.imageQuality, "high");
assert.equal(result.videoMute, false);
assert.equal(result.thumbnailPromptDismissed, "sig");
assert.equal(result.thumbnailsGenerated, true);
// The dialog shows the defaults again and the page applies them.
assert.equal(controls.lazyLoadingToggle.checked, DEFAULT_SETTINGS.lazyLoading);
assert.equal(controls.apiTimeoutInput.value, DEFAULT_SETTINGS.apiTimeout);
assert.deepEqual(calls, [["confirm", "Reset these page options to their defaults?"], ["apply", DEFAULT_SETTINGS.lazyLoading],
["success", "Page options reset to defaults"]]);
});
test("Page options no longer hold the default view; System › Appearance's Default layout is the one control", () => {
const { manager } = legacyGallery({}, {});
assert.deepEqual([...manager.constructor.PAGE_OPTION_KEYS], ["checkThumbnailsAtStartup", "lazyLoading", "defaultLimit",
"gridColumns", "infiniteScroll", "debugMode", "apiTimeout"]);
});
test("a viewer mute or loop click remembers only that key and never reads the Page options dialog", () => {
const { manager, saved, calls } = legacyGallery({ defaultLimit: 200, lazyLoading: false, videoMute: true }, {});
manager.storeSetting("videoMute", false);
const result = saved();
assert.equal(result.videoMute, false);
assert.equal(result.defaultLimit, 200);
assert.equal(result.lazyLoading, false);
assert.deepEqual(calls, []);
const saved = JSON.parse(storage.get("gallerySettings"));
assert.equal(saved.workspaceLayout, "masonry");
assert.equal(saved.workspaceDensity, "compact");
assert.equal(saved.someOtherTool, 1);
assert.equal(saved.defaultViewMode, "list");
assert.equal(saved.showImageInfo, false);
assert.equal(saved.videoMute, false);
});
+12 -15
View File
@@ -112,12 +112,11 @@ test('saving gallery settings retains the remembered thumbnail dismissal', () =>
});
// #249 System S5: the stream's work is now an Activity job that outlives the page's connection, so the gallery's
// Cancel asks the server to stop the job. Since #330 the startup dialog is the gallery's only thumbnail stream (System
// › Sources & scans generates the rest). The stream's first frame, `job {id, started}`, names the job; the gallery
// Cancel asks the server to stop the job. The stream's first frame, `job {id, started}`, names the job; the gallery
// cancels by that id, only a job this stream started, and a Cancel before the id waits for it (review I1).
const JOB = 'b'.repeat(32);
function streamFixture({ postFails = false } = {}) {
function streamFixture({ modal = false, postFails = false } = {}) {
const posts = [];
const warnings = [];
const gets = [];
@@ -147,20 +146,18 @@ function streamFixture({ postFails = false } = {}) {
return { success: true };
} } });
const emit = (type, data) => (listeners[type] || []).forEach(handler => handler({ data: JSON.stringify(data) }));
// The dialog's Cancel runs the handler and from then on reads as cancelled, as the startup dialog does.
let cancelled = false;
const ui = { addStatusMessage() {}, progressText: element(), progressPercent: element(), progressBar: element(),
progressDetails: element(), progressETA: element(), isCancelled: () => cancelled,
onCancel: handler => { ui.cancel = () => { cancelled = true; handler(); }; } };
const result = manager.generateThumbnailsWithProgressInModal(ui);
progressDetails: element(), progressETA: element(), isCancelled: () => false,
onCancel: handler => { ui.cancel = handler; } };
const result = modal ? manager.generateThumbnailsWithProgressInModal(ui)
: manager.generateThumbnailsWithProgress({ quality: 'medium' });
const outcome = result.then(value => ({ value }), error => ({ error: error.message }));
const cancel = () => ui.cancel();
const cancel = () => (modal ? ui.cancel() : elements.cancelThumbnailsBtn.click());
const settle = () => new Promise(resolve => setImmediate(resolve));
return { emit, source, posts, gets, warnings, outcome, cancel, settle };
}
const cancelUrl = id => `/prompt_manager/jobs/${id}/cancel`;
const CANCELLED = 'Thumbnail generation cancelled; previews already written are kept';
test('Cancel before the job id arrives cancels that job once it does', async () => {
const f = streamFixture();
@@ -168,8 +165,8 @@ test('Cancel before the job id arrives cancels that job once it does', async ()
await f.settle();
assert.deepEqual(f.posts, []);
assert.equal(f.source.closed, 0); // kept open until the stream names its job
assert.deepEqual(await f.outcome, { error: CANCELLED });
// The job frame is the stream's first (the dialog closes the stream on any later frame once cancelled).
assert.deepEqual(await f.outcome, { error: 'Thumbnail generation cancelled by user' });
f.emit('status', { phase: 'scanning', message: 'Scanning…' });
f.emit('job', { id: JOB, started: true });
await f.settle();
assert.equal(JSON.stringify(f.posts), JSON.stringify([[cancelUrl(JOB), {}]]));
@@ -188,7 +185,7 @@ test('Cancel after the id cancels it at once, and a double Cancel posts once', a
await f.settle();
assert.equal(JSON.stringify(f.posts), JSON.stringify([[cancelUrl(JOB), {}]]));
assert.equal(f.source.closed, 1);
assert.deepEqual(await f.outcome, { error: CANCELLED });
assert.deepEqual(await f.outcome, { error: 'Thumbnail generation cancelled by user' });
assert.deepEqual(f.gets, []);
});
@@ -219,11 +216,11 @@ test('a cancel post that fails is reported, not thrown', async () => {
await f.settle();
assert.equal(f.posts.length, 1);
assert.equal(f.warnings.length, 1);
assert.deepEqual(await f.outcome, { error: CANCELLED });
assert.deepEqual(await f.outcome, { error: 'Thumbnail generation cancelled by user' });
});
test('the startup dialog cancels its own job by id too', async () => {
const f = streamFixture();
const f = streamFixture({ modal: true });
f.cancel();
await f.settle();
assert.deepEqual(f.posts, []);
+113 -19
View File
@@ -65,12 +65,109 @@ function load(file, className, fetchUrl, extra = {}, dom = elements()) {
const BUSY = 'Scan for metadata already running';
test('gallery cache clear reports a refused server clear and never claims success', async () => {
const calls = [];
const { page, dom, notifications, storage } = load('gallery.js', 'GalleryManager', async url => {
calls.push(url);
return jsonResponse(409, { success: false, error: BUSY });
});
page.getSettings = () => ({ thumbnailsGenerated: true });
dom.getElementById('thumbnailStatus').textContent = 'Generated';
await page.clearCache();
assert.deepEqual(calls, ['/prompt_manager/images/clear-thumbnails']);
assert.deepEqual(notifications, [['error', `Thumbnails were not cleared: ${BUSY}`]]);
assert.equal(dom.nodes.thumbnailStatus.textContent, 'Generated');
assert.equal(storage.get('gallerySettings'), undefined); // thumbnailsGenerated is not reset
});
test('gallery cache clear says what it cleared and keeps preferences and other data (#222)', async () => {
const calls = [];
const { page, dom, notifications, storage } = load('gallery.js', 'GalleryManager', async url => {
calls.push(url);
return jsonResponse(200, { success: true, cleared_count: 3 });
});
page.getSettings = () => ({ thumbnailsGenerated: true });
await page.clearCache();
// The server's thumbnails are cleared by their own explicit call; nothing Prompt Manager owns is cached in this
// browser, so no browser entry is removed and the toast says exactly that.
assert.deepEqual(calls, ['/prompt_manager/images/clear-thumbnails']);
assert.deepEqual(notifications, [['success',
"Thumbnails cleared; original images untouched. Nothing of Prompt Manager's is cached in this browser."]]);
assert.equal(dom.nodes.thumbnailStatus.textContent, 'Not generated');
assert.equal(JSON.parse(storage.get('gallerySettings')).thumbnailsGenerated, false);
assert.equal(storage.get('userData'), 'keep');
assert.equal(storage.get('pm_cached'), '1');
assert.equal(storage.get('pm_workspace_theme'), 'light');
assert.equal(storage.get('pm_update_notice_dismissed'), '3.3.0');
});
test("gallery cache clear never deletes another tool's gallery or thumbnail caches (#222)", async () => {
const names = new Set(['comfy-gallery', 'thumbnail-cache']);
const caches = { keys: async () => [...names], delete: async name => names.delete(name) };
const { page, notifications } = load('gallery.js', 'GalleryManager',
async () => jsonResponse(200, { success: true }), { caches, window: { caches } });
page.getSettings = () => ({});
await page.clearCache();
assert.deepEqual([...names], ['comfy-gallery', 'thumbnail-cache']);
assert.equal(notifications.at(-1)[0], 'success');
});
test('gallery cache clear reports a lost server clear instead of success', async () => {
const { page, dom, notifications, storage } = load('gallery.js', 'GalleryManager', async () => {
throw new TypeError('Failed to fetch');
});
page.getSettings = () => ({ thumbnailsGenerated: true });
dom.getElementById('thumbnailStatus').textContent = 'Generated';
await page.clearCache();
assert.deepEqual(notifications, [['error', 'Thumbnails were not cleared: the server could not be reached.']]);
assert.equal(dom.nodes.thumbnailStatus.textContent, 'Generated');
assert.equal(storage.get('gallerySettings'), undefined);
});
test('gallery cache clear says so when browser storage is denied', async () => {
const denied = { get length() { throw new Error('SecurityError'); }, key() { throw new Error('SecurityError'); },
getItem() { throw new Error('SecurityError'); }, setItem() { throw new Error('SecurityError'); },
removeItem() { throw new Error('SecurityError'); } };
const { page, notifications } = load('gallery.js', 'GalleryManager',
async () => jsonResponse(200, { success: true }), { localStorage: denied });
page.getSettings = () => ({});
await page.clearCache();
assert.deepEqual(notifications, [['warning',
'Thumbnails cleared; original images untouched. This browser refused access to its storage, so nothing there was cleared.']]);
});
test('admin LoRA import shows a refusal frame and closes its modal', async () => {
const { page, dom, timers, notifications } = load('admin.js', 'PromptAdmin',
async () => sseResponse([{ type: 'error', status: BUSY, progress: 0, message: BUSY }]));
page.saveLoraSettings = async () => true;
page.loadStatistics = () => assert.fail('a refused import must not reload statistics');
await page.runLoraImport();
assert.equal(dom.nodes.loraImportStatus.textContent, `Error: ${BUSY}`);
assert.deepEqual(notifications.filter(([type]) => type === 'error'), [['error', `LoRA import failed: ${BUSY}`]]);
assert.equal(timers.length, 1);
timers[0]();
assert.deepEqual(notifications.at(-1), ['hide', 'loraImportModal']);
});
test('admin settings save shows why config.json refused it', async () => {
const message = 'config.json cannot be read, so nothing was saved. Fix or remove the file, then save again.';
const { page, notifications } = load('admin.js', 'PromptAdmin',
async () => jsonResponse(409, { success: false, persisted: false, code: 'config-unreadable', error: message }));
page.settings = {};
page.setInfiniteScroll = () => assert.fail('nothing changes after a refusal');
page._collectScanPaths = () => [];
page.saveLoraSettings = async () => assert.fail('nothing else is saved after a refusal');
await page.saveSettings();
@@ -81,30 +178,27 @@ test('admin settings save keeps its plain message when the reply has no reason',
const { page, notifications } = load('admin.js', 'PromptAdmin', async () => ({ ok: false, status: 500,
json: async () => { throw new SyntaxError('not json'); } }));
page.settings = {};
page._collectScanPaths = () => [];
await page.saveSettings();
assert.deepEqual(notifications, [['error', 'Failed to save settings']]);
});
// #330 S13-9: the admin page's reduced Page options save sends only the list's scroll default, so a stale legacy form
// can no longer overwrite scan folders or other settings edited in System.
test('admin Page options save posts only infinite_scroll', async () => {
const sent = [];
const { page, dom, notifications } = load('admin.js', 'PromptAdmin', async (url, init) => {
sent.push([url, init.method, JSON.parse(init.body)]);
return jsonResponse(200, { success: true, restart_required: true });
});
page.settings = { infiniteScroll: false };
const scrolled = [];
page.setInfiniteScroll = (on, options) => scrolled.push([on, { ...options }]);
dom.getElementById('infiniteScrollDefault').checked = true;
test('admin maintenance names the job that refused it', async () => {
const { page, dom, notifications } = load('admin.js', 'PromptAdmin',
async () => jsonResponse(409, { success: false, error: BUSY }), {
escapeHtml: text => String(text),
document: { readyState: 'loading', addEventListener() {}, getElementById: id => dom.getElementById(id),
querySelectorAll: selector => (selector === '.maintenance-option:checked' ? [{ id: 'vacuum' }] : []) },
});
await page.saveSettings();
await page.runMaintenance();
assert.deepEqual(sent, [['/prompt_manager/settings', 'POST', { infinite_scroll: true }]]);
assert.deepEqual(scrolled, [[true, { session: false }]]);
assert.equal(page.settings.infiniteScroll, true);
assert.deepEqual(notifications, [['success', 'Page options saved'], ['hide', 'settingsModal']]);
const results = dom.nodes.maintenanceResults.innerHTML;
assert.match(results, /Maintenance Failed/);
assert.match(results, new RegExp(BUSY));
assert.doesNotMatch(results, /HTTP 409/);
assert.deepEqual(notifications, [['error', '❌ Maintenance failed']]);
assert.equal(dom.nodes.runMaintenanceBtn.disabled, false);
});
-68
View File
@@ -1,68 +0,0 @@
// #90: the node's save_to_library switch (web/js/save-to-library.js).
const test = require('node:test');
const assert = require('node:assert/strict');
const source = import('../../web/js/save-to-library.js');
function classicNode(saved) {
const names = ['text', 'category', 'tags', 'search_text', 'prepend_text', 'append_text', 'save_to_library', 'prompt_manager_search_ui'];
const widgets = names.map(name => ({name, value: name === 'save_to_library' ? true : ''}));
const node = {widgets, widgets_values: saved ? [...saved] : undefined};
// What a host configure does: each saved value goes to the widget at its index.
(saved || []).forEach((value, index) => { if (widgets[index]) widgets[index].value = value; });
return node;
}
test('an older workflow stored the editor slot where the switch is now: it loads on', async () => {
const {normalizeSaveWidget, savesToLibrary} = await source;
const node = classicNode(['a cat', 'c', 't', '', '', '', '']);
assert.equal(node.widgets[6].value, '', 'the host put the old DOM slot value into the switch');
assert.equal(normalizeSaveWidget(node), true);
assert.equal(node.widgets[6].value, true);
assert.equal(node.widgets_values[6], true);
assert.equal(savesToLibrary(node), true);
assert.deepEqual(node.widgets_values.slice(0, 6), ['a cat', 'c', 't', '', '', ''], 'no earlier value moved');
});
test('null, undefined and missing slots load on; saved booleans are kept', async () => {
const {normalizeSaveWidget, savesToLibrary} = await source;
for (const value of [null, undefined, 0, 'false']) {
const node = classicNode(['a', '', '', '', '', '', value]);
normalizeSaveWidget(node);
assert.equal(savesToLibrary(node), true, String(value));
}
const short = classicNode(['a', '', '', '', '', '']);
assert.equal(normalizeSaveWidget(short), false);
assert.equal(savesToLibrary(short), true);
const off = classicNode(['a', '', '', '', '', '', false]);
assert.equal(normalizeSaveWidget(off), false);
assert.equal(savesToLibrary(off), false);
});
test('a node without the widget (an older backend) saves', async () => {
const {normalizeSaveWidget, savesToLibrary} = await source;
assert.equal(savesToLibrary({widgets: [{name: 'text', value: ''}]}), true);
assert.equal(normalizeSaveWidget({widgets: []}), false);
assert.equal(savesToLibrary(undefined), true);
});
test('the switch moves after the editor fallbacks so every frontend has one order', async () => {
const {placeSaveWidgetLast} = await source;
const widgets = ['text', 'category', 'tags', 'search_text', 'save_to_library', 'prepend_text', 'append_text'].map(name => ({name}));
const node = {widgets};
assert.equal(placeSaveWidgetLast(node), true);
assert.deepEqual(node.widgets.map(widget => widget.name), ['text', 'category', 'tags', 'search_text', 'prepend_text', 'append_text', 'save_to_library']);
assert.equal(placeSaveWidgetLast(node), false, 'already last');
assert.equal(placeSaveWidgetLast({widgets: [{name: 'text'}]}), false);
});
test('setting the switch updates the widget, its saved value and notifies the host', async () => {
const {setSaveToLibrary, savesToLibrary} = await source;
const calls = [];
const node = classicNode(['a', '', '', '', '', '', true]);
node.widgets[6].callback = value => calls.push(value);
setSaveToLibrary(node, false);
assert.equal(savesToLibrary(node), false);
assert.equal(node.widgets_values[6], false);
assert.deepEqual(calls, [false]);
setSaveToLibrary({widgets: []}, false);
});
-49
View File
@@ -185,26 +185,6 @@ test('legacy fill applies to local records', async () => {
assert.notEqual(zeroed.prompts.positive.text, '');
});
test('a branch named by recorded run history shows the Recorded run badge', async () => {
// #386: the server labels sides whose save node recorded history named; Execution recorded stays stronger.
const base = await localRecord(new File([fs.readFileSync(new URL('f09.png', PARITY))], 'f09.png'));
const recorded = {...base, prompts:{positive:{...base.prompts.positive, source:'recorded-run'}, negative:{...base.prompts.negative, source:'recorded-run'}}};
let view = renderFull(recorded);
try {
assert.deepEqual(view.texts(n => n.className === 'chip blue'), ['Recorded run']);
assert.ok(view.texts(n => n.tagName === 'p').includes('Prompt evidence: recorded run'));
} finally { view.done(); }
view = renderFull({...recorded, prompts:{...recorded.prompts, negative:{...recorded.prompts.negative, source:'execution'}}});
try {
assert.deepEqual(view.texts(n => n.className === 'chip blue'), ['Execution recorded']);
} finally { view.done(); }
// A recorded branch whose negative left out a run-time part is still inferred from the workflow.
view = renderFull({...recorded, prompts:{...recorded.prompts, negative:{...recorded.prompts.negative, source:'runtime-parts-omitted'}}});
try {
assert.deepEqual(view.texts(n => n.className === 'chip blue'), ['Workflow inferred']);
} finally { view.done(); }
});
// #249 PR C Task C3 (D88): one chip row, the file's current image tags first, then D21's linked tags not already shown.
function tagged(imageTags, library = {status:'linked', tags:['Minimal', 'warm_light', 'editorial'], tags_scope:'positive'}) {
return {...record(), imageTags, library};
@@ -383,32 +363,3 @@ test('the D117 wrap rule targets only the settings summary button', async () =>
assert.deepEqual(rules.map(line => line.trim()), ['.ca-tools #viewer-details .settings-summary { white-space: normal; /* D117 */ }']);
assert.doesNotMatch(css, /#viewer-details > div > button/);
});
test('category chip names its origin visibly and shows an own category for any link status (#332 D146)', async () => {
const base = await localRecord(localPng('a.png'));
const chipTexts = record => {
const {f, done} = renderFull(record);
try {
return f.walk(f.container, n => /(^| )chip( |$)/.test(n.className ?? '')).map(n => n.textContent);
} finally { done(); }
};
const cases = [
[{status:'none', category:'Scans', category_scope:'image'}, 'Image category: Scans · this image'],
[{status:'ambiguous', category:'Scans', category_scope:'image'}, 'Image category: Scans · this image'],
[{status:'linked', category:'Portraits', category_scope:'image'}, 'Image category: Portraits · this image'],
[{status:'linked', category:'Landscapes', category_scope:'positive'}, 'Positive prompt category: Landscapes · prompt'],
[{status:'linked', category:'Recipes', category_scope:'pair'}, 'Recipe category: Recipes · recipe'],
];
for (const [library, text] of cases) {
const chips = chipTexts({...base, library:{...base.library, tags:[], tags_scope:null, ...library}});
assert.ok(chips.includes(text), `${JSON.stringify(library)} -> ${chips}`);
}
for (const library of [
{status:'none', category:'Inherited', category_scope:'positive'},
{status:'linked', category:null, category_scope:null},
{status:'none', category:null, category_scope:null},
]) {
const chips = chipTexts({...base, library:{...base.library, tags:[], tags_scope:null, ...library}});
assert.equal(chips.some(chip => /category: /.test(chip)), false, JSON.stringify(library));
}
});
+42 -50
View File
@@ -1,61 +1,53 @@
const test = require('node:test');
const assert = require('node:assert/strict');
// #332 D140: Set category writes each selected image's own category; Use prompt category removes it.
const source = import('../../web/js/workspace/images-category.mjs');
test('the dialog keeps the concept\'s simpleInput words and names the routes', async () => {
const {CATEGORY_DIALOG, USE_PROMPT_CATEGORY, REMOVE_DIALOG, CATEGORY_URL, IMAGE_CATEGORIES_URL, CATEGORY_FAILED,
CATEGORY_MAX_LENGTH, MAX_CATEGORY_IMAGES} = await source;
assert.deepEqual({...CATEGORY_DIALOG}, {title:'Categorize selected images', label:'Category', action:'Apply'});
assert.equal(USE_PROMPT_CATEGORY, 'Use prompt category');
assert.deepEqual({...REMOVE_DIALOG}, {title:'Use prompt category', action:'Remove'});
assert.equal(CATEGORY_URL, '/prompt_manager/images/category');
assert.equal(IMAGE_CATEGORIES_URL, '/prompt_manager/images/categories');
const linked = (id, extra = {}) => ({path:`p${id}.png`, linked:{status:'linked', positive_id:id, ...extra}});
test('targets are the distinct linked positives, in selection order, and the rest count as skipped', async () => {
const {categoryTargets} = await source;
const images = [
linked(7), linked(3), linked(7),
{path:'u.png', linked:{status:'unlinked', positive_id:null}},
{path:'a.png', linked:{status:'ambiguous', positive_id:4}},
{path:'v.png', linked:{status:'unavailable', positive_id:null}},
{path:'none.png'},
{path:'bad.png', linked:{status:'linked', positive_id:'9'}},
{path:'zero.png', linked:{status:'linked', positive_id:0}},
{path:'frac.png', linked:{status:'linked', positive_id:1.5}},
null,
];
assert.deepEqual(categoryTargets(images), {promptIds:[7, 3], skipped:8});
assert.deepEqual(categoryTargets([]), {promptIds:[], skipped:0});
assert.deepEqual(categoryTargets([linked(2)]), {promptIds:[2], skipped:0});
});
test('the outcome toast counts the prompts and the skipped images', async () => {
const {categoryOutcome} = await source;
assert.equal(categoryOutcome(3, 0), 'Category set on 3 prompts.');
assert.equal(categoryOutcome(1, 0), 'Category set on 1 prompt.');
assert.equal(categoryOutcome(2, 4), 'Category set on 2 prompts. 4 selected images had no linked prompt and were skipped.');
assert.equal(categoryOutcome(1, 1), 'Category set on 1 prompt. 1 selected image had no linked prompt and was skipped.');
});
test('the dialog and failure copy is the plan\'s', async () => {
const {CATEGORY_DIALOG, CATEGORY_NONE_LINKED, CATEGORY_FAILED} = await source;
assert.deepEqual(CATEGORY_DIALOG, {title:'Categorize selected images', label:'Category', action:'Apply'});
assert.equal(CATEGORY_NONE_LINKED, "None of the selected images is linked to a saved prompt, so there is no prompt category to set.");
assert.equal(CATEGORY_FAILED, 'Category could not be set. Try again.');
assert.equal(CATEGORY_MAX_LENGTH, 100);
assert.equal(MAX_CATEGORY_IMAGES, 500);
assert.ok(Object.isFrozen(CATEGORY_DIALOG));
});
test('a set counts unchanged files as set and names files that could not be found', async () => {
const {setOutcome} = await source;
assert.equal(setOutcome({updated:1, unchanged:0, not_found:0}, 'Portraits'), 'Category “Portraits” set on 1 image.');
assert.equal(setOutcome({updated:1, unchanged:2, not_found:0}, 'Scans'), 'Category “Scans” set on 3 images.');
assert.equal(setOutcome({updated:2, unchanged:0, not_found:1}, 'Scans'), 'Category “Scans” set on 2 images. 1 could not be found.');
assert.equal(setOutcome({updated:0, unchanged:1, not_found:3}, 'Scans'), 'Category “Scans” set on 1 image. 3 could not be found.');
assert.equal(setOutcome({}, 'Scans'), 'Category “Scans” set on 0 images.');
});
test('a removal says how many lost their own category, or that none had one', async () => {
const {clearOutcome, NONE_HAD_OWN} = await source;
assert.equal(clearOutcome({cleared:1}), 'Removed the image category from 1 image.');
assert.equal(clearOutcome({cleared:4}), 'Removed the image category from 4 images.');
assert.equal(clearOutcome({cleared:0}), NONE_HAD_OWN);
assert.equal(NONE_HAD_OWN, 'None of the selected images had its own category.');
});
test('the confirmation states the count', async () => {
const {removeQuestion} = await source;
assert.equal(removeQuestion(1), "Remove the image category from the 1 selected image? Each one then shows its prompt's " +
'category, or none if it has no linked prompt. This cannot be undone.');
assert.match(removeQuestion(1200), /^Remove the image category from the 1,200 selected images\? /);
});
test('coded failures have their own sentences, anything else the fixed one', async () => {
const {failureMessage, NONE_FOUND, SOURCES_CHANGED, CATEGORY_FAILED} = await source;
assert.equal(failureMessage({code:'files-not-found'}), NONE_FOUND);
assert.equal(NONE_FOUND, 'None of the selected images could be found. Refresh the images and try again.');
assert.equal(failureMessage({code:'source-changed'}), SOURCES_CHANGED);
assert.equal(SOURCES_CHANGED, 'The sources changed. Refresh the images and try again.');
assert.equal(failureMessage({code:'too-many-images', message:'Select at most 500 images.'}), 'Select at most 500 images.');
for (const error of [null, undefined, new Error('x'), {code:'other', message:'secret'}]) {
assert.equal(failureMessage(error), CATEGORY_FAILED);
test('the updated count is the server\'s when it is a count, else the number of prompts posted', async () => {
const {updatedCount} = await source;
assert.equal(updatedCount({updated_count:0}, 3), 0);
assert.equal(updatedCount({updated_count:2}, 3), 2);
for (const result of [{}, {updated_count:-1}, {updated_count:1.5}, {updated_count:'2'}, null, undefined]) {
assert.equal(updatedCount(result, 3), 3, JSON.stringify(result));
}
});
test('the toolbar list merges prompt and image categories once each', async () => {
const {mergeCategories} = await source;
assert.deepEqual(mergeCategories(['Landscapes', 'Portraits'], ['Scans', 'Landscapes']), ['Landscapes', 'Portraits', 'Scans']);
assert.deepEqual(mergeCategories(['A', '', ' ', 7], null), ['A']);
assert.deepEqual(mergeCategories(undefined, ['landscapes', 'Landscapes']), ['landscapes', 'Landscapes']);
test('the category field is limited to the server\'s 100 characters', async () => {
const {CATEGORY_MAX_LENGTH} = await source;
assert.equal(CATEGORY_MAX_LENGTH, 100);
});
@@ -135,41 +135,6 @@ test('a model that is not downloaded is named with the server sentence', () => {
assert.throws(() => modelUnavailable({}, 'nope'), TypeError);
});
test('a downloaded model whose backend package is missing names the package and the command (#424)', () => {
const { modelUnavailable, needsPackages } = state;
const gguf = { downloaded: true, name: 'GGUF (Recommended)', label: 'GGUF JoyCaption', missing_packages: ['llama-cpp-python'],
install_command: 'pip install llama-cpp-python',
install_note: 'For a CUDA build: CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python' };
const sentence = 'GGUF JoyCaption needs llama-cpp-python. Install it into ComfyUI\'s Python: pip install llama-cpp-python. '
+ 'For a CUDA build: CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python';
assert.equal(modelUnavailable({ gguf }, 'gguf'), sentence);
// Not downloaded is named first, in its own sentence.
assert.equal(modelUnavailable({ gguf: { ...gguf, downloaded: false } }, 'gguf'),
'GGUF (Recommended) is not downloaded. Download it in System › Auto-tagging.');
assert.equal(modelUnavailable({ gguf: { ...gguf, missing_packages: [] } }, 'gguf'), '');
assert.equal(needsPackages('HuggingFace JoyCaption', { missing_packages: ['accelerate', 'bitsandbytes'] }),
'HuggingFace JoyCaption needs accelerate and bitsandbytes. Install them into ComfyUI\'s Python: pip install accelerate bitsandbytes.');
assert.equal(needsPackages('X', { missing_packages: ['a', 'b', 'c'], install_command: 'pip install a b c' }),
'X needs a, b and c. Install them into ComfyUI\'s Python: pip install a b c.');
assert.equal(needsPackages('X', null), '');
assert.equal(needsPackages('X', { missing_packages: [1, ''] }), '');
// Without the server's label the option's own name is used.
assert.ok(modelUnavailable({ 'wd14-vit': { downloaded: true, missing_packages: ['onnxruntime'] } }, 'wd14-vit')
.startsWith('WD14 ViT needs onnxruntime.'));
});
test('the bulk dialog keeps Start disabled while the chosen model misses a package (#424)', async () => {
const { createTaggingOptionsController } = state;
const models = { 'wd14-vit': { downloaded: true, label: 'WD14 ViT', missing_packages: ['onnxruntime'],
install_command: 'pip install onnxruntime' } };
const options = createTaggingOptionsController({ api: { get: async () => ({ success: true, models, model_type: 'wd14-vit' }) } });
await options.open();
assert.equal(options.state.model, 'wd14-vit');
assert.equal(options.state.canStart, false);
assert.equal(options.state.unavailable, 'WD14 ViT needs onnxruntime. Install it into ComfyUI\'s Python: pip install onnxruntime.');
options.dispose();
});
test('the bulk options start from the server defaults and wait for them before Start', async () => {
const { createTaggingOptionsController } = state;
let answer;
@@ -278,7 +278,6 @@ test('record rejects root path media and invalid versioned fields with retryable
data => { data.resolution.sampler_id = {}; },
data => { data.library.tags_scope = 'recipe'; },
data => { data.library.category_scope = 'recipe'; },
data => { data.library.tags_scope = 'image'; },
data => { delete data.library.category_scope; },
data => { delete data.library.tags_scope; },
]) {
@@ -943,14 +942,3 @@ test("names the server quotes (parentheses, '!*') still match their original and
assert.equal(record.download.url, entry.url);
assert.deepEqual(record.preview, { url: entry.thumbnail_url });
});
test('record accepts an image\'s own category on any link status (#332)', () => {
const toRecord = contract('toInspectionRecord');
for (const status of ['linked', 'none', 'ambiguous', 'unavailable']) {
const data = envelope();
Object.assign(data.library, { status, category: 'Scans', category_scope: 'image' });
const record = toRecord(data, image(), extractGenerationFacts);
assert.equal(record.library.category, 'Scans');
assert.equal(record.library.category_scope, 'image');
}
});
+2 -2
View File
@@ -63,9 +63,9 @@ test('status line shows only server-reported backups (D103)', async () => {
const PREVIEW = Object.freeze({
success: true, token: 'r'.repeat(43), expires_at: '2026-10-25T12:15:00+00:00',
summary: { prompts: 1234, positives: 900, pairs: 3, archived_pairs: 0, image_links: 56, runs: 1, saved_views: 2, collections: 1 },
summary: { prompts: 1234, positives: 900, pairs: 3, archived_pairs: 0, image_links: 56, runs: 1, saved_views: 2 },
schema: { migrated: true, library_state: true }, integrity: 'ok', documents: { repaired: 0, cleared: 0, images: 0 },
live: { prompts: 1, positives: 1, pairs: 0, archived_pairs: 0, image_links: 0, runs: 0, saved_views: 0, collections: 0 }, size: 4096,
live: { prompts: 1, positives: 1, pairs: 0, archived_pairs: 0, image_links: 0, runs: 0, saved_views: 0 }, size: 4096,
safety_backup: { destination_label: 'prompts.db.backup_<date and time>' },
});
-54
View File
@@ -482,60 +482,6 @@ test('Tag output images › Review previews the drafts, else says why; a refused
} finally { scope.dispose(); dom.restore(); }
});
// #424: GGUF on disk without llama-cpp-python.
const NEEDS_LLAMA = { missing_packages: ['llama-cpp-python'], install_command: 'pip install llama-cpp-python',
install_note: 'For a CUDA build: CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python' };
const NEEDS_SENTENCE = 'GGUF JoyCaption needs llama-cpp-python. Install it into ComfyUI\'s Python: pip install llama-cpp-python. '
+ 'For a CUDA build: CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python';
test('a model on disk whose backend package is missing never reads as ready (#424)', async () => {
const { modelDescription, readStatus, batchCloseNote, STATUS_COPY } = await tagging;
const status = readStatus({ ...MODELS, models: { ...MODELS.models,
gguf: entry('GGUF JoyCaption', '~4.5 GB', { ...NEEDS_LLAMA }),
'wd14-vit': entry('WD14 ViT', '~400 MB', { downloaded: false, missing_packages: ['onnxruntime'],
install_command: 'pip install onnxruntime', install_note: 'Use onnxruntime-gpu for CUDA.' }),
hf: entry('HuggingFace JoyCaption', '~16 GB', { downloaded: false, model_exists: undefined,
missing_packages: ['accelerate', 'bitsandbytes'], install_command: 'pip install accelerate bitsandbytes' }) } });
const ready = modelDescription(status, 'gguf');
assert.notEqual(ready, STATUS_COPY.downloaded);
assert.equal(ready, 'Downloaded. Needs llama-cpp-python: pip install llama-cpp-python. '
+ 'For a CUDA build: CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python');
// Not downloaded reads differently, and still says what it will need.
assert.equal(modelDescription(status, 'wd14-vit'), 'Not downloaded. Download size ~400 MB. Also needs onnxruntime.');
assert.equal(modelDescription(status, 'hf'), 'Not downloaded. Download size ~16 GB. Also needs accelerate and bitsandbytes.');
const partial = readStatus({ ...MODELS, models: { ...MODELS.models, gguf: entry('GGUF JoyCaption', '~4.5 GB',
{ downloaded: false, model_exists: true, mmproj_exists: false, ...NEEDS_LLAMA }) } });
assert.equal(modelDescription(partial, 'gguf'), 'Partly downloaded. Download again to finish. Also needs llama-cpp-python.');
// Untrusted shapes are ignored; a missing command is built from the names.
const odd = readStatus({ ...MODELS, models: { ...MODELS.models, gguf: entry('GGUF JoyCaption', '~4.5 GB',
{ missing_packages: ['llama-cpp-python', 3, ''], install_command: 7, install_note: {} }) } });
assert.equal(modelDescription(odd, 'gguf'), 'Downloaded. Needs llama-cpp-python: pip install llama-cpp-python.');
const junk = readStatus({ ...MODELS, models: { ...MODELS.models, gguf: entry('GGUF JoyCaption', '~4.5 GB',
{ missing_packages: 'llama-cpp-python' }) } });
assert.equal(modelDescription(junk, 'gguf'), STATUS_COPY.downloaded);
// The batch review says why it cannot start, in the server's sentence.
assert.equal(batchCloseNote({ ...PLAN, model: { type: 'gguf', label: 'GGUF JoyCaption', downloaded: true, ...NEEDS_LLAMA } }),
NEEDS_SENTENCE);
assert.equal(batchCloseNote({ ...PLAN, model: { ...PLAN.model, downloaded: false, ...NEEDS_LLAMA } }),
'WD14 SwinV2 is not downloaded. Download it first.');
});
test('Review explains a missing backend package instead of reviewing (#424)', async () => {
const models = { ...MODELS, models: { ...MODELS.models, gguf: entry('GGUF JoyCaption', '~4.5 GB', { ...NEEDS_LLAMA }) } };
const { dom, scope, button, byId, calls } = await mounted({ models });
try {
byId('tag-model').value = 'gguf'; await byId('tag-model').dispatch('change');
const review = button('Review');
assert.equal(review.getAttribute('aria-disabled'), 'true');
assert.equal(review.dataset.reason, NEEDS_SENTENCE);
await review.dispatch('click');
assert.equal(calls.toasts.at(-1), NEEDS_SENTENCE);
assert.deepEqual(calls.post, []);
// Download stays offered: files and packages are separate.
assert.equal(button('Download again').getAttribute('aria-disabled'), null);
} finally { scope.dispose(); dom.restore(); }
});
test('settingRefusal marks only an Auto-tagging save\'s plain 400 as invalid-setting (CUJ 2a)', async () => {
const { settingRefusal } = await import('../../web/js/workspace/system.mjs');
const refusal = (extra = {}) => Object.assign(new Error('General-tag confidence must be between 0 and 1.'),
-37
View File
@@ -567,43 +567,6 @@ test('an input dialog with a fallback applies it for an empty value', async () =
}
});
// #332: Set category's field suggests the categories under its own datalist id, and "Use prompt category" sits
// between Cancel and Apply, closing the dialog before it runs.
test('an input dialog offers suggestions and a secondary action', async () => {
const { openInputDialog, closeWorkspaceDialog, INPUT_SUGGESTIONS_ID } = await dialogsModule;
const dom = fakeDOM();
const scope = (await coreModule).createScope();
try {
const calls = [];
openInputDialog({ scope, title: 'Categorize selected images', label: 'Category', action: 'Apply', maxLength: 100,
suggestions: ['Landscapes', 'Scans'], secondary: { label: 'Use prompt category', onClick: () => calls.push('secondary') },
onApply: value => calls.push(value) });
assert.equal(INPUT_SUGGESTIONS_ID, 'workspace-input-suggestions');
const body = byId(dom, 'modal-body');
const [field, list, actions] = body.children;
const input = field.children[1];
assert.equal(input.getAttribute('list'), 'workspace-input-suggestions');
assert.equal(list.tagName.toLowerCase(), 'datalist');
assert.equal(list.id, 'workspace-input-suggestions');
assert.deepEqual(list.children.map(option => option.value), ['Landscapes', 'Scans']);
assert.deepEqual(actions.children.map(node => node.textContent), ['Cancel', 'Use prompt category', 'Apply']);
actions.children[1].click();
assert.deepEqual(calls, ['secondary']);
// No suggestions: no datalist and no list attribute.
openInputDialog({ scope, title: 'T', label: 'L', action: 'A', onApply() {} });
const [plainField, plainActions] = byId(dom, 'modal-body').children;
assert.equal(plainField.children[1].getAttribute('list'), null);
assert.deepEqual(plainActions.children.map(node => node.textContent), ['Cancel', 'A']);
assert.throws(() => openInputDialog({ scope, title: 'T', label: 'L', action: 'A', suggestions: [1], onApply() {} }), TypeError);
assert.throws(() => openInputDialog({ scope, title: 'T', label: 'L', action: 'A', secondary: { label: '' }, onApply() {} }), TypeError);
assert.throws(() => openInputDialog({ scope, title: 'T', label: 'L', action: 'A', secondary: { label: 'X' }, onApply() {} }), TypeError);
} finally {
closeWorkspaceDialog();
scope.dispose();
dom.restore();
}
});
test('dialogs restore focus to the trigger on Escape', async () => {
const { openRefineDialog, openImportDialog, openCopyFallback, closeWorkspaceDialog } = await dialogsModule;
const dom = fakeDOM();
@@ -553,16 +553,3 @@ test('delete text names the tag, its real links and what stays unchanged', async
assert.equal(deleteTagText('unused', zero),
'“unused” is not linked to any saved prompt or image. Deleting it removes only the name.');
});
test('a category row counts prompts and images, and an image-only row browses Images (#332 D143)', async () => {
const {categoryRowText, categoryBrowseTarget, categoryPreviewText} = await import('../../web/js/workspace/vocabulary-state.mjs');
assert.equal(categoryRowText({value: 'L', prompt_count: 3}), '3 positive prompts');
assert.equal(categoryRowText({value: 'L', prompt_count: 1, image_count: 0}), '1 positive prompt');
assert.equal(categoryRowText({value: 'L', prompt_count: 3, image_count: 2}), '3 positive prompts · 2 images');
assert.equal(categoryRowText({value: 'S', prompt_count: 0, image_count: 1}), '1 image');
assert.equal(categoryBrowseTarget({value: 'L', prompt_count: 3, image_count: 2}), 'library');
assert.equal(categoryBrowseTarget({value: 'S', prompt_count: 0, image_count: 2}), 'images');
const impact = {positive_prompts: 3, other_prompts: 0, negative_variants: 0, archived_negative_variants: 0, images: 2};
assert.equal(categoryPreviewText({source: 'Landscapes', target: 'Landscape', target_exists: false, impact}),
'“Landscapes” is used by 3 positive prompts and 2 images. Apply renames it to “Landscape”.');
});
+1 -44
View File
@@ -29,7 +29,6 @@ except ImportError:
model_gate = None
from py import config as config_module
from py.autotag import MissingBackendPackages, package_fields
AutotagConfig = getattr(config_module, "AutotagConfig", None)
@@ -38,11 +37,6 @@ WAIT = 30.0
WD14 = {"general_threshold": 0.35, "character_threshold": 0.85}
LLM = {"instructions": "Generate tags."}
UNAVAILABLE = "Download it in System › Auto-tagging."
NEEDS_LLAMA = (
"GGUF JoyCaption needs llama-cpp-python. Install it into ComfyUI's Python: "
'pip install llama-cpp-python. For a CUDA build: CMAKE_ARGS="-DGGML_CUDA=on" '
"pip install llama-cpp-python"
)
SCORED = [("sandstone", 0.98), ("desert", 0.96)] + [
(f"tag_{i:03d}", round(0.9 - i * 0.005, 4)) for i in range(150)
@@ -65,8 +59,6 @@ class FakeTagger:
"hf": {},
}
self.downloaded = {name: True for name in self.models_config}
# pip names of the backend packages that cannot be imported (#424)
self.missing = {name: [] for name in self.models_config}
self.loaded = None
self.load_calls = []
self.unload_calls = 0
@@ -79,11 +71,7 @@ class FakeTagger:
def get_models_status(self):
return {
name: {
"downloaded": self.downloaded[name],
"model_path": None,
**package_fields(name, self.missing[name]),
}
name: {"downloaded": self.downloaded[name], "model_path": None}
for name in self.models_config
}
@@ -327,37 +315,6 @@ class TestModelAvailability(SuggestTestCase):
self.assertEqual(self.tagger.calls, [])
self.assertIsNone(model_gate.holder())
async def test_missing_backend_package_is_refused_before_the_model(self):
observed = await self.observed()
self.tagger.missing["gguf"] = ["llama-cpp-python"]
status, body = await self.suggest(self.suggest_body(observed, "gguf"))
self.assertEqual((status, body["code"]), (409, "model-unavailable"))
self.assertEqual(body["success"], False)
self.assertEqual(body["error"], NEEDS_LLAMA)
self.assertEqual(self.tagger.load_calls, [])
self.assertIsNone(model_gate.holder())
# Not downloaded still reads as before, whatever the packages.
self.tagger.downloaded["gguf"] = False
status, body = await self.suggest(self.suggest_body(observed, "gguf"))
self.assertIn("is not downloaded", body["error"])
async def test_a_package_lost_after_the_check_is_the_same_409_without_a_traceback(
self,
):
observed = await self.observed()
self.tagger.load_error = MissingBackendPackages("gguf", ["llama-cpp-python"])
with self.assertNoLogs(self.api.logger, level="ERROR"):
status, body = await self.suggest(self.suggest_body(observed, "gguf"))
self.assertEqual((status, body["code"]), (409, "model-unavailable"))
self.assertEqual(body["error"], NEEDS_LLAMA)
self.assertIsNone(model_gate.holder())
entry = self.api._captures._entries[observed["capture"]["token"]]
self.assertEqual(entry.uses, 0)
async def test_load_failure_is_a_generic_error_and_frees_everything(self):
observed = await self.observed()
self.tagger.load_error = RuntimeError(f"cannot load {self.output_dir}")
-24
View File
@@ -73,7 +73,6 @@ class FakeAutotagService:
self.status_calls = 0
self.status_threads = []
self.status_paths = {}
self.status_extra = {}
self.suggest_calls = 0
def get_models_status(self):
@@ -83,7 +82,6 @@ class FakeAutotagService:
name: {
"downloaded": self.downloaded,
"model_path": self.status_paths.get(name),
**self.status_extra.get(name, {}),
}
for name in self.models_config
}
@@ -1113,24 +1111,6 @@ class TestModelStatus(AutotagAPITestCase):
leaked = [text for text in _strings(data) if _absolute(text)]
self.assertEqual(leaked, [])
async def test_models_status_names_missing_backend_packages(self):
fields = {
"missing_packages": ["llama-cpp-python"],
"install_command": "pip install llama-cpp-python",
"install_note": "For a CUDA build: ...",
}
self.service.status_extra = {"gguf": fields}
models = (await self._models())["models"]
self.assertTrue(models["gguf"]["downloaded"])
self.assertEqual(
{key: models["gguf"][key] for key in fields},
fields,
)
self.assertEqual(
{key: models["wd14-vit"][key] for key in fields},
{"missing_packages": [], "install_command": None, "install_note": None},
)
async def test_models_status_adds_label_size_label_and_loaded(self):
self.service._loaded_type = "wd14-vit"
data = await self._models()
@@ -1689,10 +1669,6 @@ class TestTagOutputImagesPreview(BatchJobTestCase):
"label": "WD14 ViT",
"size_label": "~400 MB",
"downloaded": True,
# #424: the backend packages it cannot import, none here.
"missing_packages": [],
"install_command": None,
"install_note": None,
},
)
self.assertEqual(skip["existing"], "skip")
-35
View File
@@ -264,41 +264,6 @@ class TestRestoreRoute(BackupRestoreTestCase):
texts = sorted(p["text"] for p in self.api.db.search_prompts())
self.assertEqual(texts, ["restored a", "restored b"])
async def test_restore_round_trip_keeps_tags_and_image_links(self):
"""A downloaded backup brings back every prompt with its tags and images.
The legacy admin page's Restore form left for System (#330); this keeps
its browser test's data check at the route.
"""
db = self.api.db
originals = {}
for text, tags, image in (
("round trip cat", ["cats", "pets"], "cat.png"),
("round trip dog", ["dogs"], "dog.png"),
):
prompt_id = db.save_prompt(
text=text, tags=tags, prompt_hash=generate_prompt_hash(text)
)
path = self._write(image, b"\x89PNG\r\n\x1a\n")
self.assertTrue(db.link_image_to_prompt(str(prompt_id), path))
originals[text] = (prompt_id, sorted(tags), [path])
resp = await self.client.get("/prompt_manager/backup")
self.assertEqual(resp.status, 200)
backup = self._write("downloaded.db", await resp.read())
self.assertTrue(db.delete_prompt(originals["round trip dog"][0]))
resp = await self._restore(backup)
self.assertEqual(resp.status, 200, await resp.text())
restored = {p["text"]: p for p in self.api.db.search_prompts()}
self.assertEqual(sorted(restored), sorted(originals))
for text, (prompt_id, tags, paths) in originals.items():
self.assertEqual(restored[text]["id"], prompt_id)
self.assertEqual(sorted(restored[text]["tags"]), tags)
images = self.api.db.get_prompt_images(str(prompt_id))
self.assertEqual([image["image_path"] for image in images], paths)
async def test_restore_leaves_no_temp_files_behind(self):
upload = self._make_db_file(["restored"])
created = []
-601
View File
@@ -1,601 +0,0 @@
"""Collections (#331): the routes under /prompt_manager/collections."""
import json
import ntpath
import sqlite3
from unittest.mock import patch
try:
from test_api_images import ImageAPITestCase, make_png
from open_handles import assert_closed
except ImportError:
from tests.test_api_images import ImageAPITestCase, make_png
from tests.open_handles import assert_closed
BASE = "/prompt_manager/collections"
ITEMS = BASE + "/items"
MEMBERSHIP = BASE + "/membership"
SOURCES_CHANGED = "The sources changed. Refresh the images and try again."
class CollectionRoutesTestCase(ImageAPITestCase):
async def tearDownAsync(self):
self.api.db.close_all()
assert_closed(self, self._temp_db.name, self._output_tmp.name)
await super().tearDownAsync()
def make(self, *names, root=None):
for name in names:
make_png((root or self.output_dir) / name)
def scope(self, index=0):
roots = list(self.api._get_all_output_dirs())
return self.api._gallery_root_scope(roots[index], index)
def item(self, path, index=0, scope=None):
return {
"root_index": index,
"root_scope": scope or self.scope(index),
"path": path,
}
def two_sources(self):
second = self.comfy_dir / "second"
second.mkdir()
self.api._get_all_output_dirs = lambda: [self.output_dir, second]
return second
async def call(self, method, url, body=None, status=200, **params):
response = await self.client.request(method, url, json=body, params=params)
text = await response.text()
self.assertEqual(response.status, status, text)
data = json.loads(text)
self.assertNotIn(self.comfy_dir.as_posix(), text)
self.assertNotIn(str(self.comfy_dir).replace("\\", "\\\\"), text)
return data
async def add(self, status=200, **body):
return await self.call("POST", ITEMS, body, status)
async def listed(self):
return (await self.call("GET", BASE))["collections"]
class TestCollectionRoutes(CollectionRoutesTestCase):
async def test_each_route_succeeds(self):
self.make("a.png", "b.png", "c.png")
self.assertEqual(
await self.call("GET", BASE), {"success": True, "collections": []}
)
made = await self.add(
name=" LoRa testing ", items=[self.item("a.png"), self.item("b.png")]
)
grids = made["collection"]
self.assertEqual(grids["name"], "LoRa testing")
self.assertEqual(
made,
{
"success": True,
"collection": grids,
"created": True,
"added": 2,
"already": 0,
"not_found": 0,
},
)
again = await self.add(
collection_id=grids["id"], items=[self.item("b.png"), self.item("c.png")]
)
self.assertEqual(
(again["created"], again["added"], again["already"]), (False, 1, 1)
)
self.assertEqual(
await self.listed(),
[{"id": grids["id"], "name": "LoRa testing", "count": 3, "missing": 0}],
)
membership = await self.call(
"GET", MEMBERSHIP, root_index="0", root_scope=self.scope(), path="a.png"
)
self.assertEqual(membership, {"success": True, "collection_ids": [grids["id"]]})
removed = await self.call(
"DELETE",
ITEMS,
{"collection_id": grids["id"], "items": [self.item("a.png")]},
)
self.assertEqual(removed, {"success": True, "collection": grids, "removed": 1})
renamed = await self.call(
"PATCH", f"{BASE}/{grids['id']}", {"name": "LoRA grids"}
)
self.assertEqual(
renamed,
{"success": True, "collection": {"id": grids["id"], "name": "LoRA grids"}},
)
deleted = await self.call("DELETE", f"{BASE}/{grids['id']}")
self.assertEqual(deleted, {"success": True, "deleted": True})
self.assertEqual(await self.listed(), [])
# Files are never touched.
self.assertTrue((self.output_dir / "a.png").is_file())
async def test_names_are_unique_ignoring_case(self):
self.make("a.png", "b.png")
first = (await self.add(name="LoRa testing", items=[self.item("a.png")]))[
"collection"
]
second = await self.add(name="lora TESTING", items=[self.item("b.png")])
self.assertEqual(second["collection"], first)
self.assertFalse(second["created"])
self.assertEqual(len(await self.listed()), 1)
async def test_bad_requests_are_400(self):
self.make("a.png")
made = await self.add(name="Set", items=[self.item("a.png")])
cid = made["collection"]["id"]
good = [self.item("a.png")]
for body in (
{"items": good},
{"name": "x", "collection_id": cid, "items": good},
{"name": "x"},
{"name": "x", "items": good, "extra": 1},
{"collection_id": "1", "items": good},
{"collection_id": True, "items": good},
{"collection_id": 0, "items": good},
{"collection_id": 10**10, "items": good},
{"name": "", "items": good},
{"name": "x" * 101, "items": good},
{"name": "bell\x07", "items": good},
{"name": 5, "items": good},
{"name": "x", "items": []},
{"name": "x", "items": "a.png"},
{"name": "x", "items": [self.item("a.png")] * 501},
{"name": "x", "items": [{"root_index": 0, "path": "a.png"}]},
{"name": "x", "items": [{**self.item("a.png"), "root_index": "0"}]},
{"name": "x", "items": [{**self.item("a.png"), "root_scope": "xyz"}]},
):
with self.subTest(body=body):
data = await self.add(400, **body)
self.assertFalse(data["success"])
for body in ({"items": good}, {"collection_id": cid}):
with self.subTest(remove=body):
await self.call("DELETE", ITEMS, body, 400)
for raw in ("abc", "0", "1234567890", "-1", "1.0"):
with self.subTest(path_id=raw):
await self.call("PATCH", f"{BASE}/{raw}", {"name": "x"}, 400)
await self.call("DELETE", f"{BASE}/{raw}", None, 400)
await self.call("PATCH", f"{BASE}/{cid}", {"name": ""}, 400)
await self.call("PATCH", f"{BASE}/{cid}", {"label": "x"}, 400)
for params in (
{"root_index": "x", "root_scope": self.scope(), "path": "a.png"},
{"root_index": "0", "path": "a.png"},
{"root_index": "0", "root_scope": "nope", "path": "a.png"},
):
with self.subTest(membership=params):
await self.call("GET", MEMBERSHIP, status=400, **params)
self.assertEqual((await self.listed())[0]["count"], 1)
async def test_unsafe_paths_are_403(self):
self.make("a.png")
for path in (
"../a.png",
"sub/../../a.png",
"C:/a.png",
"a\\b.png",
"/a.png",
"notes.txt",
):
with self.subTest(path=path):
await self.add(403, name="Set", items=[self.item(path)])
await self.call(
"GET",
MEMBERSHIP,
status=403,
root_index="0",
root_scope=self.scope(),
path=path,
)
self.assertEqual(await self.listed(), [])
async def test_a_body_over_the_cap_is_413(self):
response = await self.client.post(
ITEMS, data=b'{"name":"x","items":[' + b" " * (1024 * 1024 + 1) + b"]}"
)
self.assertEqual(response.status, 413)
async def test_missing_collection_and_files_are_404(self):
self.make("a.png")
data = await self.add(404, collection_id=999, items=[self.item("a.png")])
self.assertEqual(data["code"], "collection-not-found")
data = await self.call(
"DELETE", ITEMS, {"collection_id": 999, "items": [self.item("a.png")]}, 404
)
self.assertEqual(data["code"], "collection-not-found")
data = await self.call("PATCH", f"{BASE}/999", {"name": "x"}, 404)
self.assertEqual(data["code"], "collection-not-found")
data = await self.call("DELETE", f"{BASE}/999", None, 404)
self.assertEqual(data["code"], "collection-not-found")
data = await self.add(404, name="New", items=[self.item("never.png")])
self.assertEqual(data["code"], "files-not-found")
self.assertEqual(await self.listed(), [])
async def test_unlisted_files_count_as_not_found(self):
self.make("a.png")
made = await self.add(
name="Set", items=[self.item("a.png"), self.item("later.png")]
)
self.assertEqual((made["added"], made["not_found"]), (1, 1))
again = await self.add(
collection_id=made["collection"]["id"],
items=[self.item("a.png"), self.item("later.png")],
)
self.assertEqual(
(again["added"], again["already"], again["not_found"]), (0, 1, 1)
)
async def test_name_taken_is_409(self):
self.make("a.png")
first = (await self.add(name="First", items=[self.item("a.png")]))["collection"]
second = (await self.add(name="Second", items=[self.item("a.png")]))[
"collection"
]
data = await self.call(
"PATCH", f"{BASE}/{second['id']}", {"name": "FIRST"}, 409
)
self.assertEqual(
(data["code"], data["collection_id"]), ("name-taken", first["id"])
)
renamed = await self.call("PATCH", f"{BASE}/{second['id']}", {"name": "SECOND"})
self.assertEqual(renamed["collection"]["name"], "SECOND")
async def test_stale_scope_is_409_after_reordering_sources(self):
second = self.two_sources()
self.make("a.png")
self.make("b.png", root=second)
stale = self.item("b.png", 1)
made = await self.add(name="Set", items=[self.item("a.png")])
cid = made["collection"]["id"]
self.api._get_all_output_dirs = lambda: [second, self.output_dir]
for method, body in (
("POST", {"collection_id": cid, "items": [self.item("b.png", 0), stale]}),
("DELETE", {"collection_id": cid, "items": [stale]}),
):
with self.subTest(method=method):
data = await self.call(method, ITEMS, body, 409)
self.assertEqual(data["code"], "source-changed")
self.assertEqual(data["error"], SOURCES_CHANGED)
data = await self.call(
"GET",
MEMBERSHIP,
status=409,
root_index="1",
root_scope=stale["root_scope"],
path="b.png",
)
self.assertEqual(data["code"], "source-changed")
# Nothing was written by the refused add; the member follows its folder.
listed = await self.listed()
self.assertEqual((listed[0]["count"], listed[0]["missing"]), (1, 0))
membership = await self.call(
"GET", MEMBERSHIP, root_index="1", root_scope=self.scope(1), path="a.png"
)
self.assertEqual(membership["collection_ids"], [cid])
async def test_counts_exclude_a_missing_member(self):
second = self.two_sources()
self.make("a.png", "b.png")
self.make("c.png", root=second)
made = await self.add(
name="Grids",
items=[self.item("a.png"), self.item("b.png"), self.item("c.png", 1)],
)
other = await self.add(name="Other", items=[self.item("c.png", 1)])
(self.output_dir / "b.png").unlink()
self.api.invalidate_gallery_cache()
listed = await self.listed()
self.assertEqual(
listed,
[
{
"id": made["collection"]["id"],
"name": "Grids",
"count": 2,
"missing": 1,
},
{
"id": other["collection"]["id"],
"name": "Other",
"count": 1,
"missing": 0,
},
],
)
# The member is removable although it is no longer listed.
removed = await self.call(
"DELETE",
ITEMS,
{"collection_id": made["collection"]["id"], "items": [self.item("b.png")]},
)
self.assertEqual(removed["removed"], 1)
async def test_counts_are_null_when_a_source_scan_fails(self):
self.make("a.png")
await self.add(name="Set", items=[self.item("a.png")])
async def broken(_root):
raise OSError("scan failed at /secret/place")
with patch.object(self.api, "_get_gallery_files", side_effect=broken):
data = await self.call("GET", BASE)
self.assertEqual(
data["collections"][0] | {"id": 0},
{"id": 0, "name": "Set", "count": None, "missing": None},
)
self.assertNotIn("secret", json.dumps(data))
async def test_windows_case_spellings_are_one_member(self):
self.make("Sub/A.png")
with patch("py.api.images._path_flavor", ntpath):
made = await self.add(
name="Set", items=[self.item("Sub/A.png"), self.item("SUB/a.PNG")]
)
self.assertEqual((made["added"], made["already"]), (1, 0))
(self.output_dir / "Sub" / "A.png").rename(
self.output_dir / "Sub" / "a.png"
)
(self.output_dir / "Sub").rename(self.output_dir / "sub")
self.api.invalidate_gallery_cache()
listed = await self.listed()
self.assertEqual((listed[0]["count"], listed[0]["missing"]), (1, 0))
membership = await self.call(
"GET",
MEMBERSHIP,
root_index="0",
root_scope=self.scope(),
path="sub/a.png",
)
self.assertEqual(membership["collection_ids"], [made["collection"]["id"]])
response = await self.client.get(
"/prompt_manager/images/output",
params={
"mode": "workspace",
"collection": str(made["collection"]["id"]),
},
)
page = await response.json()
self.assertEqual([i["path"] for i in page["images"]], ["sub/a.png"])
async def test_posix_case_distinct_files_stay_two_members(self):
self.make("A.png", "a.png")
if len(list(self.output_dir.iterdir())) != 2:
self.skipTest("case-insensitive filesystem")
made = await self.add(
name="Set", items=[self.item("A.png"), self.item("a.png")]
)
self.assertEqual(made["added"], 2)
async def test_a_500_never_carries_the_exception_text(self):
self.make("a.png")
made = await self.add(name="Set", items=[self.item("a.png")])
cid = made["collection"]["id"]
def broken(*_args, **_kwargs):
raise sqlite3.OperationalError("disk I/O error at /secret/prompts.db")
cases = (
(
"list_collections",
"GET",
BASE,
None,
{},
"Collections could not be read.",
),
(
"file_collections",
"GET",
MEMBERSHIP,
None,
{"root_index": "0", "root_scope": self.scope(), "path": "a.png"},
"Collections could not be read.",
),
(
"add_to_collection",
"POST",
ITEMS,
{"collection_id": cid, "items": [self.item("a.png")]},
{},
"The collection could not be changed.",
),
(
"remove_from_collection",
"DELETE",
ITEMS,
{"collection_id": cid, "items": [self.item("a.png")]},
{},
"The collection could not be changed.",
),
(
"rename_collection",
"PATCH",
f"{BASE}/{cid}",
{"name": "x"},
{},
"The collection could not be changed.",
),
(
"delete_collection",
"DELETE",
f"{BASE}/{cid}",
None,
{},
"The collection could not be changed.",
),
)
for method_name, method, url, body, params, message in cases:
with self.subTest(method=method_name):
with patch.object(self.api.db, method_name, side_effect=broken):
data = await self.call(method, url, body, 500, **params)
self.assertEqual(data, {"success": False, "error": message})
class TestCollectionListingFilter(CollectionRoutesTestCase):
"""#392's "all N matching" filter carries ``collection`` like the listing."""
async def test_a_filter_with_a_collection_resolves_only_members(self):
second = self.two_sources()
self.make("a.png", "b.png")
self.make("c.png", root=second)
made = await self.add(
name="Set", items=[self.item("a.png"), self.item("c.png", 1)]
)
cid = str(made["collection"]["id"])
selection = await self.api.resolve_listing_filter({"collection": cid})
self.assertEqual(
sorted((index, path) for index, _scope, path in selection.targets),
[(0, "a.png"), (1, "c.png")],
)
selection = await self.api.resolve_listing_filter(
{"collection": cid, "root": "1", "root_scope": self.scope(1)}
)
self.assertEqual([t[2] for t in selection.targets], ["c.png"])
for bad in ("0", "x", "1234567890"):
with self.subTest(bad=bad):
with self.assertRaises(ValueError):
from py.api.images import parse_listing_filter
parse_listing_filter({"collection": bad})
async def test_bulk_category_acts_on_collection_members_only(self):
self.make("a.png", "b.png")
inside = self._save_prompt("Inside", category="Old")
outside = self._save_prompt("Outside", category="Old")
self.api.db.link_image_to_prompt(inside, str(self.output_dir / "a.png"))
self.api.db.link_image_to_prompt(outside, str(self.output_dir / "b.png"))
made = await self.add(name="Set", items=[self.item("a.png")])
cid = str(made["collection"]["id"])
data = await self.call(
"POST",
"/prompt_manager/bulk/category",
{"images_filter": {"collection": cid}, "category": "New"},
)
self.assertEqual((data["updated_count"], data["prompt_count"]), (1, 1))
self.assertEqual(self.api.db.get_prompt_by_id(inside)["category"], "New")
self.assertEqual(self.api.db.get_prompt_by_id(outside)["category"], "Old")
async def test_a_deleted_collection_in_a_bulk_filter_is_a_404(self):
self.make("a.png")
data = await self.call(
"POST",
"/prompt_manager/bulk/category",
{"images_filter": {"collection": "77"}, "category": "New"},
status=404,
)
self.assertEqual(data["code"], "collection-not-found")
with self.assertRaises(Exception) as caught:
await self.api.resolve_listing_filter({"collection": "77"})
self.assertEqual(type(caught.exception).__name__, "CollectionMissing")
class TestReviewFindings(CollectionRoutesTestCase):
"""PR #400 review: loop work, counts snapshot, id and query rules."""
async def test_no_realpath_runs_on_the_event_loop(self):
import os
import threading
names = [f"img{n}.png" for n in range(20)]
self.make(*names)
await self.call("GET", BASE) # warm the listing cache
loop_thread = threading.current_thread()
on_loop = []
real = os.path.realpath
def watched(*args, **kwargs):
if threading.current_thread() is loop_thread:
on_loop.append(args[0])
return real(*args, **kwargs)
items = [self.item(name) for name in names]
cid = None
with patch("os.path.realpath", side_effect=watched):
made = await self.add(name="Set", items=items)
cid = made["collection"]["id"]
await self.call("DELETE", ITEMS, {"collection_id": cid, "items": items[:5]})
await self.call(
"GET",
MEMBERSHIP,
root_index="0",
root_scope=items[0]["root_scope"],
path="img9.png",
)
await self.call("GET", BASE)
self.assertEqual(on_loop, [])
async def test_missing_is_never_negative(self):
self.make("a.png", "b.png")
made = await self.add(
name="Set", items=[self.item("a.png"), self.item("b.png")]
)
cid = made["collection"]["id"]
real = self.api.db.collection_count_snapshot
def raced(root_keys):
members, stored = real(root_keys)
return members, {cid: 1} # a remove landed between two reads
with patch.object(self.api.db, "collection_count_snapshot", side_effect=raced):
listed = await self.listed()
self.assertEqual((listed[0]["count"], listed[0]["missing"]), (2, 0))
async def test_path_ids_follow_the_listing_rule(self):
self.make("a.png")
made = await self.add(name="Set", items=[self.item("a.png")])
cid = made["collection"]["id"]
for raw in ("0", "007", f"0{cid}"):
with self.subTest(raw=raw):
await self.call("PATCH", f"{BASE}/{raw}", {"name": "x"}, 400)
await self.call("DELETE", f"{BASE}/{raw}", None, 400)
self.assertEqual(len(await self.listed()), 1)
async def test_membership_refuses_repeated_query_keys(self):
self.make("a.png", "b.png")
response = await self.client.get(
MEMBERSHIP
+ f"?root_index=0&root_scope={self.scope()}&path=a.png&path=b.png"
)
self.assertEqual(response.status, 400)
response = await self.client.get(
MEMBERSHIP
+ f"?root_index=0&root_index=0&root_scope={self.scope()}&path=a.png"
)
self.assertEqual(response.status, 400)
async def test_deleting_a_collection_leaves_everything_else(self):
from py.api.images import _favorite_root_key
try:
from tests.test_annotation_schema import insert_group_rows, table_rows
except ImportError:
from test_annotation_schema import insert_group_rows, table_rows
self.make("a.png")
db = self.api.db
db.set_image_favorite(_favorite_root_key(self.output_dir), "a.png")
prompt = self._save_prompt("Linked")
db.link_image_to_prompt(prompt, str(self.output_dir / "a.png"))
with db.model.get_connection() as conn:
insert_group_rows(conn, locator=str(self.output_dir / "a.png"))
made = await self.add(name="Set", items=[self.item("a.png")])
tables = (
"image_favorites",
"generated_images",
"prompts",
"tags",
"media_annotation_files",
"media_annotation_bytes",
"media_image_tags",
)
before = table_rows(db.model.get_connection(), tables)
file_bytes = (self.output_dir / "a.png").read_bytes()
await self.call("DELETE", f"{BASE}/{made['collection']['id']}")
self.assertEqual(table_rows(db.model.get_connection(), tables), before)
self.assertEqual((self.output_dir / "a.png").read_bytes(), file_bytes)
self.assertTrue(all(before[t] for t in tables))
-36
View File
@@ -693,39 +693,6 @@ class TestCleanupJob(DuplicateFilesTestCase):
"Moved 2 database links to the kept copies", "\n".join(cleanup.log)
)
async def test_duplicate_removal_moves_collection_memberships(self):
self.trio()
db = self.api.db
root_key = duplicate_files.favorite_root_key(self.output_dir)
grids = db.add_to_collection(name="LoRA grids", items=[(root_key, "b.png")])
grids = grids["collection"]["id"]
other = db.add_to_collection(
name="Other", items=[(root_key, "sub/c.png"), (root_key, "d.png")]
)["collection"]["id"]
db.add_to_collection(collection_id=grids, items=[(root_key, "sub/c.png")])
_plan, cleanup = await self.remove_all_but("a.png")
# The union: a.png joins both; c.png's second grids row was a repeat.
self.assertEqual(db.collection_members(grids, root_key), {"a.png"})
self.assertEqual(db.collection_members(other, root_key), {"a.png", "d.png"})
self.assertEqual(sorted(db.file_collections(root_key, "a.png")), [grids, other])
self.assertEqual(db.file_collections(root_key, "b.png"), [])
self.assertEqual(db.file_collections(root_key, "sub/c.png"), [])
self.assertIn(
"Moved 2 collection memberships to the kept copies", "\n".join(cleanup.log)
)
def test_one_collection_membership_is_singular(self):
job = type(
"Job", (), {"log": [], "log_line": lambda s, line: s.log.append(line)}
)()
totals = dict.fromkeys(
("moved", "merged", "favorites", "tags", "review", "collections"), 0
)
duplicate_files.DuplicateFilesMixin._log_moved_records(
job, {**totals, "collections": 1}
)
self.assertEqual(job.log, ["Moved 1 collection membership to the kept copies"])
async def test_a_failed_removal_leaves_its_links_on_the_kept_copy(self):
self.trio()
prompt = self.save("Stays on c")
@@ -882,11 +849,8 @@ class TestLinkQueries(DuplicateFilesTestCase):
with self.subTest(sql=sql):
self.assertNotIn("SCAN generated_images", plan)
self.assertNotIn("SCAN image_favorites", plan)
self.assertNotIn("SCAN image_collection_items", plan)
finally:
conn.close()
statements = duplicate_files.query_plan_sql()
self.assertTrue(any("image_collection_items" in sql for sql in statements))
class TestReviewHelpers(DuplicateFilesTestCase):
-87
View File
@@ -21,8 +21,6 @@ from tests.test_api_images import ImageAPITestCase, make_png
from tests.test_api_prompt_inspector import graph, png
from tests.test_execution_context_database import bundle as context_bundle
from tests.test_execution_history_database import JOB, SESSION
from tests.saver_anchor_graphs import BRANCH_TEXTS, shared_prefix
from tests.test_recorded_saver_anchor import LLM_TEXT, text_generate_run
from utils.execution_contexts import ContextBundle
from utils.execution_history_events import make_finish, make_start
from utils.execution_outputs import OutputBundle, OutputReference, snapshot_output_roots
@@ -1507,88 +1505,3 @@ class HistoryOutputAPITests(ImageAPITestCase):
body = await response.json()
self.assertEqual(response.status, 200, body)
self.assertEqual(body["prompts"]["positive"]["text"], "Readable portrait")
async def recorded_shared_prefix_run(self):
"""#386: the dataset shape, branch 3's saver wrote the file in a run."""
path = png(self.output_dir / "IMG_GEN_00371_.png", prompt=shared_prefix())
# make_start/make_finish observe the run at 0 ms: the file was written then.
os.utime(path, ns=(1_000_000, 1_000_000))
start = self.capture(
[self.ref(node="33", filename=path.name), self.ref("43", 1, "other.png")]
)
return start
async def test_both_inspect_routes_name_the_branch_recorded_history_saved(self):
start = await self.recorded_shared_prefix_run()
_, body = await self.outputs(start)
descriptor = next(
item["current_candidate"]["media"]
for item in body["items"]
if item["occurrence"]["output_node_id"] == "33"
)
response = await self.current(descriptor["inspection_url"])
recorded = await response.json()
self.assertEqual(response.status, 200, recorded)
response = await self.client.get(
"/prompt_manager/images/inspect/IMG_GEN_00371_.png", params={"root": "0"}
)
images = await response.json()
self.assertEqual(response.status, 200, images)
for result in (recorded, images):
self.assertEqual(result["resolution"]["state"], "resolved")
self.assertEqual(
result["prompts"]["positive"],
{
"status": "resolved",
"text": BRANCH_TEXTS[2],
"source": "recorded-run",
},
)
async def test_a_file_rewritten_after_its_run_keeps_several_prompts(self):
await self.recorded_shared_prefix_run()
os.utime(self.output_dir / "IMG_GEN_00371_.png") # written now: not that run
response = await self.client.get(
"/prompt_manager/images/inspect/IMG_GEN_00371_.png", params={"root": "0"}
)
body = await response.json()
self.assertEqual(response.status, 200, body)
self.assertEqual(body["resolution"]["state"], "ambiguous")
self.assertEqual(body["prompts"]["positive"]["status"], "ambiguous")
async def test_a_failing_history_lookup_judges_the_file_alone(self):
await self.recorded_shared_prefix_run()
with (
patch(
"database.run_attribution.recorded_output",
side_effect=sqlite3.OperationalError("locked"),
),
self.assertLogs("prompt_manager.run_attribution", "WARNING"),
):
response = await self.client.get(
"/prompt_manager/images/inspect/IMG_GEN_00371_.png",
params={"root": "0"},
)
body = await response.json()
self.assertEqual(response.status, 200, body)
self.assertEqual(body["resolution"]["state"], "ambiguous")
async def test_a_language_model_text_shows_what_promptmanager_recorded(self):
path = png(self.output_dir / "moss_00026_.png", prompt=text_generate_run())
os.utime(path, ns=(1_000_000, 1_000_000))
start = self.capture([self.ref(node="9", filename=path.name)])
self.api.db.write_execution_contexts(
start.run_id,
start.storage_epoch,
context_bundle("", LLM_TEXT, anchor="7"),
)
response = await self.client.get(
"/prompt_manager/images/inspect/moss_00026_.png", params={"root": "0"}
)
body = await response.json()
self.assertEqual(response.status, 200, body)
self.assertEqual(
body["prompts"]["positive"],
{"status": "resolved", "text": LLM_TEXT, "source": "execution"},
)
self.assertEqual(body["prompts"]["negative"]["text"], "")
-375
View File
@@ -1,375 +0,0 @@
"""Image category routes (#332): PUT/DELETE /images/category, GET /images/categories."""
import json
import ntpath
import sqlite3
from unittest.mock import patch
try:
from test_api_images import ImageAPITestCase, make_png
from open_handles import assert_closed
from test_image_categories import category_checksum
except ImportError:
from tests.test_api_images import ImageAPITestCase, make_png
from tests.open_handles import assert_closed
from tests.test_image_categories import category_checksum
from database.image_categories import MAX_KEY_LENGTH, TABLE
from py.api.images import _favorite_root_key, image_category_key
CATEGORY = "/prompt_manager/images/category"
CATEGORIES = "/prompt_manager/images/categories"
class ImageCategoryTestCase(ImageAPITestCase):
async def tearDownAsync(self):
self.api.db.close_all()
assert_closed(self, self._temp_db.name, self._output_tmp.name)
await super().tearDownAsync()
def target(self, path, index=0):
roots = self.api._get_all_output_dirs()
return {
"root_index": index,
"root_scope": self.api._gallery_root_scope(roots[index], index),
"path": path,
}
async def send(self, method, body, status=200):
response = await self.client.request(method, CATEGORY, json=body)
text = await response.text()
self.assertEqual(response.status, status, text)
return json.loads(text)
async def put(self, paths, category, status=200):
items = [p if isinstance(p, dict) else self.target(p) for p in paths]
return await self.send("PUT", {"items": items, "category": category}, status)
async def delete(self, paths, status=200):
items = [p if isinstance(p, dict) else self.target(p) for p in paths]
return await self.send("DELETE", {"items": items}, status)
async def listed(self, status=200):
response = await self.client.get(CATEGORIES)
text = await response.text()
self.assertEqual(response.status, status, text)
return json.loads(text)
def rows(self):
return [
tuple(row)
for row in self.api.db.model.get_connection().execute(
f"SELECT root_key, relative_path, category FROM {TABLE} "
"ORDER BY root_key, relative_path"
)
]
def key(self, relative, root=None):
roots = self.api._get_all_output_dirs()
return image_category_key(roots, (root or self.output_dir) / relative)
class TestSetAndClear(ImageCategoryTestCase):
async def test_put_sets_listed_and_counts_missing(self):
make_png(self.output_dir / "a.png")
make_png(self.output_dir / "sub" / "b.png")
data = await self.put(["a.png", "sub/b.png", "missing.png"], " Portraits ")
self.assertEqual(
data,
{
"success": True,
"category": "Portraits",
"updated": 2,
"unchanged": 0,
"not_found": 1,
},
)
root_key = _favorite_root_key(self.output_dir)
self.assertEqual(
self.rows(),
[(root_key, "a.png", "Portraits"), (root_key, "sub/b.png", "Portraits")],
)
data = await self.put(["a.png", "sub/b.png"], "Portraits")
self.assertEqual((data["updated"], data["unchanged"]), (0, 2))
data = await self.put(["a.png"], "Scans")
self.assertEqual((data["updated"], data["unchanged"]), (1, 0))
self.assertEqual(self.api.db.image_category(*self.key("a.png")), "Scans")
async def test_put_with_no_listed_file_is_404_and_writes_nothing(self):
data = await self.put(["missing.png", "gone/x.png"], "Scans", status=404)
self.assertEqual(data["code"], "files-not-found")
self.assertEqual(self.rows(), [])
async def test_duplicate_targets_are_written_and_counted_once(self):
make_png(self.output_dir / "a.png")
item = self.target("a.png")
data = await self.put([item, dict(item)], "Scans")
self.assertEqual((data["updated"], data["not_found"]), (1, 0))
async def test_delete_clears_listed_and_missing_files(self):
make_png(self.output_dir / "a.png")
await self.put(["a.png"], "Scans")
self.api.db.set_image_categories([self.key("gone.png")], "Scans")
data = await self.delete(["a.png", "gone.png", "never.png"])
self.assertEqual(data, {"success": True, "cleared": 2})
self.assertEqual(self.rows(), [])
self.assertEqual((await self.delete(["a.png"]))["cleared"], 0)
async def test_put_never_writes_prompts(self):
make_png(self.output_dir / "a.png")
make_png(self.output_dir / "b.png")
positive = self._save_prompt("castle at dusk", category="Landscapes")
self.api.db.link_image_to_prompt(positive, str(self.output_dir / "a.png"))
negative = self._save_prompt("blurry", category="Negatives")
self.api.db.save_prompt_pair("castle at dusk", "blurry", category="Recipes")
conn = self.api.db.model.get_connection()
before = category_checksum(conn)
links = conn.execute("SELECT * FROM generated_images").fetchall()
await self.put(["a.png", "b.png"], "Portraits")
await self.listed()
await self.delete(["a.png"])
self.assertEqual(category_checksum(conn), before)
self.assertEqual(
conn.execute("SELECT * FROM generated_images").fetchall(), links
)
self.assertIsNotNone(negative)
async def test_two_sources_in_one_request(self):
second = self.comfy_dir / "renders"
make_png(self.output_dir / "a.png")
make_png(second / "a.png")
self.api._get_all_output_dirs = lambda: [self.output_dir, second]
data = await self.put([self.target("a.png", 0), self.target("a.png", 1)], "X")
self.assertEqual(data["updated"], 2)
self.assertEqual(
{row[0] for row in self.rows()},
{_favorite_root_key(self.output_dir), _favorite_root_key(second)},
)
self.assertEqual((await self.delete([self.target("a.png", 1)]))["cleared"], 1)
async def test_nested_sources_key_by_the_deepest(self):
inner = self.output_dir / "nested"
make_png(inner / "a.png")
self.api._get_all_output_dirs = lambda: [self.output_dir, inner]
# Named through the outer source, stored under the inner one.
data = await self.put([self.target("nested/a.png", 0)], "Scans")
self.assertEqual(data["updated"], 1)
self.assertEqual(self.rows(), [(_favorite_root_key(inner), "a.png", "Scans")])
# The same file through the inner source is the same key.
data = await self.put([self.target("a.png", 1)], "Scans")
self.assertEqual(data["unchanged"], 1)
self.assertEqual((await self.listed())["categories"], ["Scans"])
self.assertEqual((await self.delete([self.target("a.png", 1)]))["cleared"], 1)
async def test_video_files_can_have_a_category(self):
(self.output_dir / "clip.mp4").write_bytes(b"\x00\x00\x00\x18ftypmp42")
data = await self.put(["clip.mp4"], "Clips")
self.assertEqual(data["updated"], 1)
async def test_windows_case_folds_the_key(self):
make_png(self.output_dir / "Sub" / "A.png")
with patch("py.api.images._path_flavor", ntpath):
data = await self.put(["Sub/A.png"], "Scans")
self.assertEqual(data["updated"], 1)
data = await self.put(["sub/a.PNG"], "Scans")
self.assertEqual(data["unchanged"], 1)
self.assertEqual(self.rows()[0][1], "sub/a.png")
self.assertEqual((await self.delete(["SUB/A.png"]))["cleared"], 1)
async def test_a_key_beyond_the_stored_bound_is_403(self):
make_png(self.output_dir / "a.png")
too_long = "/" + "r" * MAX_KEY_LENGTH
with patch("py.api.images._favorite_root_key", return_value=too_long):
data = await self.put(["a.png"], "Scans", status=403)
self.assertNotIn(too_long, json.dumps(data))
await self.delete(["a.png"], status=403)
self.assertEqual(self.rows(), [])
async def test_an_unexpected_failure_is_a_logged_500(self):
make_png(self.output_dir / "a.png")
def broken(*_args, **_kwargs):
raise sqlite3.OperationalError("secret detail")
self.api.db.set_image_categories = broken
self.api.db.clear_image_categories = broken
with self.assertLogs("prompt_manager", level="ERROR"):
data = await self.put(["a.png"], "Scans", status=500)
self.assertEqual(data["error"], "The category could not be changed.")
self.assertNotIn("secret", json.dumps(data))
with self.assertLogs("prompt_manager", level="ERROR"):
data = await self.delete(["a.png"], status=500)
self.assertNotIn("secret", json.dumps(data))
async def test_a_symlinked_output_folder_keys_like_its_favorites(self):
"""A detected output folder can be a symlink: keys use its real path."""
real = self.comfy_dir / "real-output"
make_png(real / "a.png")
link = self.comfy_dir / "linked-output"
self._make_symlink(real, link, target_is_directory=True)
self.api._get_all_output_dirs = lambda: [link]
data = await self.put(["a.png"], "Scans")
self.assertEqual(data["updated"], 1)
self.assertEqual(self.rows(), [(_favorite_root_key(link), "a.png", "Scans")])
self.assertEqual((await self.listed())["categories"], ["Scans"])
self.assertEqual((await self.delete(["a.png"]))["cleared"], 1)
async def test_lone_surrogates_are_400_not_500(self):
make_png(self.output_dir / "a.png")
await self.send(
"PUT", {"items": [self.target("a.png")], "category": "x\ud800"}, 400
)
await self.send("DELETE", {"items": [self.target("a\ud800.png")]}, 400)
self.assertEqual(self.rows(), [])
async def test_delete_category_never_reaches_delete_image(self):
make_png(self.output_dir / "a.png")
calls = []
async def recorded(request):
calls.append(request.match_info)
raise AssertionError("delete_image reached")
self.api.delete_image = recorded
await self.put(["a.png"], "Scans")
data = await self.delete(["a.png"])
self.assertEqual(data["cleared"], 1)
self.assertEqual(calls, [])
class TestRefusals(ImageCategoryTestCase):
async def test_malformed_bodies_are_400(self):
make_png(self.output_dir / "a.png")
item = self.target("a.png")
cases = [
{"items": [item]},
{"category": "x"},
{"items": [item], "category": "x", "extra": 1},
{"items": item, "category": "x"},
{"items": [], "category": "x"},
{"items": [item] * 501, "category": "x"},
{"items": [item], "category": 3},
{"items": [item], "category": None},
{"items": [item], "category": " \t "},
{"items": [item], "category": "x" * 101},
{"items": [item], "category": "a\x07b"},
{"items": [{**item, "root_index": "0"}], "category": "x"},
{"items": [{**item, "root_scope": "nothex"}], "category": "x"},
{"items": [{"root_index": 0, "path": "a.png"}], "category": "x"},
{"items": [{**item, "root_index": 1}], "category": "x"},
]
for body in cases:
with self.subTest(body=str(body)[:80]):
data = await self.send("PUT", body, status=400)
self.assertFalse(data["success"])
await self.send("DELETE", {"items": [item], "category": "x"}, status=400)
await self.send("DELETE", {"items": []}, status=400)
response = await self.client.request(
"PUT", CATEGORY, data="not json", headers={"Content-Type": "text/plain"}
)
self.assertEqual(response.status, 400)
self.assertEqual(self.rows(), [])
async def test_unsafe_paths_are_403(self):
(self.output_dir / "notes.txt").write_text("not media")
for path in ("../outside.png", "/a.png", "C:/a.png", "a\\b.png", "a/./b.png"):
with self.subTest(path=path):
await self.put([self.target(path)], "x", status=403)
data = await self.put([self.target("notes.txt")], "x", status=403)
self.assertEqual(data["error"], "Only media files can have a category")
await self.delete([self.target("notes.txt")], status=403)
async def test_a_stale_scope_is_409(self):
make_png(self.output_dir / "a.png")
item = {**self.target("a.png"), "root_scope": "0" * 64}
data = await self.put([item], "x", status=409)
self.assertEqual(data["code"], "source-changed")
data = await self.delete([item], status=409)
self.assertEqual(data["code"], "source-changed")
async def test_an_oversized_body_is_413(self):
response = await self.client.request(
"PUT",
CATEGORY,
data=b'{"items": [], "category": "' + b"x" * (2 * 1024 * 1024) + b'"}',
headers={"Content-Type": "application/json"},
)
self.assertEqual(response.status, 413)
class TestCategoryList(ImageCategoryTestCase):
async def test_lists_only_values_whose_file_is_listed(self):
make_png(self.output_dir / "a.png")
make_png(self.output_dir / "c.png")
await self.put(["a.png"], "B")
await self.put(["c.png"], "A")
self.api.db.set_image_categories([self.key("gone.png")], "Z")
self.assertEqual(
await self.listed(), {"success": True, "categories": ["A", "B"]}
)
async def test_empty_library_lists_nothing(self):
self.assertEqual((await self.listed())["categories"], [])
async def test_a_failed_read_is_500(self):
make_png(self.output_dir / "a.png")
def broken(*_args, **_kwargs):
raise sqlite3.OperationalError("secret detail")
self.api.db.own_categories_of = broken
with self.assertLogs("prompt_manager", level="ERROR"):
data = await self.listed(status=500)
self.assertEqual(data["error"], "Image categories could not be read.")
self.assertNotIn("secret", json.dumps(data))
class TestAllMatchingFilter(ImageCategoryTestCase):
"""#392 "Select all N matching": the listing filter, resolved now."""
async def test_filter_sets_and_clears_every_match(self):
for name in ("bulk/a.png", "bulk/b.png", "other/c.png"):
make_png(self.output_dir / name)
data = await self.send(
"PUT", {"filter": {"subfolder": "bulk"}, "category": "Scans"}
)
self.assertEqual((data["updated"], data["not_found"]), (2, 0))
self.assertEqual(
sorted(row[1] for row in self.rows()), ["bulk/a.png", "bulk/b.png"]
)
data = await self.send("DELETE", {"filter": {"subfolder": "bulk"}})
self.assertEqual(data["cleared"], 2)
data = await self.send(
"PUT", {"filter": {"subfolder": "none"}, "category": "Scans"}, 404
)
self.assertEqual(data["code"], "files-not-found")
self.assertEqual(
(await self.send("DELETE", {"filter": {"q": "zz"}}))["cleared"], 0
)
async def test_filter_refusals(self):
make_png(self.output_dir / "a.png")
item = self.target("a.png")
for body in (
{"filter": {}, "items": [item], "category": "x"},
{"filter": {"unknown": "1"}, "category": "x"},
{"filter": "q=a", "category": "x"},
{"filter": {"root": "0", "root_scope": "nothex"}, "category": "x"},
):
with self.subTest(body=body):
await self.send("PUT", body, 400)
data = await self.send(
"PUT",
{"filter": {"root": "0", "root_scope": "0" * 64}, "category": "x"},
409,
)
self.assertEqual(data["code"], "source-changed")
self.assertEqual(self.rows(), [])
async def test_more_than_500_matching_is_refused(self):
for number in range(501):
make_png(self.output_dir / f"many/{number:03}.png", size=(1, 1))
data = await self.send("PUT", {"filter": {}, "category": "x"}, 400)
self.assertEqual(data["code"], "too-many-images")
self.assertEqual(data["error"], "Select at most 500 images.")
self.assertEqual(self.rows(), [])
+2 -74
View File
@@ -62,16 +62,12 @@ class TestVocabularyAPI(ImageAPITestCase):
},
)
self.assertEqual(
categories["categories"],
[{"value": "Animal", "prompt_count": 1, "image_count": 0}],
categories["categories"], [{"value": "Animal", "prompt_count": 1}]
)
_, rest = await self.get(
"categories", limit=1, cursor=categories["next_cursor"]
)
self.assertEqual(
rest["categories"],
[{"value": "Pet", "prompt_count": 1, "image_count": 0}],
)
self.assertEqual(rest["categories"], [{"value": "Pet", "prompt_count": 1}])
self.assertFalse(rest["has_more"])
async def test_vocabulary_cursor_is_signed_epoch_scoped(self):
@@ -492,7 +488,6 @@ class TestVocabularyAPI(ImageAPITestCase):
"other_prompts": 0,
"negative_variants": 1,
"archived_negative_variants": 0,
"images": 0,
}
self.assertEqual(
preview,
@@ -533,73 +528,6 @@ class TestVocabularyAPI(ImageAPITestCase):
self.assertEqual(response.status, 409)
self.assertEqual(replay["code"], "vocabulary-conflict")
async def test_category_rows_report_images_and_image_only_values(self):
self._save_prompt("Owner", category="Prompt value")
_, options_before = await self.get_options()
self.api.db.set_image_categories(
[("root", "a.png"), ("root", "b.png")], "Prompt value"
)
self.api.db.set_image_categories([("root", "c.png")], "Image value")
response, first = await self.get("categories", limit=1)
self.assertEqual(response.status, 200)
self.assertEqual(
first["categories"],
[{"value": "Prompt value", "prompt_count": 1, "image_count": 2}],
)
self.assertEqual(first["total"], 2)
_, rest = await self.get("categories", limit=1, cursor=first["next_cursor"])
self.assertEqual(
rest["categories"],
[{"value": "Image value", "prompt_count": 0, "image_count": 1}],
)
self.assertFalse(rest["has_more"])
# The Prompt library's category list does not gain image values.
_, options_after = await self.get_options()
self.assertEqual(options_after, options_before)
self.assertEqual(options_after["categories"], ["Prompt value"])
async def get_options(self):
response = await self.client.get("/prompt_manager/library/options")
return response, await response.json()
async def test_category_routes_rename_image_only_value(self):
epoch = self.api.db.get_execution_epoch()
self._save_prompt("Bystander", category="Kept")
self.api.db.set_image_categories(
[("root", "a.png"), ("second", "b.png")], "Image value"
)
prompts_before = self.category_rows()
body = {"expected_epoch": epoch, "source": "Image value", "target": "Moved"}
response, preview = await self.send("POST", "categories/preview", body)
self.assertEqual(response.status, 200, preview)
impact = {
"positive_prompts": 0,
"other_prompts": 0,
"negative_variants": 0,
"archived_negative_variants": 0,
"images": 2,
}
self.assertEqual(preview["impact"], impact)
stale = preview["impact_token"]
self.api.db.set_image_categories([("root", "late.png")], "Image value")
response, refused = await self.send(
"POST", "categories/apply", {**body, "impact_token": stale}
)
self.assertEqual(response.status, 409)
self.assertEqual(refused["code"], "vocabulary-conflict")
self.assertEqual(self.api.db.image_category("root", "a.png"), "Image value")
_, preview = await self.send("POST", "categories/preview", body)
response, applied = await self.send(
"POST",
"categories/apply",
{**body, "impact_token": preview["impact_token"]},
)
self.assertEqual(response.status, 200, applied)
self.assertEqual(applied["impact"], {**impact, "images": 3})
for key in (("root", "a.png"), ("second", "b.png"), ("root", "late.png")):
self.assertEqual(self.api.db.image_category(*key), "Moved")
self.assertEqual(self.category_rows(), prompts_before)
async def test_preview_validation(self):
epoch = self.api.db.get_execution_epoch()
self._save_prompt("Owner", category="Scenery")
@@ -1,262 +0,0 @@
"""Images filters by the effective category (#332 MC4, MC5, DC6).
An image's effective category is its own category when it has one, else the
category its link supplies (the recipe's when set, else the positive's), else
none. The category filter, "select all N matching" and saved views match it.
"""
import os
import sqlite3
from unittest.mock import patch
try:
import test_api_workspace_images as workspace
from test_image_categories import category_checksum
except ImportError:
from tests import test_api_workspace_images as workspace
from tests.test_image_categories import category_checksum
from py.api.images import image_category_key, parse_listing_filter
class WorkspaceCategoryCase(workspace.ImageAPITestCase):
"""Listing helpers plus own-category writes keyed as the routes key them."""
media = workspace.TestWorkspaceImages.media
get = workspace.TestWorkspaceImages.get
tag = workspace.TestWorkspaceImageTags.tag
names = workspace.TestWorkspaceImageTags.names
async def tearDownAsync(self):
self.api.db.close_all()
workspace.assert_closed(self, self._temp_db.name, self._output_tmp.name)
await super().tearDownAsync()
def own(self, path, value, sources=None):
sources = sources or self.api._get_all_output_dirs()
self.api.db.set_image_categories([image_category_key(sources, path)], value)
async def listed(self, **params):
response, body = await self.get(sort="name", **params)
self.assertEqual(response.status, 200, body)
return self.names(body), body
class TestEffectiveCategoryFilter(WorkspaceCategoryCase):
def batch(self):
"""Four images of "castle at dusk" (Landscapes); one is overridden."""
castle = self._save_prompt("castle at dusk", category="Landscapes", rating=4)
files = [self.media(f"castle_{n}.png", mtime=100 + n) for n in range(4)]
for path in files:
self.api.db.link_image_to_prompt(castle, str(path))
self.own(files[0], "Portraits")
return castle, files
async def test_category_filter_prefers_own_category(self):
castle, _files = self.batch()
conn = self.api.db.model.get_connection()
before = category_checksum(conn)
names, body = await self.listed(category="Portraits")
self.assertEqual(names, ["castle_0.png"])
# The caption still counts the prompt behind the image shown.
self.assertEqual(body["positive_count"], 1)
names, body = await self.listed(category="Landscapes")
self.assertEqual(names, ["castle_1.png", "castle_2.png", "castle_3.png"])
self.assertEqual(body["total"], 3)
names, _ = await self.listed()
self.assertEqual(len(names), 4)
self.assertEqual(category_checksum(conn), before)
self.assertEqual(self.api.db.get_prompt_by_id(castle)["category"], "Landscapes")
async def test_category_filter_lists_unlinked_with_own(self):
scans = [self.media(f"scan_{n}.png", mtime=100 + n) for n in range(2)]
self.media("plain.png", mtime=50)
for path in scans:
self.own(path, "Scans")
names, body = await self.listed(category="Scans")
self.assertEqual(names, ["scan_0.png", "scan_1.png"])
self.assertEqual(body["positive_count"], 0)
async def test_recipe_category_is_inherited(self):
"""MC4: a recipe's own category, not the positive's, is inherited."""
db = self.api.db
self._save_prompt("Pair positive", category="Night")
pair = db.save_prompt_pair("Pair positive", "Noise", category="Recipes")
blank = db.save_prompt_pair("Pair positive", "Other noise", category=" ")
recipe = self.media("recipe.png", mtime=103)
blank_recipe = self.media("blank_recipe.png", mtime=102)
positive = db.get_prompt_pair(pair)["positive_prompt_id"]
for path, pair_id in ((recipe, pair), (blank_recipe, blank)):
image = db.link_image_to_prompt(positive, str(path))
db.link_image_to_pair(pair_id, image, {"source": "fixture"})
self.assertEqual((await self.listed(category="Recipes"))[0], ["recipe.png"])
# A blank recipe category falls back to the positive's.
self.assertEqual((await self.listed(category="Night"))[0], ["blank_recipe.png"])
# An own category still wins over the recipe's.
self.own(recipe, "Mine")
self.assertEqual((await self.listed(category="Recipes"))[0], [])
self.assertEqual((await self.listed(category="Mine"))[0], ["recipe.png"])
async def test_ambiguous_files_inherit_nothing_but_keep_their_own(self):
first = self._save_prompt("Owner one", category="Night")
second = self._save_prompt("Owner two", category="Night")
ambiguous = self.media("ambiguous.png")
for owner in (first, second):
self.api.db.link_image_to_prompt(owner, str(ambiguous))
self.assertEqual((await self.listed(category="Night"))[0], [])
self.own(ambiguous, "Night")
self.assertEqual((await self.listed(category="Night"))[0], ["ambiguous.png"])
async def test_combines_with_tags_search_and_rating(self):
stone = self._save_prompt(
"castle at dusk", category="Landscapes", tags=["stone"], rating=4
)
files = [self.media(f"castle_{n}.png", mtime=100 + n) for n in range(2)]
for path in files:
self.api.db.link_image_to_prompt(stone, str(path))
self.own(files[0], "Portraits")
other = self._save_prompt("harbor at noon", category="Portraits", rating=2)
harbor = self.media("harbor.png", mtime=90)
self.api.db.link_image_to_prompt(other, str(harbor))
names, _ = await self.listed(category="Portraits", tags="stone")
self.assertEqual(names, ["castle_0.png"])
names, _ = await self.listed(category="Portraits", q="harbor")
self.assertEqual(names, ["harbor.png"])
names, _ = await self.listed(category="Portraits", q="castle_0")
self.assertEqual(names, ["castle_0.png"])
response, body = await self.get(category="Portraits", sort="rating")
self.assertEqual(response.status, 200, body)
self.assertEqual(self.names(body), ["castle_0.png", "harbor.png"])
self.assertEqual(body["positive_count"], 2)
async def test_image_tag_admitted_file_keeps_its_inherited_category(self):
night = self._save_prompt("Night owl", category="Night", tags=["a"])
tagged = self.media("tagged.png", mtime=103)
unlinked = self.media("unlinked.png", mtime=102)
self.api.db.link_image_to_prompt(night, str(tagged))
self.tag(tagged, ["b"])
self.tag(unlinked, ["a", "b"])
names, _ = await self.listed(tags="a,b", category="Night")
self.assertEqual(names, ["tagged.png"])
self.own(unlinked, "Night")
names, _ = await self.listed(tags="a,b", category="Night")
self.assertEqual(names, ["tagged.png", "unlinked.png"])
async def test_cursor_pages_walk_only_the_category(self):
files = [self.media(f"img_{n:02}.png", mtime=100 + n) for n in range(7)]
for path in files[::2]:
self.own(path, "Even")
seen, cursor = [], None
while True:
params = {"category": "Even", "limit": 2, "sort": "name"}
if cursor:
params["cursor"] = cursor
response, body = await self.get(**params)
self.assertEqual(response.status, 200, body)
seen += self.names(body)
self.assertEqual(body["total"], 4)
cursor = body["next_cursor"]
if not cursor:
break
self.assertEqual(seen, ["img_00.png", "img_02.png", "img_04.png", "img_06.png"])
async def test_an_outer_source_lists_a_nested_sources_own_category(self):
inner = self.output_dir / "nested"
nested = self.media("nested/a.png")
self.media("b.png")
self.api._get_all_output_dirs = lambda: [self.output_dir, inner]
self.own(nested, "Scans")
names, _ = await self.listed(category="Scans", root="0")
self.assertEqual(names, ["nested/a.png"])
names, _ = await self.listed(category="Scans", root="1")
self.assertEqual(names, ["a.png"])
# The outer source's own key for that path is not the file's key.
from py.api.images import _favorite_root_key
self.api.db.clear_image_categories([image_category_key([inner], nested)])
self.api.db.set_image_categories(
[(_favorite_root_key(self.output_dir), "nested/a.png")], "Stale"
)
self.assertEqual((await self.listed(category="Stale"))[0], [])
async def test_a_gone_files_category_is_not_listed(self):
gone = self.media("gone.png")
self.own(gone, "Scans")
os.unlink(gone)
self.api.invalidate_gallery_cache()
self.assertEqual((await self.listed(category="Scans"))[0], [])
async def test_a_failed_own_category_read_is_listing_failed(self):
self.media("a.png")
def broken(*_args, **_kwargs):
raise sqlite3.OperationalError("secret detail")
with patch.object(self.api.db, "own_categories_of", side_effect=broken):
with self.assertLogs("prompt_manager", level="ERROR"):
response, body = await self.get(category="Scans")
self.assertEqual(response.status, 500)
self.assertNotIn("secret", str(body))
# Without a category nothing new is read.
response, body = await self.get()
self.assertEqual(response.status, 200, body)
async def test_select_all_matching_uses_the_effective_category(self):
_castle, files = self.batch()
resolved = await self.api.resolve_listing_filter(
parse_listing_filter({"category": "Portraits"}), 100
)
self.assertEqual([t[2] for t in resolved.targets], ["castle_0.png"])
resolved = await self.api.resolve_listing_filter(
parse_listing_filter({"category": "Landscapes"}), 100
)
self.assertEqual(len(resolved.targets), 3)
self.assertEqual(len(files), 4)
async def test_saved_view_category_means_the_effective_category(self):
"""CUJ 8: a saved view's category is the listing's query value."""
_castle, files = self.batch()
self.own(files[1], "Portraits")
names, _ = await self.listed(category="Portraits")
self.assertEqual(names, ["castle_0.png", "castle_1.png"])
class TestFilterAndChipAgreeOnBlanks(WorkspaceCategoryCase):
"""The filter and the inspector use one inherited-category rule."""
async def chip(self, path):
response = await self.client.get(
f"/prompt_manager/images/inspect/{path}", params={"root": "0"}
)
body = await response.json()
self.assertEqual(response.status, 200, body)
return body["library"]["category"]
async def test_blank_recipe_values_agree(self):
db = self.api.db
positive = self._save_prompt("Pair positive", category="Night")
blanks = {"spaces": " ", "tab": "\t", "nbsp": " ", "set": "Recipes"}
for index, (label, value) in enumerate(blanks.items()):
pair = db.save_prompt_pair("Pair positive", f"noise {label}")
path = self.media(f"{label}.png", mtime=100 + index)
image = db.link_image_to_prompt(positive, str(path))
db.link_image_to_pair(pair, image, {"source": "fixture"})
with db.model.get_connection() as conn:
conn.execute(
"UPDATE prompt_pairs SET category=? WHERE id=?", (value, pair)
)
blob = db.save_prompt_pair("Pair positive", "noise blob")
path = self.media("blob.png", mtime=90)
image = db.link_image_to_prompt(positive, str(path))
db.link_image_to_pair(blob, image, {"source": "fixture"})
with db.model.get_connection() as conn:
conn.execute(
"UPDATE prompt_pairs SET category=CAST(' ' AS BLOB) WHERE id=?",
(blob,),
)
for name in ("spaces", "tab", "nbsp", "set", "blob"):
with self.subTest(name=name):
shown = await self.chip(f"{name}.png")
for value in ("Night", "Recipes"):
listed = f"{name}.png" in (await self.listed(category=value))[0]
self.assertEqual(listed, shown == value, (name, value, shown))
+13 -273
View File
@@ -1347,7 +1347,7 @@ class TestWorkspaceImages(ImageAPITestCase):
_, data = await self.get(q="harbor")
self.assertEqual(self._names(data), ["night_a.png"])
self.assertEqual(
self.api.db.get_image_paths_matching("zebra", []),
self.api.db.get_image_paths_matching("zebra", None, []),
{
normalize_image_path(str(files[name]))
for name in ("night_ab", "day_ab", "titled")
@@ -1407,10 +1407,8 @@ class TestWorkspaceImages(ImageAPITestCase):
for query in statements
if query.lstrip().upper().startswith(("SELECT", "WITH"))
]
# Plus the one stream of tagged image records q and tags read (B3),
# and the category's own-category read of the one source (#332).
self.assertEqual(len(selects), 5, selects)
self.assertEqual(sum("image_categories" in q for q in selects), 1, selects)
# Plus the one stream of tagged image records q and tags read (B3).
self.assertEqual(len(selects), 4, selects)
self.assertEqual(sum("image_favorites" in q for q in selects), 1, selects)
self.assertEqual(sum("media_image_tags" in q for q in selects), 1, selects)
@@ -1530,16 +1528,15 @@ class TestWorkspaceImages(ImageAPITestCase):
elsewhere = self.api.db.get_prompt_id_for_image("/elsewhere/7.png")
self.assertEqual(
self.api.db.get_image_link_matches(
"", [], every_link=True, wanted=wanted, category="Night"
"", "Night", [], every_link=True, wanted=wanted
),
{
normalize_image_path(str(files["night_ab"])): (
True,
5,
self.api.db.get_prompt_id_for_image(str(files["night_ab"])),
"Night",
),
"/elsewhere/7.png": (True, 5, elsewhere, "Night"),
"/elsewhere/7.png": (True, 5, elsewhere),
},
)
@@ -1591,24 +1588,6 @@ class TestWorkspaceImages(ImageAPITestCase):
for i in range(links)
],
)
# 2,000 images (one in 25 at 50,000 files) carry their own category
# (#332), so a category filter also reads and matches those.
from py.api.images import _favorite_root_key
root_key = _favorite_root_key(self.output_dir)
with self.api.db.model.get_connection() as conn:
conn.executemany(
"INSERT INTO image_categories VALUES(?,?,?,?)",
[
(
root_key,
listed[i][0].relative_to(self.output_dir).as_posix(),
f"cat-{i % 10}",
"2026-10-27T00:00:00+00:00",
)
for i in range(1, files, 25)[:2_000]
],
)
return listed
_BUDGET_REQUESTS = (
@@ -1636,7 +1615,7 @@ class TestWorkspaceImages(ImageAPITestCase):
import math
from py.api import images as images_module
from database.operations import link_filter_query, link_matches_json_query
from database.operations import link_filter_query
listed = await asyncio.get_running_loop().run_in_executor(
None, self._budget_fixture, 5_000, 2_000
@@ -1699,13 +1678,7 @@ class TestWorkspaceImages(ImageAPITestCase):
# q and tags also stream the tagged image records once
# (#249 B3); none are tagged here, so nothing is stat'ed.
streamed = "q" in params or "tags" in params
# A category also reads the own categories, one primary
# key range per source (#332).
own = "category" in params
self.assertEqual(len(found), 1 + streamed + own, found)
self.assertEqual(
sum("image_categories" in query for query in found), int(own)
)
self.assertEqual(len(found), 2 if streamed else 1, found)
self.assertEqual(
sum("media_image_tags" in query for query in found),
int(streamed),
@@ -1765,42 +1738,13 @@ class TestWorkspaceImages(ImageAPITestCase):
self.assertEqual(self.api._workspace_listed_cache, {})
self.assertEqual(self.api._workspace_order_cache, {})
# The per-row ambiguity probe is an index search, never a scan.
# The read runs it wrapped in one JSON aggregate (#354): same plan.
sql, params = link_filter_query("lantern", ["tag-1"], every_link=True)
for query in (sql, link_matches_json_query(sql)):
with self.subTest(wrapped=query is not sql):
with self.api.db.model.get_connection() as conn:
plan = [
row[3]
for row in conn.execute("EXPLAIN QUERY PLAN " + query, params)
]
self.assertIn(
"SEARCH other USING COVERING INDEX idx_image_file_path "
"(file_path=?)",
plan,
)
self.assertFalse(
[step for step in plan if step.startswith("SCAN other")]
)
# The category's narrowing clause (#332) keeps every join an index
# search, plain and wrapped with the raw categories.
sql, params = link_filter_query(
"", ["tag-1"], every_link=True, category="cat-3"
sql, params = link_filter_query("lantern", "cat-3", ["tag-1"], every_link=True)
with self.api.db.model.get_connection() as conn:
plan = [row[3] for row in conn.execute("EXPLAIN QUERY PLAN " + sql, params)]
self.assertIn(
"SEARCH other USING COVERING INDEX idx_image_file_path (file_path=?)", plan
)
for query in (sql, link_matches_json_query(sql, categories=True)):
with self.subTest(category_wrapped=query is not sql):
with self.api.db.model.get_connection() as conn:
plan = [
row[3]
for row in conn.execute("EXPLAIN QUERY PLAN " + query, params)
]
self.assertIn(
"SEARCH other USING COVERING INDEX idx_image_file_path "
"(file_path=?)",
plan,
)
scans = [step for step in plan if step.startswith("SCAN")]
self.assertTrue(all(step.startswith("SCAN gi") for step in scans), plan)
self.assertFalse([step for step in plan if step.startswith("SCAN other")])
@unittest.skipUnless(
PERF_BUDGET and not TRACED,
@@ -2550,207 +2494,3 @@ class TestListingFilterSelection(ImageAPITestCase):
self.assertEqual(response.status, 409)
self.assertEqual((await response.json())["code"], "source-changed")
class TestWorkspaceCollectionFilter(ImageAPITestCase):
"""#331: ``collection=<id>`` keeps only that collection's listed members."""
media = TestWorkspaceImages.media
get = TestWorkspaceImages.get
_names = TestWorkspaceImages._names
async def tearDownAsync(self):
self.api.db.close_all()
assert_closed(self, self._temp_db.name, self._output_tmp.name)
await super().tearDownAsync()
def keys(self, root, relative):
from py.api.images import _favorite_relative_key, _favorite_root_key
return (_favorite_root_key(root), _favorite_relative_key(relative))
def collect(self, name, *members):
"""A collection holding ``(root, relative)`` members; its id."""
result = self.api.db.add_to_collection(
name=name, items=[self.keys(root, rel) for root, rel in members]
)
return result["collection"]["id"]
def two_sources(self):
second = self.comfy_dir / "second"
second.mkdir()
self.api._get_all_output_dirs = lambda: [self.output_dir, second]
return second
async def test_listing_keeps_only_collection_members(self):
second = self.two_sources()
self.media("a.png", 300)
self.media("b.png", 200)
self.media("sub/c.png", 100, root=second)
self.media("d.png", 50, root=second)
grids = self.collect(
"Grids",
(self.output_dir, "a.png"),
(second, "sub/c.png"),
(self.output_dir, "gone.png"), # stored, never listed
)
self.collect("Other", (self.output_dir, "b.png"))
response, data = await self.get(collection=str(grids))
self.assertEqual(response.status, 200, data)
self.assertEqual(self._names(data), ["a.png", "c.png"])
self.assertEqual(data["total"], 2)
_, data = await self.get(collection=str(grids), root="1")
self.assertEqual(self._names(data), ["c.png"])
# Without the filter the listing is unchanged.
_, data = await self.get()
self.assertEqual(data["total"], 4)
async def test_collection_filter_combines_with_the_other_filters(self):
night = self._save_prompt("Neon zebra", category="Night", tags=["a"])
day = self._save_prompt("Morning field", category="Day", tags=["a"])
files = {
name: self.media(name, 100 + index)
for index, name in enumerate(
("night.png", "day.png", "zebra_plain.png", "clip.mp4", "out.png")
)
}
self.api.db.link_image_to_prompt(night, str(files["night.png"]))
self.api.db.link_image_to_prompt(day, str(files["day.png"]))
members = [(self.output_dir, name) for name in files if name != "out.png"]
grids = str(self.collect("Grids", *members))
cases = (
({}, ["clip.mp4", "day.png", "night.png", "zebra_plain.png"]),
({"q": "zebra"}, ["night.png", "zebra_plain.png"]),
({"category": "Night"}, ["night.png"]),
({"tags": "a"}, ["day.png", "night.png"]),
({"media": "video"}, ["clip.mp4"]),
({"media": "image", "q": "day"}, ["day.png"]),
)
for params, expected in cases:
with self.subTest(**params):
response, data = await self.get(collection=grids, sort="name", **params)
self.assertEqual(response.status, 200, data)
self.assertEqual(self._names(data), expected)
self.assertEqual(data["total"], len(expected))
async def test_collection_filter_combines_with_an_images_own_category(self):
"""#331 with #332: a member is admitted by its effective category."""
from py.api.images import image_category_key
night = self._save_prompt("Neon zebra", category="Night")
files = {
name: self.media(name, 100 + index)
for index, name in enumerate(
("linked.png", "overridden.png", "own.png", "outside.png")
)
}
for name in ("linked.png", "overridden.png"):
self.api.db.link_image_to_prompt(night, str(files[name]))
sources = self.api._get_all_output_dirs()
def own(name, value):
key = image_category_key(sources, files[name])
self.api.db.set_image_categories([key], value)
own("overridden.png", "Portraits")
own("own.png", "Night") # unlinked: only its own category
own("outside.png", "Night") # not a member
members = [(self.output_dir, name) for name in files if name != "outside.png"]
grids = str(self.collect("Grids", *members))
for category, expected in (
("Night", ["linked.png", "own.png"]),
("Portraits", ["overridden.png"]),
):
with self.subTest(category=category):
response, data = await self.get(
collection=grids, category=category, sort="name"
)
self.assertEqual(response.status, 200, data)
self.assertEqual(self._names(data), expected)
self.assertEqual(data["total"], len(expected))
async def test_invalid_collection_is_a_400(self):
for value in ("x", "0", "-1", "1.5", "1234567890", " 1"):
with self.subTest(value=value):
response, data = await self.get(collection=value)
self.assertEqual(response.status, 400, data)
async def test_unknown_collection_is_a_404(self):
self.media("a.png")
response, data = await self.get(collection="77")
self.assertEqual(response.status, 404)
self.assertEqual(data["code"], "collection-not-found")
self.assertNotIn("images", data)
self.api._get_all_output_dirs = lambda: []
response, data = await self.get(collection="77")
self.assertEqual((response.status, data["code"]), (404, "collection-not-found"))
async def test_a_collection_deleted_during_the_read_is_a_404(self):
self.media("a.png")
grids = self.collect("Grids", (self.output_dir, "a.png"))
real = self.api.db.collection_members
def deleted_meanwhile(collection_id, root_key):
self.api.db.delete_collection(collection_id)
return real(collection_id, root_key)
with patch.object(
self.api.db, "collection_members", side_effect=deleted_meanwhile
):
response, data = await self.get(collection=str(grids))
self.assertEqual((response.status, data["code"]), (404, "collection-not-found"))
async def test_a_failed_members_read_is_a_500_never_an_unfiltered_page(self):
self.media("a.png")
self.media("b.png")
grids = self.collect("Grids", (self.output_dir, "a.png"))
def broken(*_args, **_kwargs):
raise sqlite3.OperationalError("database is locked")
with patch.object(self.api.db, "collection_members", side_effect=broken):
response, data = await self.get(collection=str(grids))
self.assertEqual(response.status, 500)
self.assertNotIn("images", data)
self.assertNotIn("locked", json.dumps(data))
async def test_reordering_sources_keeps_members(self):
second = self.two_sources()
self.media("a.png", root=second)
self.media("b.png")
grids = str(self.collect("Grids", (second, "a.png")))
_, data = await self.get(collection=grids)
self.assertEqual(
[(i["root_index"], i["path"]) for i in data["images"]], [(1, "a.png")]
)
self.api._get_all_output_dirs = lambda: [second, self.output_dir]
_, data = await self.get(collection=grids)
self.assertEqual(
[(i["root_index"], i["path"]) for i in data["images"]], [(0, "a.png")]
)
async def test_cursor_from_another_collection_is_refused(self):
for index, name in enumerate(("a.png", "b.png", "c.png")):
self.media(name, 100 + index)
everything = [(self.output_dir, name) for name in ("a.png", "b.png", "c.png")]
first = str(self.collect("First", *everything))
second = str(self.collect("Second", *everything))
_, page = await self.get(collection=first, limit=1)
token = page["next_cursor"]
self.assertIsInstance(token, str)
response, _ = await self.get(collection=first, limit=1, cursor=token)
self.assertEqual(response.status, 200)
for params in ({"collection": second}, {}, {"favorites": "1"}):
with self.subTest(**params):
response, _ = await self.get(cursor=token, limit=1, **params)
self.assertEqual(response.status, 400)
async def test_collection_wins_over_favorites(self):
self.media("a.png")
self.media("b.png")
grids = str(self.collect("Grids", (self.output_dir, "a.png")))
from py.api.images import _favorite_root_key
self.api.db.set_image_favorite(_favorite_root_key(self.output_dir), "b.png")
_, data = await self.get(collection=grids, favorites="1")
self.assertEqual(self._names(data), ["a.png"])
-43
View File
@@ -553,35 +553,6 @@ class TestStartRefusals(BatchTestCase):
self.assertEqual(self.api._jobs.list(), [])
self.assertIsNone(model_gate.holder())
async def test_a_missing_backend_package_refuses_the_start(self):
self.tagger.missing["gguf"] = ["llama-cpp-python"]
status, data = await self.start_batch(model_type="gguf")
self.assertEqual((status, data["code"]), (409, "model-unavailable"))
self.assertEqual(data["success"], False)
self.assertTrue(
data["error"].startswith("GGUF JoyCaption needs llama-cpp-python."), data
)
self.assertIn("pip install llama-cpp-python", data["error"])
self.assertEqual(self.api._jobs.list(), [])
self.assertEqual(self.tagger.load_calls, [])
self.assertIsNone(model_gate.holder())
async def test_a_package_lost_after_the_start_fails_the_job_without_a_traceback(
self,
):
from py.autotag import MissingBackendPackages
self.image("a.png", (1, 1, 1))
self.tagger.load_error = MissingBackendPackages("gguf", ["llama-cpp-python"])
with self.assertNoLogs(self.api.logger, level="ERROR"):
job = await self.run_batch(model_type="gguf")
self.assertEqual(job.state, "failed")
self.assertTrue(
job.status_line.startswith("GGUF JoyCaption needs"), job.status_line
)
self.assertTrue(any("pip install llama-cpp-python" in line for line in job.log))
self.assertIsNone(model_gate.holder())
async def test_model_busy_when_the_gate_is_held(self):
owner = model_gate.new_owner("image-annotations")
self.assertTrue(model_gate.try_acquire(owner))
@@ -687,20 +658,6 @@ class TestPreview(BatchTestCase):
self.assertIsNone(llm["estimate_device"])
self.assertEqual([s for s in _strings(skip) if _absolute(s)], [])
async def test_preview_names_missing_backend_packages(self):
self.image("a.png", (1, 1, 1))
data = (await self.preview(model_type="gguf"))[1]
self.assertEqual(data["model"]["missing_packages"], [])
self.assertIsNone(data["model"]["install_command"])
self.tagger.missing["gguf"] = ["llama-cpp-python"]
data = (await self.preview(model_type="gguf"))[1]
self.assertTrue(data["model"]["downloaded"])
self.assertEqual(data["model"]["missing_packages"], ["llama-cpp-python"])
self.assertEqual(
data["model"]["install_command"], "pip install llama-cpp-python"
)
self.assertIn("DGGML_CUDA", data["model"]["install_note"])
async def test_preview_uses_the_graphics_card_rate_when_wd14_runs_on_cuda(self):
self.image("a.png", (1, 1, 1))
self.tagger.wd14_device = lambda: "gpu"
+1 -5
View File
@@ -800,12 +800,8 @@ class TestDefaultLoaders(EngineTestCase):
transformers, model, processor, inputs = self._fake_transformers()
pixel = inputs["pixel_values"]
service = AutoTagService(models_dir=self.models_dir)
# The 8-bit load also needs these two (#424); stand-ins satisfy the check.
stubs = {
name: types.ModuleType(name) for name in ("accelerate", "bitsandbytes")
}
with _patch_modules({"torch": torch, "transformers": transformers, **stubs}):
with _patch_modules({"torch": torch, "transformers": transformers}):
service.load_model("hf")
tags = service.generate_tags(str(self.image_path))
-33
View File
@@ -161,39 +161,6 @@ class TestLegacyBatch(LegacyTestCase):
self.assertEqual(self.tagger.calls[-1]["prompt"], AutotagConfig.TEMPLATE)
self.assertEqual(self.tagger.custom_prompt, "legacy prompt")
async def test_a_missing_backend_package_is_one_error_frame(self):
self.tagger.missing["gguf"] = ["llama-cpp-python"]
_, frames = await self.legacy("model_type=gguf")
self.assertEqual(len(frames), 1)
self.assertEqual(frames[0]["type"], "error")
self.assertTrue(
frames[0]["message"].startswith("GGUF JoyCaption needs llama-cpp-python.")
)
self.assertEqual(self.tagger.load_calls, [])
async def test_legacy_single_refuses_a_missing_backend_package(self):
from py.autotag import MissingBackendPackages
self.image("a.png", (1, 1, 1))
await self.relist()
self.tagger.missing["gguf"] = ["llama-cpp-python"]
status, data = await self.single(
{"path": "a.png", "root_index": 0, "model_type": "gguf"}
)
self.assertEqual((status, data["code"]), (409, "model-unavailable"))
self.assertIn("pip install llama-cpp-python", data["error"])
self.assertEqual(self.tagger.load_calls, [])
# Lost after the check: the same 409, no ERROR record, nothing held.
self.tagger.missing["gguf"] = []
self.tagger.load_error = MissingBackendPackages("gguf", ["llama-cpp-python"])
with self.assertNoLogs(self.api.logger, level="ERROR"):
status, data = await self.single(
{"path": "a.png", "root_index": 0, "model_type": "gguf"}
)
self.assertEqual((status, data["code"]), (409, "model-unavailable"))
self.assertIn("pip install llama-cpp-python", data["error"])
self.assertIsNone(model_gate.holder())
async def test_each_refusal_is_one_error_frame(self):
self.tagger.downloaded["wd14-vit"] = False
_, frames = await self.legacy()
-248
View File
@@ -1,248 +0,0 @@
"""#424: each tagging backend reports the Python packages it cannot import.
``find_spec`` is patched throughout: no ML backend is installed or imported.
"""
import os
import sys
import types
import unittest
from unittest.mock import patch
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
try:
from test_autotag_engine import (
EngineTestCase,
FakeGgufModel,
_patch_modules,
_touch_model_files,
)
except ImportError: # pragma: no cover - run as a package
from tests.test_autotag_engine import (
EngineTestCase,
FakeGgufModel,
_patch_modules,
_touch_model_files,
)
from py import autotag # noqa: E402
from py.autotag import ( # noqa: E402
BACKEND_PACKAGES,
MODELS,
AutoTagService,
MissingBackendPackages,
missing_packages,
package_fields,
)
GGUF_CUDA = 'For a CUDA build: CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python'
def _find_spec_without(*absent):
"""A ``find_spec`` that knows every module except *absent*."""
def find_spec(name, *args):
return None if name in absent else object()
return find_spec
def _missing_from(*absent):
return patch.object(
autotag.importlib.util, "find_spec", side_effect=_find_spec_without(*absent)
)
class TestMissingPackages(unittest.TestCase):
def test_every_backend_names_its_modules_and_pip_names(self):
self.assertEqual(BACKEND_PACKAGES["gguf"], (("llama_cpp", "llama-cpp-python"),))
self.assertEqual(
[pip for _module, pip in BACKEND_PACKAGES["hf"]],
["torch", "transformers", "accelerate", "bitsandbytes"],
)
for model in ("wd14-swinv2", "wd14-vit"):
self.assertEqual(BACKEND_PACKAGES[model], (("onnxruntime", "onnxruntime"),))
self.assertEqual(set(BACKEND_PACKAGES), set(MODELS))
def test_reports_pip_names_of_modules_find_spec_cannot_find(self):
with _missing_from("llama_cpp", "accelerate", "bitsandbytes"):
self.assertEqual(missing_packages("gguf"), ["llama-cpp-python"])
self.assertEqual(missing_packages("hf"), ["accelerate", "bitsandbytes"])
self.assertEqual(missing_packages("wd14-vit"), [])
def test_find_spec_errors_count_as_missing(self):
for error in (ValueError("no spec"), ImportError("broken"), OSError("io")):
with self.subTest(error=type(error).__name__):
with patch.object(
autotag.importlib.util, "find_spec", side_effect=error
):
self.assertEqual(missing_packages("wd14-vit"), ["onnxruntime"])
def test_a_module_already_imported_is_present_without_a_lookup(self):
with _patch_modules({"llama_cpp": types.ModuleType("llama_cpp")}):
with patch.object(
autotag.importlib.util, "find_spec", side_effect=AssertionError
):
self.assertEqual(missing_packages("gguf"), [])
def test_the_check_imports_nothing(self):
before = set(sys.modules)
missing_packages("gguf")
missing_packages("hf")
missing_packages("wd14-vit")
imported = set(sys.modules) - before
heavy = {"llama_cpp", "torch", "transformers", "onnxruntime"}
self.assertFalse(imported & heavy, imported)
def test_unknown_model_has_no_packages(self):
self.assertEqual(missing_packages("nope"), [])
def test_package_fields_and_the_one_sentence(self):
self.assertEqual(
package_fields("gguf", []),
{"missing_packages": [], "install_command": None, "install_note": None},
)
fields = package_fields("gguf", ["llama-cpp-python"])
self.assertEqual(fields["missing_packages"], ["llama-cpp-python"])
self.assertEqual(fields["install_command"], "pip install llama-cpp-python")
self.assertEqual(fields["install_note"], GGUF_CUDA)
self.assertEqual(
package_fields("hf", ["torch", "bitsandbytes"])["install_command"],
"pip install torch bitsandbytes",
)
self.assertIsNone(package_fields("hf", ["torch"])["install_note"])
self.assertIn(
"onnxruntime-gpu",
package_fields("wd14-vit", ["onnxruntime"])["install_note"],
)
error = MissingBackendPackages("gguf", ["llama-cpp-python"])
self.assertIsInstance(error, RuntimeError)
self.assertEqual(
str(error),
"GGUF JoyCaption needs llama-cpp-python. Install it into ComfyUI's "
"Python: pip install llama-cpp-python. " + GGUF_CUDA,
)
self.assertEqual(
str(MissingBackendPackages("hf", ["accelerate", "bitsandbytes"])),
"HuggingFace JoyCaption needs accelerate and bitsandbytes. Install them "
"into ComfyUI's Python: pip install accelerate bitsandbytes.",
)
class TestStatusReportsPackages(EngineTestCase):
def test_downloaded_keeps_its_files_only_meaning(self):
_touch_model_files(self.models_dir, "gguf")
_touch_model_files(self.models_dir, "wd14-vit")
service = AutoTagService(models_dir=self.models_dir)
with _missing_from("llama_cpp"):
status = service.get_models_status()
self.assertTrue(status["gguf"]["downloaded"])
self.assertEqual(status["gguf"]["missing_packages"], ["llama-cpp-python"])
self.assertEqual(
status["gguf"]["install_command"], "pip install llama-cpp-python"
)
self.assertEqual(status["gguf"]["install_note"], GGUF_CUDA)
self.assertEqual(status["wd14-vit"]["missing_packages"], [])
self.assertIsNone(status["wd14-vit"]["install_command"])
# A model not on disk still says which packages it will need.
self.assertFalse(status["wd14-swinv2"]["downloaded"])
self.assertIn("missing_packages", status["hf"])
def test_an_injected_loader_brings_its_own_backend(self):
service = self._service(("gguf", FakeGgufModel("x")))
with _missing_from("llama_cpp"):
self.assertEqual(
service.get_models_status()["gguf"]["missing_packages"], []
)
def test_an_injected_probe_is_used_for_status_and_load(self):
_touch_model_files(self.models_dir, "gguf")
service = AutoTagService(
models_dir=self.models_dir,
model_loader=lambda *_: ("gguf", FakeGgufModel("x")),
package_probe=lambda model_type: ["llama-cpp-python"],
)
self.assertEqual(
service.get_models_status()["gguf"]["missing_packages"],
["llama-cpp-python"],
)
with self.assertRaises(MissingBackendPackages):
service.load_model("gguf")
class TestLoadFailsFast(EngineTestCase):
def test_missing_package_refuses_before_the_loader_without_an_error_log(self):
_touch_model_files(self.models_dir, "gguf")
service = AutoTagService(models_dir=self.models_dir)
calls = []
service._model_loader = lambda *args: calls.append(args)
with (
_missing_from("llama_cpp"),
self.assertLogs(service.logger, level="DEBUG") as logs,
):
with self.assertRaises(MissingBackendPackages) as ctx:
service.load_model("gguf")
self.assertEqual(calls, [])
self.assertEqual(ctx.exception.model_type, "gguf")
self.assertEqual(ctx.exception.missing, ["llama-cpp-python"])
self.assertIn("pip install llama-cpp-python", str(ctx.exception))
self.assertFalse(service.is_model_loaded())
levels = {record.levelname for record in logs.records}
self.assertNotIn("ERROR", levels)
self.assertIn("WARNING", levels)
self.assertTrue(all(record.exc_info is None for record in logs.records))
def test_not_downloaded_is_reported_before_packages(self):
service = AutoTagService(models_dir=self.models_dir)
with _missing_from("llama_cpp"):
with self.assertRaises(RuntimeError) as ctx:
service.load_model("gguf")
self.assertNotIsInstance(ctx.exception, MissingBackendPackages)
def test_a_module_not_found_from_the_loader_becomes_the_same_refusal(self):
_touch_model_files(self.models_dir, "gguf")
def loader(model_type, use_gpu):
raise ModuleNotFoundError("No module named 'llama_cpp'", name="llama_cpp")
service = AutoTagService(
models_dir=self.models_dir,
model_loader=loader,
package_probe=lambda _model: [],
)
with self.assertLogs(service.logger, level="DEBUG") as logs:
with self.assertRaises(MissingBackendPackages) as ctx:
service.load_model("gguf")
self.assertEqual(ctx.exception.missing, ["llama-cpp-python"])
self.assertNotIn("ERROR", {record.levelname for record in logs.records})
self.assertFalse(service.is_model_loaded())
def test_an_unknown_module_not_found_names_the_module(self):
def loader(model_type, use_gpu):
raise ModuleNotFoundError(
"No module named 'sentencepiece'", name="sentencepiece"
)
hf_dir = self.models_dir / MODELS["hf"]["subdir"]
hf_dir.mkdir(parents=True, exist_ok=True)
(hf_dir / "config.json").write_text("{}")
service = AutoTagService(
models_dir=self.models_dir,
model_loader=loader,
package_probe=lambda _model: [],
)
with self.assertRaises(MissingBackendPackages) as ctx:
service.load_model("hf")
self.assertEqual(ctx.exception.missing, ["sentencepiece"])
if __name__ == "__main__":
unittest.main()
-863
View File
@@ -1,863 +0,0 @@
"""Cached re-runs of runtime-linked prompt text count once per job (#347).
Jobs are simulated end to end: the queue hook (submit + dequeue), the
``execution_cached`` replay, the real node executions and ComfyUI's output
cache (fakes of ``execution`` and ``comfy_execution.caching``, as in
tests/test_execution_context_host.py).
"""
import logging
import os
import sys
import tempfile
import threading
import types
import unittest
from collections import namedtuple
from unittest import mock
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
try:
from tests.open_handles import assert_closed
except ImportError: # discovered with tests/ as the top-level directory
from open_handles import assert_closed
from database.operations import PromptDatabase
from utils import cached_runs as module
from utils.cached_runs import (
CachedRuns,
Facts,
Role,
Seen,
graph_role,
graph_seen,
own_counts,
register_cached_runs,
stored_facts,
)
from utils.execution_context_host import cached_string_outputs
from utils.hashing import generate_prompt_hash
from utils.run_keys import prompt_node_key, upstream_closure, upstream_nodes
from utils.usage_tracking import (
PendingFirstUse,
QueuedRuns,
find_pair_id,
find_pair_id_by_hashes,
handle_queued_prompt,
on_dequeue,
)
Entry = namedtuple("CacheEntry", "ui outputs")
LLM = "a lighthouse the LLM wrote"
class _Cache:
def __init__(self):
self.entries = {}
def get_local(self, node_id):
return self.entries.get(node_id)
class HierarchicalCache(_Cache):
pass
class LRUCache(_Cache):
pass
class RAMPressureCache(LRUCache):
pass
class NullCache(_Cache):
pass
def host_modules():
"""Fake ``execution`` and ``comfy_execution.caching`` for the cache reader."""
execution = types.ModuleType("execution")
execution.CacheEntry = Entry
class PromptExecutor:
def add_message(self, event, data, broadcast):
return None
execution.PromptExecutor = PromptExecutor
caching = types.ModuleType("comfy_execution.caching")
for cls in (HierarchicalCache, LRUCache, RAMPressureCache, NullCache):
setattr(caching, cls.__name__, cls)
package = types.ModuleType("comfy_execution")
package.caching = caching
return {
"execution": execution,
"comfy_execution": package,
"comfy_execution.caching": caching,
}
def graph(
kind="PromptManager",
text=None,
role="positive",
sampler=True,
negative_text="",
seed=1,
):
"""PrimitiveString -> PreviewAny -> ``kind`` -> KSampler (runtime text).
``text`` set: typed text instead of the runtime link. ``role`` "negative":
the node feeds the sampler's negative input. Pair: its negative editor holds
``negative_text`` and it feeds both sampler inputs.
"""
g = {
"str": {"class_type": "PrimitiveString", "inputs": {"value": "seed text"}},
"llm": {"class_type": "PreviewAny", "inputs": {"source": ["str", 0]}},
"ckpt": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt": "m"}},
"pm": {
"class_type": kind,
"inputs": {"text": ["llm", 0] if text is None else text},
},
"other": {
"class_type": "CLIPTextEncode",
"inputs": {"text": "", "clip": ["ckpt", 1]},
},
"save": {"class_type": "SaveImage", "inputs": {"images": ["ks", 0]}},
}
if kind != "PromptManagerText":
g["pm"]["inputs"]["clip"] = ["ckpt", 1]
if kind == "PromptManagerPair":
g["pm"]["inputs"]["negative_text"] = negative_text
source = ["pm", 0]
if kind == "PromptManagerText":
g["enc"] = {
"class_type": "CLIPTextEncode",
"inputs": {"text": ["pm", 0], "clip": ["ckpt", 1]},
}
source = ["enc", 0]
positive, negative = (source, ["other", 0])
if role == "negative":
positive, negative = negative, positive
if kind == "PromptManagerPair":
positive, negative = ["pm", 0], ["pm", 1]
if sampler:
g["ks"] = {
"class_type": "KSampler",
"inputs": {
"model": ["ckpt", 0],
"positive": positive,
"negative": negative,
"seed": seed,
},
}
else:
g["save"]["inputs"]["images"] = positive
return g
class JobCase(unittest.TestCase):
"""A database, the queue hook stashes and one ``CachedRuns`` per test."""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.addCleanup(assert_closed, self, self.tmp.name)
self.db = PromptDatabase(os.path.join(self.tmp.name, "runs.db"))
self.addCleanup(self.db.close_all)
patcher = mock.patch.dict(sys.modules, host_modules())
patcher.start()
self.addCleanup(patcher.stop)
self.queued, self.pair_queued = QueuedRuns(), QueuedRuns()
self.pending, self.pair_pending = PendingFirstUse(), PendingFirstUse()
self.context = (None, None)
self.runs = CachedRuns(context=lambda: self.context)
self.queue = object()
self.executor = types.SimpleNamespace(
server=types.SimpleNamespace(prompt_queue=self.queue),
caches=types.SimpleNamespace(outputs=HierarchicalCache()),
)
self.item_id = 0
self.trackers = []
# -- nodes ----------------------------------------------------------------
def node(self, kind):
if kind == "PromptManagerPair":
from prompt_manager_pair import PromptManagerPair as cls
elif kind == "PromptManagerText":
from prompt_manager_text import PromptManagerText as cls
else:
import prompt_manager
cls = getattr(prompt_manager, kind)
node = cls.__new__(cls)
node.db = self.db
node.logger = logging.getLogger("test.cached_runs")
node.prompt_tracker = mock.Mock()
self.trackers.append(node.prompt_tracker)
node.comfyui_integration = mock.Mock()
node._inject_lora_trigger_words = lambda text: text
node.pending_first_use = self.pending
node.pending_pair_first_use = self.pair_pending
node.cached_runs = self.runs
return node
def execute(self, g, node_id="pm", text=LLM, negative="", index=None):
"""Run the node as ComfyUI would; returns its STRING outputs."""
kind = g[node_id]["class_type"]
node = self.node(kind)
self.context = (self.context[0], index)
clip = mock.Mock()
# The node receives its switch as the queued graph holds it (#90)
save = g[node_id]["inputs"].get("save_to_library", True)
if kind == "PromptManagerText":
return node.process_text(
text, prompt=g, unique_id=node_id, save_to_library=save
)
if kind == "PromptManagerPair":
return node.encode_prompt(
clip,
text,
negative_text=negative,
prompt=g,
unique_id=node_id,
save_to_library=save,
)
return node.encode_prompt(
clip, text, prompt=g, unique_id=node_id, save_to_library=save
)
def cache(self, node_id, *results):
"""Store executed results (one per list index) as ComfyUI's cache entry."""
width = len(results[0])
outputs = [[result[slot] for result in results] for slot in range(width)]
self.executor.caches.outputs.entries[node_id] = Entry({}, outputs)
# -- jobs -----------------------------------------------------------------
def job(self, g, cached=(), run=None, outputs=("save",), prompt_id=None):
"""One queued job: hook, dequeue, ``execution_cached``, executions, done."""
self.item_id += 1
prompt_id = prompt_id or f"job-{self.item_id}"
request = handle_queued_prompt(
{"prompt": g, "prompt_id": prompt_id},
self.queued,
pair_queued=self.pair_queued,
)
on_dequeue(
request["prompt_id"],
self.queued,
lambda: self.db,
self.pending,
pair_queued=self.pair_queued,
pair_pending=self.pair_pending,
)
self.runs.begin_attempt(
((self.item_id, prompt_id, g, {}, list(outputs), {}), self.item_id)
)
self.context = (prompt_id, None)
self.runs.replay(self.executor, prompt_id, list(cached), lambda: self.db)
result = run() if run else None
self.runs.end_attempt(self.item_id)
self.context = (None, None)
return result
def first(self, g, text=LLM, negative="", node_id="pm"):
"""A job that executes the node and fills the cache."""
result = self.job(
g, run=lambda: self.execute(g, node_id, text=text, negative=negative)
)
self.cache(node_id, result)
return result
def count(self, text):
row = self.db.get_prompt_by_hash(generate_prompt_hash(text))
return row["run_count"] if row else None
def pair_count(self, positive, negative):
pair_id = find_pair_id(self.db, positive, negative)
return self.db.get_prompt_pair(pair_id)["run_count"] if pair_id else None
class TestCachedJobsCount(JobCase):
def test_runtime_text_counts_every_cached_job_for_each_node_kind(self):
for kind in (
"PromptManager",
"PromptManagerFocused",
"PromptManagerSplit",
"PromptManagerText",
):
with self.subTest(kind=kind):
text = f"{LLM} for {kind}"
g = graph(kind)
self.first(g, text=text)
self.job(graph(kind, seed=2), cached=["pm"])
self.job(graph(kind, seed=3), cached=["pm"])
self.assertEqual(self.count(text), 3)
def test_typed_text_still_counts_once_per_job(self):
g = graph(text="typed castle")
self.first(g, text="typed castle")
self.job(g, cached=["pm"])
self.job(g, cached=["pm"])
self.assertEqual(self.count("typed castle"), 3)
def test_rewired_to_negative_while_cached_counts_nothing(self):
self.first(graph())
self.job(graph(role="negative"), cached=["pm"])
self.assertEqual(self.count(LLM), 1)
# Rewired back to positive: the receipt still proves the text
self.job(graph(), cached=["pm"])
self.assertEqual(self.count(LLM), 2)
def test_no_sampler_counts_every_node_on_replay(self):
g = graph(sampler=False)
self.first(g)
self.job(g, cached=["pm"])
self.assertEqual(self.count(LLM), 2)
def test_empty_editor_replays_nothing(self):
g = graph()
self.first(g, text="")
self.job(g, cached=["pm"])
self.assertEqual(self.db.search_prompts(), [])
def test_disabled_node_then_cached_jobs_counts_0(self):
# #90 x #347: a node with Save to library off records no receipt and is
# no run prompt, so its cached jobs replay nothing. The prompt exists
# (an enabled run saved it), so any replayed count would show.
for kind in (
"PromptManager",
"PromptManagerFocused",
"PromptManagerSplit",
"PromptManagerText",
"PromptManagerPair",
):
with self.subTest(kind=kind):
text = f"{LLM} not saved by {kind}"
negative = "blurry" if kind == "PromptManagerPair" else ""
self.first(graph(kind, negative_text=negative), text, negative)
before = (self.count(text), self.pair_count(text, negative))
def off(seed):
g = graph(kind, negative_text=negative, seed=seed)
g["pm"]["inputs"]["save_to_library"] = False
return g
self.first(off(1), text, negative)
self.job(off(2), cached=["pm"])
self.job(off(3), cached=["pm"])
after = (self.count(text), self.pair_count(text, negative))
self.assertEqual(before[0], 1)
self.assertEqual(after, before)
def test_alternating_upstream_values_each_count_their_own_prompt(self):
g = graph()
self.first(g, text="value A")
self.first(g, text="value B")
self.cache("pm", self.execute_outputs("value A"))
self.job(g, cached=["pm"])
self.assertEqual((self.count("value A"), self.count("value B")), (2, 1))
def execute_outputs(self, text):
return (mock.ANY, text)
def test_list_items_replay_each_item_in_order(self):
g = graph()
results = []
def run():
for index, text in enumerate(("one", "two", "three")):
results.append(self.execute(g, text=text, index=index))
self.job(g, run=run)
self.cache("pm", *results)
self.job(g, cached=["pm"])
self.assertEqual([self.count(t) for t in ("one", "two", "three")], [2, 2, 2])
def test_a_repeated_id_or_message_never_counts_a_node_twice(self):
g = graph()
self.first(g)
self.job(g, cached=["pm", "pm"])
self.assertEqual(self.count(LLM), 2)
self.item_id += 1
self.runs.begin_attempt(((1, "again", g, {}, ["save"], {}), self.item_id))
for _ in range(2):
self.runs.replay(self.executor, "again", ["pm"], lambda: self.db)
self.assertEqual(self.count(LLM), 3)
def test_a_shorter_list_keeps_its_receipt_after_a_longer_one(self):
g = graph()
self.first(g, text="alpha")
results = []
def run():
for index, text in enumerate(("alpha", "beta")):
results.append(self.execute(g, text=text, index=index))
self.job(g, run=run)
self.cache("pm", (mock.ANY, "alpha"))
self.job(g, cached=["pm"])
self.assertEqual((self.count("alpha"), self.count("beta")), (3, 1))
def test_a_job_reusing_a_prompt_id_counts_and_executes_normally(self):
g = graph()
self.first(g)
self.job(g, cached=["pm"], prompt_id="same")
self.job(g, run=lambda: self.execute(g), prompt_id="same")
self.job(g, cached=["pm"], prompt_id="same")
self.assertEqual(self.count(LLM), 4)
class TestFailClosed(JobCase):
def setUp(self):
super().setUp()
self.g = graph()
self.first(self.g)
def assertNotCounted(self):
self.assertEqual(self.count(LLM), 1)
def test_changed_cached_text_is_not_counted(self):
self.cache("pm", (mock.ANY, LLM + " edited"))
self.job(self.g, cached=["pm"])
self.assertNotCounted()
def test_no_receipt_in_this_process_is_not_counted(self):
self.runs = CachedRuns(context=lambda: self.context)
self.job(self.g, cached=["pm"])
self.assertNotCounted()
def test_missing_cache_entry_is_not_counted(self):
self.executor.caches.outputs.entries.clear()
self.job(self.g, cached=["pm"])
self.assertNotCounted()
def test_unsupported_cache_is_not_counted(self):
entries = self.executor.caches.outputs.entries
self.executor.caches.outputs = NullCache()
self.executor.caches.outputs.entries = entries
self.job(self.g, cached=["pm"])
self.assertNotCounted()
def test_more_list_values_than_the_bound_are_not_counted(self):
self.cache("pm", *[(mock.ANY, LLM)] * 65)
self.job(self.g, cached=["pm"])
self.assertNotCounted()
def test_missing_list_index_is_not_counted(self):
self.cache("pm", (mock.ANY, LLM), (mock.ANY, LLM))
self.job(self.g, cached=["pm"])
self.assertNotCounted()
def test_node_the_job_does_not_run_is_not_counted(self):
self.job(self.g, cached=["pm"], outputs=("other",))
self.assertNotCounted()
def test_message_for_another_job_is_not_counted(self):
self.item_id += 1
self.runs.begin_attempt(((1, "this", self.g, {}, ["save"], {}), self.item_id))
self.runs.replay(self.executor, "another", ["pm"], lambda: self.db)
self.assertNotCounted()
def test_unkeyable_node_is_not_counted(self):
g = graph()
g["pm"]["inputs"]["extra"] = ["missing", 0]
self.job(g, cached=["pm"])
self.assertIsNone(prompt_node_key(g, "pm"))
self.assertNotCounted()
def test_deleted_prompt_is_skipped(self):
self.db.delete_prompt(
self.db.get_prompt_by_hash(generate_prompt_hash(LLM))["id"]
)
self.job(self.g, cached=["pm"])
self.assertIsNone(self.count(LLM))
def test_database_errors_never_raise(self):
def broken():
raise RuntimeError("database unavailable")
self.item_id += 1
self.runs.begin_attempt(((1, "j", self.g, {}, ["save"], {}), self.item_id))
self.runs.replay(self.executor, "j", ["pm"], broken)
self.assertNotCounted()
def test_malformed_messages_and_results_are_ignored(self):
self.runs.begin_attempt(None)
self.runs.begin_attempt((("x",), "not an int"))
self.runs.replay(self.executor, None, ["pm"], lambda: self.db)
self.item_id += 1
self.runs.begin_attempt(((1, "j", self.g, {}, ["save"], {}), self.item_id))
self.runs.replay(self.executor, "j", "pm", lambda: self.db)
self.runs.replay(self.executor, "j", ["pm"] * 501, lambda: self.db)
self.runs.replay(self.executor, "j", [7, None], lambda: self.db)
self.assertNotCounted()
class TestReplayThenExecution(JobCase):
def test_execution_after_replay_in_the_same_job_is_not_counted_again(self):
# RAM-pressure frees the entry after execution_cached: the node executes
g = graph()
self.first(g)
self.job(g, cached=["pm"], run=lambda: self.execute(g))
self.assertEqual(self.count(LLM), 2)
# The suppressed execution still leaves a receipt: the next job counts
self.job(g, cached=["pm"])
self.assertEqual(self.count(LLM), 3)
def test_suppression_keeps_the_pending_catch_up_and_the_tracker_queue(self):
g = graph()
self.first(g)
prompt_id = self.db.get_prompt_by_hash(generate_prompt_hash(LLM))["id"]
def run():
# The replay counted, then the prompt was deleted and re-saved
self.db.delete_prompt(prompt_id)
self.pending.add(generate_prompt_hash(LLM))
return self.execute(g)
self.job(g, cached=["pm"], run=run)
self.assertEqual(self.count(LLM), 1) # the catch-up, not the own count
push = self.trackers[-1].set_current_prompt.call_args.kwargs["push_to_queue"]
self.assertTrue(push)
def test_without_host_context_the_node_counts_as_before(self):
g = graph()
self.runs = CachedRuns(context=lambda: (None, None))
self.first(g)
self.job(g, cached=["pm"])
self.job(g, run=lambda: self.execute(g))
self.assertEqual(self.count(LLM), 2)
class TestPairReplay(JobCase):
def test_runtime_positive_counts_positive_and_pair_every_job(self):
g = graph("PromptManagerPair", negative_text="blurry")
self.first(g, negative="blurry")
self.job(graph("PromptManagerPair", negative_text="blurry", seed=2), ["pm"])
self.job(graph("PromptManagerPair", negative_text="blurry", seed=3), ["pm"])
self.assertEqual(self.count(LLM), 3)
self.assertEqual(self.pair_count(LLM, "blurry"), 3)
self.assertEqual(self.count("blurry"), 0)
def test_empty_negative_pair_replays_its_pair_count(self):
g = graph("PromptManagerPair")
self.first(g)
self.job(g, cached=["pm"])
self.assertEqual(self.pair_count(LLM, ""), 2)
self.assertEqual(self.count(LLM), 2)
def test_pair_execution_after_replay_is_not_counted_again(self):
g = graph("PromptManagerPair", negative_text="blurry")
self.first(g, negative="blurry")
self.job(g, ["pm"], run=lambda: self.execute(g, negative="blurry"))
self.assertEqual(self.count(LLM), 2)
self.assertEqual(self.pair_count(LLM, "blurry"), 2)
def test_pair_role_follows_the_current_graph(self):
g = graph("PromptManagerPair", negative_text="blurry")
self.first(g, negative="blurry")
rewired = graph("PromptManagerPair", negative_text="blurry")
rewired["ks"]["inputs"]["positive"] = ["other", 0]
self.job(rewired, cached=["pm"])
self.assertEqual(self.count(LLM), 1)
self.assertEqual(self.pair_count(LLM, "blurry"), 1)
class TestOwnCounts(unittest.TestCase):
"""The node path and replay decide through the same function."""
def test_single_node_table(self):
facts = Facts("p")
for used, seen, expected in (
({"text"}, False, [("prompt", "p")]),
({"text"}, True, []),
(set(), False, []),
):
with self.subTest(used=used, seen=seen):
counts = own_counts(
"PromptManager", facts, Role(frozenset(used)), Seen(text=seen)
)
self.assertEqual(counts, expected)
self.assertEqual(
own_counts("PromptManager", Facts(None), Role(frozenset({"text"})), Seen()),
[],
)
def test_pair_table(self):
both = Role(frozenset({"text", "negative_text"}), True)
self.assertEqual(
own_counts("PromptManagerPair", Facts("p", "n"), both, Seen()),
[("prompt", "p"), ("prompt", "n"), ("pair", "p", "n")],
)
self.assertEqual(
own_counts("PromptManagerPair", Facts("p", "p"), both, Seen()),
[("prompt", "p"), ("pair", "p", "p")],
)
self.assertEqual(
own_counts("PromptManagerPair", Facts("p"), both, Seen(text=True)),
[("pair", "p", None)],
)
self.assertEqual(
own_counts(
"PromptManagerPair", Facts("p", "n"), both, Seen(True, True, True)
),
[],
)
def test_node_and_replay_agree_on_role_and_seen(self):
for g, node_id in (
(graph(), "pm"),
(graph(text="typed"), "pm"),
(graph(role="negative"), "pm"),
(graph(sampler=False), "pm"),
(graph("PromptManagerPair", negative_text="n"), "pm"),
(graph("PromptManagerPair", text="typed", negative_text="n"), "pm"),
):
kind = g[node_id]["class_type"]
text = "typed" if isinstance(g[node_id]["inputs"]["text"], str) else LLM
facts = stored_facts(text, g[node_id]["inputs"].get("negative_text"))
with self.subTest(kind=kind, inputs=g[node_id]["inputs"]):
replay = own_counts(
kind,
facts,
graph_role(g, node_id, kind),
graph_seen(g, node_id, kind, facts),
)
self.assertEqual(replay, self.node_counts(g, node_id, kind, text))
@staticmethod
def node_counts(g, node_id, kind, text):
"""What the executing node counted itself, from a recording database."""
db = mock.Mock()
db.get_prompt_by_hash.side_effect = lambda h: {"id": h}
db.save_prompt_pair.return_value = 1
negative = g[node_id]["inputs"].get("negative_text", "")
db.get_prompt_pair.return_value = {
"positive_prompt_id": generate_prompt_hash(text),
"negative_prompt_id": (
generate_prompt_hash(negative) if negative.strip() else None
),
}
case = JobCase()
case.db = db
case.pending, case.pair_pending = PendingFirstUse(), PendingFirstUse()
case.runs = CachedRuns(context=lambda: (None, None))
case.trackers = []
case.context = (None, None)
if kind == "PromptManagerPair":
node = case.node(kind)
node.encode_prompt(
mock.Mock(), text, negative_text=negative, prompt=g, unique_id=node_id
)
else:
node = case.node(kind)
node._track_prompt_execution(text, text, None, None, {}, g, node_id)
counts = [("prompt", c.args[0]) for c in db.record_prompt_use.call_args_list]
if kind == "PromptManagerPair":
counts += [
("pair", generate_prompt_hash(text), db.get_prompt_pair()[k])
for k in ["negative_prompt_id"]
if db.record_prompt_pair_use.called
]
return counts
class TestReceipts(unittest.TestCase):
def setUp(self):
self.context = ("j1", None)
self.runs = CachedRuns(
context=lambda: self.context, max_keys=2, max_receipts=2, max_items=3
)
self.g = {
name: {"class_type": "PromptManager", "inputs": {"text": name}}
for name in ("a", "b", "c")
}
def begin(self, item_id, prompt_id="j1"):
self.context = (prompt_id, None)
self.runs.begin_attempt(
((item_id, prompt_id, self.g, {}, ["a", "b", "c"], {}), item_id)
)
def keys(self):
return list(self.runs._receipts)
def test_least_recently_used_run_keys_are_dropped(self):
self.begin(1)
for name in ("a", "b", "c"):
self.runs.record_item(name, (name,), Facts("h"))
self.assertEqual(self.keys(), [prompt_node_key(self.g, n) for n in ("b", "c")])
def test_receipts_per_key_are_bounded_and_equal_outputs_replace(self):
key = prompt_node_key(self.g, "a")
for item_id, text in ((1, "x"), (2, "y"), (3, "z"), (4, "y")):
self.begin(item_id)
self.runs.record_item("a", (text,), Facts(text))
self.runs.end_attempt(item_id)
receipts = self.runs._receipts[key]
self.assertEqual([r.item_id for r in receipts], [4, 3])
def test_list_indices_past_the_bound_are_not_recorded(self):
self.begin(1)
self.context = ("j1", 3)
self.runs.record_item("a", ("x",), Facts("x"))
self.assertEqual(self.keys(), [])
def test_non_string_outputs_and_unknown_nodes_are_not_recorded(self):
self.begin(1)
self.runs.record_item("a", (None,), Facts("x"))
self.runs.record_item("zzz", ("x",), Facts("x"))
self.assertEqual(self.keys(), [])
def test_task_done_clears_only_its_own_attempt(self):
self.begin(1)
self.runs.end_attempt(2)
self.runs.record_item("a", ("x",), Facts("x"))
self.runs.end_attempt(1)
self.runs.record_item("b", ("x",), Facts("x"))
self.assertEqual(self.keys(), [prompt_node_key(self.g, "a")])
def test_job_nodes_follow_the_outputs_to_execute(self):
self.runs.begin_attempt(((1, "j1", self.g, {}, ["b"], {}), 1))
self.assertEqual(list(self.runs._attempt.nodes), ["b"])
class TestCacheReader(unittest.TestCase):
def setUp(self):
patcher = mock.patch.dict(sys.modules, host_modules())
patcher.start()
self.addCleanup(patcher.stop)
self.cache = LRUCache()
self.executor = types.SimpleNamespace(
caches=types.SimpleNamespace(outputs=self.cache)
)
def read(self, outputs, kind="PromptManager"):
self.cache.entries["n"] = Entry({}, outputs)
return cached_string_outputs(self.executor, "n", kind)
def test_reads_each_string_slot_in_order(self):
self.assertEqual(self.read([["c"], ["a", "b"]]), (("a", "b"),))
self.assertEqual(self.read([["t"]], "PromptManagerText"), (("t",),))
self.assertEqual(
self.read([[1], [2], ["p"], ["n"]], "PromptManagerPair"), (("p",), ("n",))
)
def test_fails_closed(self):
for outputs, kind in (
([["c"], ["a", 1]], "PromptManager"),
([["c"]], "PromptManager"),
([["c"], ["a"] * 65], "PromptManager"),
([[1], [2], ["p", "q"], ["n"]], "PromptManagerPair"),
([["c"], "text"], "PromptManager"),
([["c"], ["a"]], "CLIPTextEncode"),
([[]] * 5, "PromptManager"),
):
with self.subTest(outputs=outputs, kind=kind):
self.assertIsNone(self.read(outputs, kind))
self.cache.entries["m"] = ({}, [["c"], ["a"]]) # not a CacheEntry
self.assertIsNone(cached_string_outputs(self.executor, "m", "PromptManager"))
self.executor.caches.outputs = NullCache()
self.assertIsNone(self.read([["c"], ["a"]]))
class TestGraphHelpers(unittest.TestCase):
def test_upstream_nodes_and_closure(self):
g = graph()
self.assertEqual(
set(upstream_nodes(g, ["save"])),
{"save", "ks", "pm", "llm", "str", "ckpt", "other"},
)
self.assertEqual(upstream_nodes(g, ["missing"]), [])
self.assertEqual(upstream_nodes(None, ["save"]), [])
self.assertIsNone(upstream_closure(g, ["save"], limit=2))
self.assertIsNone(upstream_closure({"a": {"inputs": {"x": ["b", 0]}}}, ["a"]))
def test_find_pair_by_hashes_matches_the_text_lookup(self):
with tempfile.TemporaryDirectory() as tmp:
db = PromptDatabase(os.path.join(tmp, "p.db"))
try:
pair_id = db.save_prompt_pair("cat", "")
self.assertEqual(
find_pair_id_by_hashes(db, generate_prompt_hash("cat"), None),
pair_id,
)
self.assertIsNone(
find_pair_id_by_hashes(
db, generate_prompt_hash("cat"), generate_prompt_hash("x")
)
)
finally:
db.close_all()
assert_closed(self, tmp)
class TestRegistration(unittest.TestCase):
def setUp(self):
self.modules = host_modules()
patcher = mock.patch.dict(sys.modules, self.modules)
patcher.start()
self.addCleanup(patcher.stop)
flag = mock.patch.object(module, "_registered", threading.Event())
flag.start()
self.addCleanup(flag.stop)
class Queue:
def __init__(self):
self.done = []
def get(self, timeout=None):
return self.result
def task_done(self, item_id, *args, **kwargs):
self.done.append(item_id)
def test_registers_once_and_follows_the_job(self):
queue = self.Queue()
server = types.SimpleNamespace(prompt_queue=queue)
runs = mock.Mock()
self.assertTrue(register_cached_runs(server, lambda: "db", runs))
self.assertFalse(register_cached_runs(server, lambda: "db", runs))
queue.result = (("item",), 5)
queue.get()
runs.begin_attempt.assert_called_once_with((("item",), 5))
executor = self.modules["execution"].PromptExecutor()
executor.server = server
executor.add_message("execution_start", {"prompt_id": "j"}, broadcast=False)
executor.add_message(
"execution_cached", {"prompt_id": "j", "nodes": ["n"]}, broadcast=False
)
runs.replay.assert_called_once()
self.assertEqual(runs.replay.call_args.args[1:3], ("j", ["n"]))
other = self.modules["execution"].PromptExecutor()
other.server = types.SimpleNamespace(prompt_queue=object())
other.add_message("execution_cached", {"prompt_id": "j", "nodes": []}, False)
self.assertEqual(runs.replay.call_count, 1)
queue.task_done(5, None, status=None)
runs.end_attempt.assert_called_once_with(5)
def test_hosts_without_the_interfaces_register_nothing(self):
class NoTaskDone:
def get(self):
return None
self.assertFalse(
register_cached_runs(
types.SimpleNamespace(prompt_queue=NoTaskDone()), lambda: None
)
)
self.assertFalse(register_cached_runs(types.SimpleNamespace(), lambda: None))
del self.modules["execution"].PromptExecutor.add_message
self.assertFalse(
register_cached_runs(
types.SimpleNamespace(prompt_queue=self.Queue()), lambda: None
)
)
if __name__ == "__main__":
unittest.main()
-44
View File
@@ -614,50 +614,6 @@ class TestRestoreFromFile(BackupTestCase):
)
assert_closed(self, str(backups[0]))
def test_backup_and_restore_keep_collections(self):
with self.db.model.get_connection() as conn:
conn.execute(
"INSERT INTO image_collections (id, name, name_key, created_at) "
"VALUES (4, 'LoRa testing', 'lora testing', '2026-10-27T00:00:00Z')"
)
conn.execute(
"INSERT INTO image_collection_items VALUES "
"(4, 'root', 'grid.png', '2026-10-27T00:00:00Z')"
)
with tempfile.TemporaryDirectory() as directory:
# The rows still sit in the WAL; a backup must carry them.
backup = os.path.join(directory, "backup.db")
self.assertTrue(self.db.model.backup_database(backup))
self.assertEqual(
PromptModel.verify_database_file(backup, require_library_state=True),
(True, "ok"),
)
with self.db.model.get_connection() as conn:
conn.execute("DELETE FROM image_collections")
self.db.model.restore_from_file(backup)
conn = self.db.model.get_connection()
self.assertEqual(
[
tuple(r)
for r in conn.execute("SELECT id, name FROM image_collections")
],
[(4, "LoRa testing")],
)
self.assertEqual(
[
tuple(r)
for r in conn.execute(
"SELECT collection_id, root_key, relative_path "
"FROM image_collection_items"
)
],
[(4, "root", "grid.png")],
)
PromptModel.close_all_instances()
assert_closed(self, directory)
def test_restore_without_a_live_database_returns_empty_backup_path(self):
src = self._make_source_db(["fresh"])
self.db.close_all()
-456
View File
@@ -1,456 +0,0 @@
"""The prompt-link read of the workspace Images page (#354).
``PromptDatabase.get_image_link_matches`` returns every row of its filter
SELECT in one SQLite step (one JSON value) instead of one step per row, so the
read does not have to win the GIL back once per linked image while ComfyUI
generates. Its result must equal the streamed read it replaces, which stays as
the fallback, on every filter and on odd values.
"""
import itertools
import os
import sqlite3
import sys
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from database import operations # noqa: E402
from database.operations import PromptDatabase, link_filter_query # noqa: E402
try:
from tests.open_handles import assert_closed # noqa: E402
except ImportError: # discovered with tests/ as the top-level directory
from open_handles import assert_closed # noqa: E402
LOGGER = "prompt_manager.database"
MISSING_FUNCTION_SQL = (
"SELECT pm_missing_function(json_group_array(json_array("
"file_path, rating, matched, positive))) FROM ({select})"
)
QUERIES = ("", "lantern", "LANTERN title", "no-such-text")
CATEGORIES = (None, "cat-a", "cat-b", "cat-none")
TAG_SETS = ([], ["t1"], ["t1", "t2"], ["t-none"])
PROMPTS = (
# id, text, category, rating, tags, library title
(1, "red lantern at dusk", "cat-a", 5, ("t1", "t2"), None),
(2, "blue harbor", "cat-a", None, ("t1",), None),
(3, "green field", "cat-b", 3, (), "Lantern title"),
(4, "pair positive", "cat-b", 4, ("t2",), None),
(5, "pair negative lantern", "cat-a", 1, ("t1",), None),
)
QUOTED = '/out/it\'s "quoted".png'
BACKSLASHED = "C:\\out\\back\\slash.png"
UNICODE = "/out/ünïcødé 画像.png"
NEWLINE = "/out/new\nline.png"
IMAGES = (
# prompt id, file path
(1, "/out/a.png"),
(2, QUOTED),
(2, BACKSLASHED),
(3, UNICODE),
(1, NEWLINE),
(1, "/out/ambiguous.png"),
(2, "/out/ambiguous.png"),
(5, "/out/pair.png"), # linked through pair 1: its positive is prompt 4
(3, "/out/two-pairs.png"), # two pairs: no single pair, so prompt 3
(1, None), # no file path: never listed
)
class LinkMatchesTestCase(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.root = Path(os.path.realpath(self.temp.name))
self.db = PromptDatabase(str(self.root / "prompts.db"))
for name in ("_one_step_unsupported", "_one_step_warned"):
patcher = patch.object(operations, name, False)
patcher.start()
self.addCleanup(patcher.stop)
self.statements = []
original = self.db.model.get_connection
def traced_connection():
connection = original()
connection.set_trace_callback(self.statements.append)
return connection
patcher = patch.object(
self.db.model, "get_connection", side_effect=traced_connection
)
patcher.start()
self.addCleanup(patcher.stop)
def tearDown(self):
self.db.close_all()
assert_closed(self, self.root)
self.temp.cleanup()
def seed(self):
with self.db.model.get_connection() as conn:
for prompt_id, text, category, rating, tags, title in PROMPTS:
conn.execute(
"INSERT INTO prompts(id,text,category,rating,hash) "
"VALUES(?,?,?,?,?)",
(prompt_id, text, category, rating, f"hash-{prompt_id}"),
)
for tag in tags:
conn.execute("INSERT OR IGNORE INTO tags(name) VALUES(?)", (tag,))
conn.execute(
"INSERT INTO prompt_tags(prompt_id,tag_id) "
"SELECT ?, id FROM tags WHERE name=?",
(prompt_id, tag),
)
if title:
conn.execute(
"INSERT INTO prompt_library_state(prompt_id,title) "
"VALUES(?,?)",
(prompt_id, title),
)
ids = {}
for prompt_id, path in IMAGES:
cursor = conn.execute(
"INSERT INTO generated_images"
"(prompt_id,image_path,filename,file_path) VALUES(?,?,?,?)",
(prompt_id, path or "/elsewhere/x.png", "x.png", path),
)
ids.setdefault(path, cursor.lastrowid)
conn.execute(
"INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) "
"VALUES(1,4,5),(2,3,5),(3,3,NULL)"
)
conn.executemany(
"INSERT INTO image_prompt_pairs(image_id,pair_id,provenance) "
"VALUES(?,?,'{}')",
[
(ids["/out/pair.png"], 1),
(ids["/out/two-pairs.png"], 2),
(ids["/out/two-pairs.png"], 3),
],
)
def add_image(self, prompt_id, path):
with self.db.model.get_connection() as conn:
conn.execute(
"INSERT INTO generated_images(prompt_id,image_path,filename,file_path)"
" VALUES(?,?,?,?)",
(prompt_id, "/blob/x.png", "x.png", path),
)
def streamed(self, q, category, tags, every_link=False, wanted=None):
"""Today's read, called directly (the fallback)."""
sql, params = link_filter_query(
q, tags, every_link=every_link, category=category
)
with self.db.model.get_connection() as conn:
return operations._streamed_link_matches(
conn, sql, params, wanted, category is not None
)
def selects(self):
found = [s for s in self.statements if s.lstrip().upper().startswith("SELECT")]
self.statements.clear()
return found
class TestOneStepLinkRead(LinkMatchesTestCase):
def test_plain_read_returns_every_unambiguous_link(self):
self.seed()
matches = self.db.get_image_link_matches("", [], every_link=True)
self.assertEqual(
matches,
{
"/out/a.png": (True, 5, 1, None),
QUOTED: (True, None, 2, None),
BACKSLASHED: (True, None, 2, None),
UNICODE: (True, 3, 3, None),
NEWLINE: (True, 5, 1, None),
"/out/pair.png": (True, 4, 4, None),
"/out/two-pairs.png": (True, 3, 3, None),
},
)
self.assertIsNone(matches[QUOTED][1])
self.assertIsInstance(matches["/out/a.png"][1], int)
self.assertIsInstance(matches["/out/a.png"][2], int)
def test_search_hits_follow_text_title_and_tags(self):
self.seed()
self.assertEqual(
self.db.get_image_link_matches("lantern", []),
{
"/out/a.png": (True, 5, 1, None),
NEWLINE: (True, 5, 1, None),
UNICODE: (True, 3, 3, None),
"/out/two-pairs.png": (True, 3, 3, None),
},
)
self.assertEqual(
self.db.get_image_link_matches("lantern", [], category="cat-a"),
{
"/out/a.png": (True, 5, 1, "cat-a"),
NEWLINE: (True, 5, 1, "cat-a"),
QUOTED: (False, None, 2, "cat-a"),
BACKSLASHED: (False, None, 2, "cat-a"),
},
)
def test_every_filter_equals_the_streamed_read(self):
self.seed()
wanted_sets = (None, {"/out/a.png", UNICODE, "/out/pair.png", "/absent.png"})
for q, category, tags, every_link, wanted in itertools.product(
QUERIES, CATEGORIES, TAG_SETS, (False, True), wanted_sets
):
with self.subTest(
q=q, category=category, tags=tags, every=every_link, wanted=wanted
):
expected = self.streamed(q, category, tags, every_link, wanted)
self.statements.clear()
actual = self.db.get_image_link_matches(
q,
list(tags),
every_link=every_link,
wanted=wanted,
category=category,
)
self.assertEqual(actual, expected)
self.assertEqual(
{path: tuple(map(type, row)) for path, row in actual.items()},
{path: tuple(map(type, row)) for path, row in expected.items()},
)
found = self.selects()
self.assertEqual(len(found), 1, found)
self.assertIn("json_group_array", found[0])
def test_empty_database_returns_nothing_in_one_statement(self):
self.assertEqual(self.db.get_image_link_matches("", [], every_link=True), {})
self.assertEqual(self.db.get_image_link_matches("x", ["t"], category="c"), {})
found = self.selects()
self.assertEqual(len(found), 2, found)
self.assertTrue(all("json_group_array" in query for query in found))
def test_wanted_limits_the_result_to_the_listing(self):
self.seed()
self.assertEqual(
self.db.get_image_link_matches(
"", [], every_link=True, wanted={"/out/pair.png", "/x.png"}
),
{"/out/pair.png": (True, 4, 4, None)},
)
self.assertEqual(
self.db.get_image_link_matches("", [], every_link=True, wanted=set()),
{},
)
def test_category_read_carries_the_inherited_category_in_one_step(self):
# A recipe's own category wins over its positive's (#332 MC4); the
# read only narrows to links holding the value, the caller compares.
self.seed()
with self.db.model.get_connection() as conn:
conn.execute("UPDATE prompt_pairs SET category='cat-a' WHERE id=1")
self.statements.clear()
for category, expected in (
("cat-a", {"/out/pair.png": (True, 4, 4, "cat-a")}),
("cat-b", {"/out/pair.png": (True, 4, 4, "cat-a")}),
):
with self.subTest(category=category):
actual = self.db.get_image_link_matches(
"", [], every_link=True, category=category, wanted={"/out/pair.png"}
)
self.assertEqual(actual, expected)
self.assertEqual(
actual, self.streamed("", category, [], True, {"/out/pair.png"})
)
found = [query for query in self.selects() if "json_group_array" in query]
self.assertEqual(len(found), 2, found)
self.assertTrue(all("pair_category" in query for query in found))
def test_plain_read_leaves_the_categories_out_of_the_json(self):
self.seed()
self.statements.clear()
self.db.get_image_link_matches("", [], every_link=True)
(query,) = self.selects()
aggregate = query[query.index("json_array(") :].split(")", 1)[0]
self.assertEqual(aggregate, "json_array(file_path, rating, matched, positive")
def test_read_releases_its_snapshot(self):
# A statement left open after the read would pin its WAL snapshot, and
# a TRUNCATE checkpoint from another connection would report busy.
self.seed()
self.assertTrue(self.db.get_image_link_matches("", [], every_link=True))
other = sqlite3.connect(str(self.root / "prompts.db"))
try:
other.execute("UPDATE prompts SET rating=2 WHERE id=1")
other.commit()
busy, _log, _done = other.execute(
"PRAGMA wal_checkpoint(TRUNCATE)"
).fetchone()
finally:
other.close()
self.assertEqual(busy, 0)
self.assertEqual(os.path.getsize(self.root / "prompts.db-wal"), 0)
def test_wrapped_query_keeps_the_ambiguity_probe_an_index_search(self):
self.seed()
sql, params = link_filter_query(
"lantern", ["t1"], every_link=True, category="cat-a"
)
with self.db.model.get_connection() as conn:
plan = [
row[3]
for row in conn.execute(
"EXPLAIN QUERY PLAN "
+ operations.link_matches_json_query(sql, categories=True),
params,
)
]
self.assertIn(
"SEARCH other USING COVERING INDEX idx_image_file_path (file_path=?)", plan
)
self.assertFalse([step for step in plan if step.startswith("SCAN other")])
class TestStreamedFallback(LinkMatchesTestCase):
def test_missing_json_functions_stream_and_stop_trying(self):
self.seed()
expected = self.streamed("", None, [], True)
self.statements.clear()
wrapper = operations.link_matches_json_query
with (
patch.object(operations, "_LINK_MATCHES_JSON_SQL", MISSING_FUNCTION_SQL),
patch.object(
operations, "link_matches_json_query", side_effect=wrapper
) as attempts,
self.assertLogs(LOGGER, "WARNING") as logged,
):
first = self.db.get_image_link_matches("", [], every_link=True)
found = self.selects()
self.assertEqual(attempts.call_count, 1)
second = self.db.get_image_link_matches("", [], every_link=True)
again = self.selects()
self.assertEqual(first, expected)
self.assertEqual(second, expected)
self.assertEqual(len(logged.records), 1, logged.output)
self.assertIn("no such function: pm_missing_function", logged.output[0])
self.assertTrue(operations._one_step_unsupported)
# SQLite refuses the wrapped statement before it runs (not traced).
self.assertEqual(len(found), 1, found)
self.assertNotIn("json_group_array", found[0])
# The next read skips the attempt for the rest of the process.
self.assertEqual(attempts.call_count, 1)
self.assertEqual(len(again), 1, again)
self.assertNotIn("json_group_array", again[0])
def test_blob_path_streams_that_read_and_keeps_trying(self):
self.seed()
self.add_image(2, b"/out/blob.png")
expected = self.streamed("", None, [], True)
self.assertEqual(expected[b"/out/blob.png"], (True, None, 2, None))
self.statements.clear()
with self.assertLogs(LOGGER, "WARNING") as logged:
first = self.db.get_image_link_matches("", [], every_link=True)
second = self.db.get_image_link_matches("", [], every_link=True)
self.assertEqual(first, expected)
self.assertEqual(second, expected)
self.assertEqual(len(logged.records), 1, logged.output)
self.assertFalse(operations._one_step_unsupported)
found = self.selects()
self.assertEqual(sum("json_group_array" in query for query in found), 2)
self.assertEqual(len(found), 4, found)
def test_blob_that_reads_as_json_still_streams(self):
# SQLite 3.45+ reads a BLOB that is valid JSONB as JSON (b"\x00" is
# null) instead of refusing it; the streamed read keeps the bytes.
self.seed()
self.add_image(2, b"\x00")
expected = self.streamed("", None, [], True)
self.assertIn(b"\x00", expected)
with self.assertLogs(LOGGER, "WARNING"):
self.assertEqual(
self.db.get_image_link_matches("", [], every_link=True), expected
)
self.assertFalse(operations._one_step_unsupported)
def test_blob_rating_streams(self):
self.seed()
with self.db.model.get_connection() as conn:
# Only a database written around the CHECK can hold one.
conn.execute("PRAGMA ignore_check_constraints=ON")
conn.execute("UPDATE prompts SET rating=? WHERE id=3", (b"\x00",))
conn.execute("PRAGMA ignore_check_constraints=OFF")
expected = self.streamed("", None, [], True)
self.assertEqual(expected[UNICODE], (True, b"\x00", 3, None))
with self.assertLogs(LOGGER, "WARNING"):
self.assertEqual(
self.db.get_image_link_matches("", [], every_link=True), expected
)
def test_blob_category_streams_a_category_read(self):
self.seed()
with self.db.model.get_connection() as conn:
# Admitted by its recipe's text category; the positive's is a BLOB.
conn.execute("UPDATE prompts SET category=? WHERE id=4", (b"cat-b",))
conn.execute("UPDATE prompt_pairs SET category='cat-b' WHERE id=1")
expected = self.streamed("", "cat-b", [], True)
self.assertEqual(expected["/out/pair.png"], (True, 4, 4, "cat-b"))
with self.assertLogs(LOGGER, "WARNING") as logged:
actual = self.db.get_image_link_matches(
"", [], every_link=True, category="cat-b"
)
self.assertEqual(actual, expected)
self.assertIn("BLOB", logged.output[0])
self.assertFalse(operations._one_step_unsupported)
def test_json_text_over_the_length_limit_streams(self):
self.seed()
expected = self.streamed("", None, [], True)
conn = self.db.model.get_connection()
limit = conn.setlimit(sqlite3.SQLITE_LIMIT_LENGTH, 200)
try:
with self.assertLogs(LOGGER, "WARNING") as logged:
actual = self.db.get_image_link_matches("", [], every_link=True)
finally:
conn.setlimit(sqlite3.SQLITE_LIMIT_LENGTH, limit)
self.assertEqual(actual, expected)
self.assertIn("too big", logged.output[0])
self.assertFalse(operations._one_step_unsupported)
def test_an_interrupted_read_raises_without_a_fallback(self):
# Move library interrupts a retired connection: the read fails as
# before, and the one warning is kept for a real fallback.
self.seed()
conn = self.db.model.get_connection()
conn.set_progress_handler(lambda: 1, 1)
self.statements.clear()
try:
with (
self.assertNoLogs(LOGGER, "WARNING"),
self.assertRaisesRegex(sqlite3.OperationalError, "^interrupted$"),
):
self.db.get_image_link_matches("", [], every_link=True)
finally:
conn.set_progress_handler(None, 1)
self.assertFalse(operations._one_step_warned)
self.assertFalse(operations._one_step_unsupported)
self.assertFalse([s for s in self.statements if "json_group_array" not in s])
def test_the_maintenance_link_rows_reader_is_not_shadowed(self):
from database.maintenance import MaintenanceOperations
self.assertIs(PromptDatabase._link_rows, MaintenanceOperations._link_rows)
def test_a_failed_streamed_read_still_raises(self):
with (
patch.object(operations, "_LINK_MATCHES_JSON_SQL", MISSING_FUNCTION_SQL),
patch.object(operations, "_LINK_FILTER_SQL", "SELECT broken FROM nowhere"),
self.assertLogs(LOGGER, "WARNING"),
):
with self.assertRaises(sqlite3.OperationalError):
self.db.get_image_link_matches("", [], every_link=True)
if __name__ == "__main__":
unittest.main()
-184
View File
@@ -1,184 +0,0 @@
"""Duplicate cleanup keeps images' own categories (#332 MC11).
The kept copy's own category wins; a kept copy without one receives the
removed copy's. The removed copy's row is never deleted, so a give-back
(the kept copy changed) and a failed delete lose nothing.
"""
import asyncio
import os
from unittest.mock import patch
try:
from tests.test_api_duplicate_files import WAIT, DuplicateFilesTestCase
except ImportError: # discovered with tests/ as the top-level directory
from test_api_duplicate_files import WAIT, DuplicateFilesTestCase
from py.api import duplicate_files # noqa: E402
from py.api.images import image_category_key # noqa: E402
from py.config import GalleryConfig # noqa: E402
class TestDuplicateCleanupKeepsCategories(DuplicateFilesTestCase):
def key(self, path):
return image_category_key(self.api._get_all_output_dirs(), path)
def category(self, path):
return self.api.db.image_category(*self.key(path))
def set_category(self, path, value):
self.api.db.set_image_categories([self.key(path)], value)
async def run_cleanup(self, keep="a.png"):
job, group = await self.found_group()
plan = await self.review(job.id, [self.keep_of(group, keep)])
body = await self.apply(plan["token"])
cleanup = self.api._jobs.get(body["job"]["id"])
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
return cleanup
async def test_kept_copy_without_a_category_receives_the_removed_one(self):
self.trio()
b = self.output_dir / "b.png"
self.set_category(b, "Portraits")
cleanup = await self.run_cleanup()
self.assertFalse(b.exists())
self.assertEqual(self.category(self.output_dir / "a.png"), "Portraits")
# The removed copy's row stays (hidden while its file is gone).
self.assertEqual(self.category(b), "Portraits")
self.assertIn(
"Kept 1 image category on the kept copies", "\n".join(cleanup.log)
)
async def test_kept_copy_keeps_its_own_category(self):
self.trio()
self.set_category(self.output_dir / "a.png", "Landscapes")
self.set_category(self.output_dir / "b.png", "Portraits")
self.set_category(self.output_dir / "sub" / "c.png", "Scans")
cleanup = await self.run_cleanup()
self.assertEqual(self.category(self.output_dir / "a.png"), "Landscapes")
self.assertNotIn("image categor", "\n".join(cleanup.log))
async def test_two_removed_copies_give_the_first_category(self):
self.trio()
self.set_category(self.output_dir / "b.png", "Portraits")
self.set_category(self.output_dir / "sub" / "c.png", "Scans")
cleanup = await self.run_cleanup()
self.assertIn(self.category(self.output_dir / "a.png"), {"Portraits", "Scans"})
self.assertIn(
"Kept 1 image category on the kept copies", "\n".join(cleanup.log)
)
async def test_give_back_loses_no_category(self):
self.trio()
self.set_category(self.output_dir / "b.png", "Portraits")
with patch.object(duplicate_files, "still_kept", return_value=False):
cleanup = await self.run_cleanup()
self.assertEqual(cleanup.counters["removed"], 0)
rows = {row["path"]: row["detail"] for row in cleanup.outcomes["kept"]}
self.assertEqual(
rows["b.png"], duplicate_files.ROW_DETAILS[duplicate_files.KEEPER_GONE]
)
self.assertTrue((self.output_dir / "b.png").is_file())
self.assertEqual(self.category(self.output_dir / "b.png"), "Portraits")
async def test_a_failed_delete_keeps_both_categories(self):
self.trio()
self.set_category(self.output_dir / "b.png", "Portraits")
real = duplicate_files._unlink
def unlink(path):
if os.path.basename(path) == "b.png":
raise PermissionError(13, "denied")
real(path)
with patch.object(duplicate_files, "_unlink", side_effect=unlink):
await self.run_cleanup()
self.assertTrue((self.output_dir / "b.png").is_file())
self.assertEqual(self.category(self.output_dir / "b.png"), "Portraits")
self.assertEqual(self.category(self.output_dir / "a.png"), "Portraits")
async def test_nested_sources_use_the_deepest_source_keys(self):
self.write("sub/a.png", b"same", mtime=1)
self.write("sub/b.png", b"same", mtime=2)
# The subfolder is listed first, so the cleanup's listed root is the
# file key's source; the outer source still holds the files.
GalleryConfig.MONITORING_DIRECTORIES = [
self._root(self.output_dir / "sub"),
self._root(self.output_dir),
]
sub_key = duplicate_files.favorite_root_key(self.output_dir / "sub")
self.api.db.set_image_categories([(sub_key, "b.png")], "Portraits")
await self.run_cleanup()
self.assertFalse((self.output_dir / "sub" / "b.png").exists())
self.assertEqual(self.api.db.image_category(sub_key, "a.png"), "Portraits")
outer_key = duplicate_files.favorite_root_key(self.output_dir)
self.assertIsNone(self.api.db.image_category(outer_key, "sub/a.png"))
async def test_outer_source_listed_first_still_keys_by_the_deepest(self):
self.write("sub/a.png", b"same", mtime=1)
self.write("sub/b.png", b"same", mtime=2)
GalleryConfig.MONITORING_DIRECTORIES = [
self._root(self.output_dir),
self._root(self.output_dir / "sub"),
]
sub_key = duplicate_files.favorite_root_key(self.output_dir / "sub")
self.api.db.set_image_categories([(sub_key, "b.png")], "Portraits")
job, group = await self.found_group()
[kept] = [f for f in group["files"] if f["path"].endswith("a.png")][:1]
plan = await self.review(
job.id, [self.keep_of(group, kept["path"], kept["root_index"])]
)
body = await self.apply(plan["token"])
cleanup = self.api._jobs.get(body["job"]["id"])
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
self.assertFalse((self.output_dir / "sub" / "b.png").exists())
self.assertEqual(self.api.db.image_category(sub_key, "a.png"), "Portraits")
async def test_a_symlinked_output_folder_still_moves_the_category(self):
link = self.output_dir.parent / "linked-output"
try:
os.symlink(self.output_dir, link, target_is_directory=True)
except (OSError, NotImplementedError):
self.skipTest("symlinks are not available")
self.trio()
with patch.object(self.api, "_get_all_output_dirs", return_value=[link]):
key = image_category_key([link], link / "b.png")
self.api.db.set_image_categories([key], "Portraits")
await self.run_cleanup()
kept = image_category_key([link], link / "a.png")
self.assertFalse((self.output_dir / "b.png").exists())
self.assertEqual(self.api.db.image_category(*kept), "Portraits")
async def test_a_copy_with_a_collection_and_a_category_moves_both(self):
# #331 + #332: one removed copy carries both records; one cleanup
# moves its membership and copies its category onto the kept copy.
self.trio()
db = self.api.db
root_key = duplicate_files.favorite_root_key(self.output_dir)
b = self.output_dir / "b.png"
grids = db.add_to_collection(name="LoRA grids", items=[(root_key, "b.png")])
grids = grids["collection"]["id"]
self.set_category(b, "Portraits")
seen = {}
real_log = duplicate_files.DuplicateFilesMixin._log_moved_records
def log_moved_records(job, totals):
seen.update(totals)
real_log(job, totals)
with patch.object(
duplicate_files.DuplicateFilesMixin,
"_log_moved_records",
staticmethod(log_moved_records),
):
cleanup = await self.run_cleanup()
self.assertFalse(b.exists())
self.assertEqual(db.collection_members(grids, root_key), {"a.png"})
self.assertEqual(db.file_collections(root_key, "b.png"), [])
self.assertEqual(self.category(self.output_dir / "a.png"), "Portraits")
self.assertEqual(seen["collections"], 1)
self.assertEqual(seen["categories"], 1)
log = "\n".join(cleanup.log)
self.assertIn("Moved 1 collection membership to the kept copies", log)
self.assertIn("Kept 1 image category on the kept copies", log)
-523
View File
@@ -1,523 +0,0 @@
"""Image categories (#332): schema, migration, restore check, store and file key."""
import hashlib
import ntpath
import os
import sqlite3
import tempfile
import unittest
from contextlib import closing
from pathlib import Path
from unittest.mock import patch
try:
from open_handles import assert_closed
except ImportError:
from tests.open_handles import assert_closed
from database.image_categories import (
COPY_SQL,
INDEX,
STATEMENTS,
TABLE,
copy_image_category,
image_categories_schema_error,
migrate_image_categories,
)
from database.models import PromptModel
from database.operations import PromptDatabase
from py.api.images import image_category_key
_DDL = (
f"CREATE TABLE {TABLE} (root_key TEXT NOT NULL, relative_path TEXT NOT NULL, "
"category TEXT NOT NULL, updated_at TEXT NOT NULL, "
"PRIMARY KEY (root_key, relative_path))"
)
_INDEX_DDL = f"CREATE INDEX {INDEX} ON {TABLE}(category)"
def category_checksum(conn) -> tuple:
"""sha256 of prompts and prompt_pairs ``(id, category, updated_at)``."""
sums = []
for table in ("prompts", "prompt_pairs"):
digest = hashlib.sha256()
for row in conn.execute( # nosec B608 - fixed table names
f"SELECT id, category, updated_at FROM {table} ORDER BY id"
):
digest.update(repr(tuple(row)).encode())
sums.append(digest.hexdigest())
return tuple(sums)
def table_checksums(conn) -> dict:
"""``{name: (schema sql, sha256 of rows)}`` of every table but ours."""
found = {}
for name, sql in conn.execute(
"SELECT name, sql FROM sqlite_master WHERE type='table' AND name != ?",
(TABLE,),
).fetchall():
digest = hashlib.sha256()
rows = conn.execute(f'SELECT * FROM "{name}"') # nosec B608
for text in sorted(repr(tuple(row)) for row in rows):
digest.update(text.encode())
found[name] = (sql, digest.hexdigest())
return found
class _TempDir(unittest.TestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
self.path = os.path.join(self.tmpdir, "categories.db")
def tearDown(self):
PromptModel.close_all_instances()
assert_closed(self, self.tmpdir)
for name in os.listdir(self.tmpdir):
os.unlink(os.path.join(self.tmpdir, name))
os.rmdir(self.tmpdir)
class TestImageCategoriesSchema(_TempDir):
def test_migration_creates_table_and_index(self):
db = PromptDatabase(self.path)
db.close_all()
PromptModel.close_all_instances()
with closing(sqlite3.connect(self.path)) as conn:
names = {
(kind, name)
for kind, name in conn.execute(
"SELECT type, name FROM sqlite_master WHERE tbl_name=?", (TABLE,)
)
}
self.assertEqual(
names,
{
("table", TABLE),
("index", INDEX),
("index", f"sqlite_autoindex_{TABLE}_1"),
},
)
self.assertEqual(
conn.execute(f"SELECT COUNT(*) FROM {TABLE}").fetchone()[0], 0
)
self.assertIsNone(image_categories_schema_error(conn, allow_missing=False))
def test_migration_leaves_prompt_categories_unchanged(self):
"""23,000 links; the migration reads and writes only its own objects."""
db = PromptDatabase(self.path)
db.close_all()
PromptModel.close_all_instances()
with closing(sqlite3.connect(self.path)) as conn:
conn.execute(f"DROP TABLE {TABLE}") # a library from before #332
categories = ["Landscapes", "Portraits", None, " ", "Scans"]
conn.executemany(
"INSERT INTO prompts (text, category, hash, updated_at) "
"VALUES (?, ?, ?, ?)",
[
(f"p{n}", categories[n % 5], f"h{n}", f"2026-01-01 00:00:{n % 60}")
for n in range(200)
],
)
conn.executemany(
"INSERT INTO prompt_pairs (positive_prompt_id, negative_prompt_id, "
"category) VALUES (?, ?, ?)",
[(n, n + 100, categories[(n + 1) % 5]) for n in range(1, 50)],
)
conn.executemany(
"INSERT INTO generated_images (prompt_id, image_path, filename, "
"file_path) VALUES (?, ?, ?, ?)",
[
(n % 200 + 1, f"/out/{n}.png", f"{n}.png", f"/out/{n}.png")
for n in range(23000)
],
)
conn.commit()
before = table_checksums(conn)
prompts_before = category_checksum(conn)
seen = []
def authorize(action, arg1, arg2, _db, _trigger):
seen.append((action, arg1, arg2))
return sqlite3.SQLITE_OK
conn.set_authorizer(authorize)
migrate_image_categories(conn)
conn.commit()
conn.set_authorizer(None)
own = {TABLE, "sqlite_master", "sqlite_schema", INDEX}
touched = set()
for action, arg1, arg2 in seen:
if action in (sqlite3.SQLITE_READ, sqlite3.SQLITE_INSERT):
touched.add(arg1)
if action == sqlite3.SQLITE_READ:
self.assertIn(arg1, own, (action, arg1, arg2))
elif action in (
sqlite3.SQLITE_INSERT,
sqlite3.SQLITE_UPDATE,
sqlite3.SQLITE_DELETE,
):
self.assertIn(arg1, own, (action, arg1, arg2))
elif action == sqlite3.SQLITE_CREATE_TABLE:
self.assertEqual(arg1, TABLE)
elif action == sqlite3.SQLITE_CREATE_INDEX:
self.assertIn(arg1, (INDEX, f"sqlite_autoindex_{TABLE}_1"))
self.assertEqual(arg2, TABLE)
elif action == sqlite3.SQLITE_PRAGMA:
self.assertEqual((arg1, arg2), ("table_info", TABLE))
self.assertIn(TABLE, touched) # the authorizer saw the work
self.assertEqual(table_checksums(conn), before)
self.assertEqual(category_checksum(conn), prompts_before)
schema = conn.execute(
"SELECT sql FROM sqlite_master ORDER BY name"
).fetchall()
migrate_image_categories(conn) # a second run changes nothing
conn.commit()
self.assertEqual(
conn.execute("SELECT sql FROM sqlite_master ORDER BY name").fetchall(),
schema,
)
# The whole startup sequence keeps them too.
PromptDatabase(self.path).close_all()
PromptModel.close_all_instances()
with closing(sqlite3.connect(self.path)) as conn:
self.assertEqual(category_checksum(conn), prompts_before)
self.assertEqual(
conn.execute(f"SELECT COUNT(*) FROM {TABLE}").fetchone()[0], 0
)
def test_structure_accepted_and_refused(self):
variants = {
"missing": None,
"extra column": (
f"CREATE TABLE {TABLE} (root_key TEXT NOT NULL, relative_path TEXT "
"NOT NULL, category TEXT NOT NULL, updated_at TEXT NOT NULL, x TEXT, "
"PRIMARY KEY (root_key, relative_path))",
_INDEX_DDL,
),
"wrong index": (_DDL, f"CREATE INDEX {INDEX} ON {TABLE}(updated_at)"),
"extra index": (
_DDL,
_INDEX_DDL,
f"CREATE INDEX extra ON {TABLE}(root_key)",
),
"trigger": (
_DDL,
_INDEX_DDL,
f"CREATE TRIGGER t AFTER INSERT ON {TABLE} BEGIN SELECT 1; END",
),
"wrong ddl": (
f"CREATE TABLE {TABLE} (root_key TEXT, relative_path TEXT, "
"category TEXT, updated_at TEXT, "
"PRIMARY KEY (root_key, relative_path))",
_INDEX_DDL,
),
"view": (f"CREATE VIEW {TABLE} AS SELECT 1",),
}
for label, statements in variants.items():
with self.subTest(label), closing(sqlite3.connect(":memory:")) as conn:
for sql in statements or ():
conn.execute(sql)
error = image_categories_schema_error(conn)
if statements is None:
self.assertIsNone(error)
self.assertIsNotNone(
image_categories_schema_error(conn, allow_missing=False)
)
continue
self.assertIsNotNone(error)
self.assertIsNotNone(
image_categories_schema_error(conn, check_values=False)
)
with self.assertRaises(sqlite3.DatabaseError):
migrate_image_categories(conn)
# A table without its category index (never written by this code) is
# refused by the check but repaired by the install.
with closing(sqlite3.connect(":memory:")) as conn:
conn.execute(_DDL)
conn.execute(f"INSERT INTO {TABLE} VALUES ('r', 'a.png', 'Scans', 'now')")
self.assertIsNotNone(image_categories_schema_error(conn))
migrate_image_categories(conn)
self.assertIsNone(image_categories_schema_error(conn, allow_missing=False))
self.assertEqual(
conn.execute(f"SELECT COUNT(*) FROM {TABLE}").fetchone()[0], 1
)
with closing(sqlite3.connect(":memory:")) as conn:
migrate_image_categories(conn)
migrate_image_categories(conn) # idempotent
self.assertIsNone(image_categories_schema_error(conn, allow_missing=False))
def test_values_refused_but_startup_checks_structure_only(self):
bad = {
"blank": "",
"101 characters": "x" * 101,
"control character": "a\tb",
"nul": "a\x00b",
"leading space": " Landscapes",
"trailing space": "Landscapes ",
}
for label, value in bad.items():
with self.subTest(label), closing(sqlite3.connect(":memory:")) as conn:
migrate_image_categories(conn)
conn.execute(
f"INSERT INTO {TABLE} VALUES ('r', 'a.png', ?, 'now')", (value,)
)
self.assertIsNotNone(image_categories_schema_error(conn))
self.assertIsNone(
image_categories_schema_error(conn, check_values=False)
)
migrate_image_categories(conn) # startup still opens it
with closing(sqlite3.connect(":memory:")) as conn:
migrate_image_categories(conn)
conn.execute(
f"INSERT INTO {TABLE} VALUES ('r', ?, 'Scans', 'now')", ("x" * 5000,)
)
self.assertIsNotNone(image_categories_schema_error(conn))
conn.execute(f"DELETE FROM {TABLE}")
conn.execute(f"INSERT INTO {TABLE} VALUES ('r', x'00ff', 'Scans', 'now')")
self.assertIsNotNone(image_categories_schema_error(conn))
conn.execute(f"DELETE FROM {TABLE}")
for value in ("x" * 100, "Scans", "Café · 風景", "a b"):
conn.execute(
f"INSERT OR REPLACE INTO {TABLE} VALUES ('r', 'a.png', ?, 'now')",
(value,),
)
self.assertIsNone(image_categories_schema_error(conn), value)
def test_restore_check(self):
PromptDatabase(self.path).close_all()
PromptModel.close_all_instances()
ok, reason = PromptModel.verify_database_file(
self.path, require_library_state=True
)
self.assertTrue(ok, reason)
with closing(sqlite3.connect(self.path)) as conn:
conn.execute(f"DROP TABLE {TABLE}")
conn.commit()
# A backup from before #332 is accepted ...
ok, reason = PromptModel.verify_database_file(self.path)
self.assertTrue(ok, reason)
# ... but a current database must have the table.
ok, reason = PromptModel.verify_database_file(
self.path, require_library_state=True
)
self.assertFalse(ok)
self.assertIn("image categories", reason.lower())
with closing(sqlite3.connect(self.path)) as conn:
conn.execute(_DDL)
conn.execute(_INDEX_DDL)
conn.execute(f"INSERT INTO {TABLE} VALUES ('r', 'a.png', ' x', 'now')")
conn.commit()
ok, reason = PromptModel.verify_database_file(self.path)
self.assertFalse(ok)
self.assertIn("image categories", reason.lower())
class TestImageCategoryStore(_TempDir):
def setUp(self):
super().setUp()
self.db = PromptDatabase(self.path)
def tearDown(self):
self.db.close_all()
super().tearDown()
def rows(self):
return (
self.db.model.get_connection()
.execute(
f"SELECT root_key, relative_path, category, updated_at FROM {TABLE} "
"ORDER BY root_key, relative_path"
)
.fetchall()
)
def test_set_counts_updated_and_unchanged(self):
keys = [("/out", "a.png"), ("/other", "a.png"), ("/out", "a.png")]
self.assertEqual(
self.db.set_image_categories(keys, "Portraits"),
{"updated": 2, "unchanged": 0},
)
stamps = {(r[0], r[1]): r[3] for r in self.rows()}
self.assertEqual(
self.db.set_image_categories(
[("/out", "a.png"), ("/out", "b.png")], "Portraits"
),
{"updated": 1, "unchanged": 1},
)
# An unchanged file keeps its updated_at.
self.assertEqual(
{(r[0], r[1]): r[3] for r in self.rows()}[("/out", "a.png")],
stamps[("/out", "a.png")],
)
self.assertEqual(
self.db.set_image_categories([("/out", "a.png")], "Scans"),
{"updated": 1, "unchanged": 0},
)
self.assertEqual(self.db.image_category("/out", "a.png"), "Scans")
self.assertEqual(self.db.image_category("/other", "a.png"), "Portraits")
self.assertIsNone(self.db.image_category("/out", "missing.png"))
self.assertEqual(
self.db.own_categories_of(["/out", "/out", "/none"]),
{("/out", "a.png"): "Scans", ("/out", "b.png"): "Portraits"},
)
def test_clear_counts_removed_rows(self):
self.db.set_image_categories([("/out", "a.png"), ("/b", "c.png")], "Scans")
self.assertEqual(
self.db.clear_image_categories(
[("/out", "a.png"), ("/b", "c.png"), ("/out", "gone.png")]
),
2,
)
self.assertEqual(self.rows(), [])
self.assertEqual(self.db.clear_image_categories([("/out", "a.png")]), 0)
def test_a_failed_write_writes_nothing(self):
self.db.set_image_categories([("/out", "a.png")], "Scans")
keys = [("/out", "b.png"), ("/out", None)] # NOT NULL fails the second
with self.assertRaises(sqlite3.IntegrityError):
self.db.set_image_categories(keys, "Portraits")
self.assertEqual([r[:3] for r in self.rows()], [("/out", "a.png", "Scans")])
def test_store_never_touches_prompt_categories(self):
conn = self.db.model.get_connection()
with conn:
conn.execute(
"INSERT INTO prompts (text, category, hash) VALUES ('p', 'L', 'h')"
)
before = category_checksum(conn)
seen = set()
conn.set_authorizer(
lambda action, arg1, *_: seen.add(arg1) or sqlite3.SQLITE_OK
)
try:
self.db.set_image_categories([("/out", "a.png")], "Scans")
self.db.own_categories_of(["/out"])
self.db.image_category("/out", "a.png")
self.db.clear_image_categories([("/out", "a.png")])
finally:
conn.set_authorizer(None)
self.assertFalse(seen & {"prompts", "prompt_pairs", "generated_images"}, seen)
self.assertEqual(category_checksum(conn), before)
def test_copy_keeps_kept_copy_category(self):
self.db.set_image_categories([("/out", "removed.png")], "Scans")
self.db.set_image_categories([("/out", "kept.png")], "Portraits")
with self.db._pair_transaction() as conn:
inserted = copy_image_category(
conn, ("/out", "removed.png"), ("/out", "kept.png")
)
self.assertEqual(inserted, 0)
self.assertEqual(self.db.image_category("/out", "kept.png"), "Portraits")
with self.db._pair_transaction() as conn:
inserted = copy_image_category(
conn, ("/out", "removed.png"), ("/out", "new.png")
)
self.assertEqual(inserted, 1)
self.assertEqual(self.db.image_category("/out", "new.png"), "Scans")
# The removed copy's row is never deleted.
self.assertEqual(self.db.image_category("/out", "removed.png"), "Scans")
with self.db._pair_transaction() as conn:
self.assertEqual(
copy_image_category(conn, ("/out", "none.png"), ("/out", "x.png")), 0
)
self.assertEqual(
copy_image_category(
conn, ("/out", "removed.png"), ("/out", "removed.png")
),
0,
)
def test_statements_never_scan_the_table(self):
conn = self.db.model.get_connection()
for sql in STATEMENTS:
with self.subTest(sql):
params = (None,) * sql.count("?")
plan = " ".join(
row[-1] for row in conn.execute(f"EXPLAIN QUERY PLAN {sql}", params)
)
self.assertNotRegex(plan, rf"SCAN {TABLE}\b", plan)
plan = " ".join(
row[-1]
for row in conn.execute(
f"EXPLAIN QUERY PLAN SELECT rowid FROM {TABLE} WHERE category=? "
"ORDER BY rowid",
("x",),
)
)
self.assertIn(INDEX, plan)
self.assertIn(COPY_SQL, STATEMENTS)
class TestImageCategoryKey(unittest.TestCase):
def setUp(self):
self._tmp = tempfile.TemporaryDirectory()
self.base = Path(os.path.realpath(self._tmp.name))
self.outer = self.base / "output"
self.inner = self.outer / "nested"
(self.inner / "sub").mkdir(parents=True)
def tearDown(self):
self._tmp.cleanup()
def test_equals_the_favorite_key_without_nesting(self):
from py.api.images import _favorite_relative_key, _favorite_root_key
key = image_category_key([self.outer], self.outer / "sub" / "a.png")
self.assertEqual(
key, (_favorite_root_key(self.outer), _favorite_relative_key("sub/a.png"))
)
def test_deepest_source_wins_in_any_order(self):
from py.api.images import _favorite_root_key
path = self.inner / "sub" / "a.png"
for sources in ([self.outer, self.inner], [self.inner, self.outer]):
with self.subTest(sources=sources):
self.assertEqual(
image_category_key(sources, path),
(_favorite_root_key(self.inner), "sub/a.png"),
)
# A file of the outer source only is keyed by it.
self.assertEqual(
image_category_key([self.outer, self.inner], self.outer / "b.png"),
(_favorite_root_key(self.outer), "b.png"),
)
def test_outside_every_source_has_no_key(self):
self.assertIsNone(image_category_key([self.inner], self.outer / "a.png"))
self.assertIsNone(image_category_key([], self.outer / "a.png"))
# A sibling sharing a name prefix is not inside.
self.assertIsNone(
image_category_key([self.outer], self.base / "output2" / "a.png")
)
self.assertIsNone(image_category_key([self.outer], self.outer))
def test_symlinked_subfolder_is_not_resolved(self):
target = self.base / "elsewhere"
target.mkdir()
link = self.outer / "linked"
try:
os.symlink(target, link, target_is_directory=True)
except (OSError, NotImplementedError):
self.skipTest("symlinks are not available")
key = image_category_key([self.outer], link / "a.png")
self.assertEqual(key[1], "linked/a.png")
def test_windows_folds_case_of_the_relative_key(self):
with patch("py.api.images._path_flavor", ntpath):
upper = image_category_key([self.outer], self.outer / "Sub" / "A.PNG")
lower = image_category_key([self.outer], self.outer / "sub" / "a.png")
self.assertEqual(upper[1], "sub/a.png")
self.assertEqual(upper, lower)
def test_posix_keeps_case_distinct(self):
if os.path.normcase("A") == "a":
self.skipTest("this OS folds case")
upper = image_category_key([self.outer], self.outer / "A.png")
lower = image_category_key([self.outer], self.outer / "a.png")
self.assertNotEqual(upper, lower)
if __name__ == "__main__":
unittest.main()
-268
View File
@@ -1,268 +0,0 @@
"""Collections (#331): the database operations."""
import sqlite3
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
try:
from tests.open_handles import assert_closed
except ImportError: # discovered with tests/ as the top-level directory
from open_handles import assert_closed
from database.models import PromptModel
from database.operations import PromptDatabase
try: # guarded so a missing module fails assertions, not test loading
from database.image_collections import (
CollectionFilesMissing,
CollectionInvalid,
CollectionMissing,
CollectionNameTaken,
READ_SQL,
)
except ImportError:
CollectionFilesMissing = CollectionMissing = CollectionNameTaken = None
CollectionInvalid = ValueError
READ_SQL = None
ROOT = "/real/output"
OTHER = "/real/other"
class ImageCollectionOperationsTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.addCleanup(assert_closed, self, self.temp.name)
self.addCleanup(PromptModel.reset_schema_cache)
self.db = PromptDatabase(str(Path(self.temp.name) / "prompts.db"))
self.addCleanup(self.db.close_all)
def require(self):
self.assertTrue(
hasattr(self.db, "add_to_collection"),
"ImageCollectionOperations is not implemented",
)
def add(self, items, **target):
return self.db.add_to_collection(items=items, **target)
def test_add_creates_a_collection_by_name(self):
self.require()
result = self.add(
[(ROOT, "a.png"), (ROOT, "b.png"), (OTHER, "a.png")], name="LoRa testing"
)
collection = result["collection"]
self.assertEqual(collection["name"], "LoRa testing")
self.assertEqual(
result,
{"collection": collection, "created": True, "added": 3, "already": 0},
)
self.assertEqual(self.db.list_collections(), [collection])
self.assertTrue(self.db.collection_exists(collection["id"]))
self.assertEqual(self.db.get_collection(collection["id"]), collection)
self.assertEqual(
self.db.collection_members(collection["id"], ROOT), {"a.png", "b.png"}
)
self.assertEqual(
self.db.root_collection_members(OTHER), {collection["id"]: {"a.png"}}
)
self.assertEqual(self.db.collection_counts(), {collection["id"]: 3})
self.assertEqual(self.db.file_collections(ROOT, "a.png"), [collection["id"]])
self.assertEqual(self.db.file_collections(ROOT, "zzz.png"), [])
def test_add_by_name_ignores_case_and_unicode_form(self):
self.require()
first = self.add([(ROOT, "a.png")], name="Café grids")["collection"]
# NFD spelling (macOS) in another case joins the same collection.
second = self.add([(ROOT, "b.png")], name="CAFÉ GRIDS")
self.assertEqual(second["collection"], first)
self.assertFalse(second["created"])
self.assertEqual(len(self.db.list_collections()), 1)
self.assertEqual(
self.db.collection_members(first["id"], ROOT), {"a.png", "b.png"}
)
def test_add_counts_already_present(self):
self.require()
made = self.add([(ROOT, "a.png"), (ROOT, "b.png")], name="Set")
result = self.add(
[(ROOT, "b.png"), (ROOT, "c.png"), (ROOT, "c.png")],
collection_id=made["collection"]["id"],
)
self.assertEqual(
(result["created"], result["added"], result["already"]), (False, 1, 1)
)
self.assertEqual(self.db.collection_counts(), {made["collection"]["id"]: 3})
def test_add_with_a_new_name_and_nothing_inserted_creates_nothing(self):
self.require()
with self.assertRaises(CollectionFilesMissing):
self.add([], name="Empty")
self.assertEqual(self.db.list_collections(), [])
# An existing name with nothing to add is not an error.
made = self.add([(ROOT, "a.png")], name="Kept")["collection"]
self.assertEqual(
self.add([], name="kept"),
{"collection": made, "created": False, "added": 0, "already": 0},
)
def test_add_to_a_deleted_collection_raises_missing(self):
self.require()
made = self.add([(ROOT, "a.png")], name="Gone")["collection"]
self.db.delete_collection(made["id"])
with self.assertRaises(CollectionMissing):
self.add([(ROOT, "b.png")], collection_id=made["id"])
with self.assertRaises(CollectionMissing):
self.db.remove_from_collection(made["id"], [(ROOT, "a.png")])
with self.assertRaises(CollectionMissing):
self.db.rename_collection(made["id"], "Back")
self.assertFalse(self.db.collection_exists(made["id"]))
self.assertIsNone(self.db.get_collection(made["id"]))
def test_remove_returns_the_count(self):
self.require()
made = self.add([(ROOT, "a.png"), (ROOT, "b.png")], name="Set")["collection"]
result = self.db.remove_from_collection(
made["id"], [(ROOT, "a.png"), (ROOT, "a.png"), (ROOT, "never.png")]
)
self.assertEqual(result, {"collection": made, "removed": 1})
self.assertEqual(self.db.collection_members(made["id"], ROOT), {"b.png"})
# Removing the last member keeps the (now empty) collection.
self.db.remove_from_collection(made["id"], [(ROOT, "b.png")])
self.assertEqual(self.db.list_collections(), [made])
self.assertEqual(self.db.collection_counts(), {made["id"]: 0})
def test_rename_refuses_a_taken_name(self):
self.require()
first = self.add([(ROOT, "a.png")], name="First")["collection"]
second = self.add([(ROOT, "a.png")], name="Second")["collection"]
with self.assertRaises(CollectionNameTaken) as caught:
self.db.rename_collection(second["id"], " first ")
self.assertEqual(caught.exception.collection_id, first["id"])
self.assertEqual(self.db.get_collection(second["id"]), second)
def test_rename_accepts_a_case_variant_of_its_own_name(self):
self.require()
made = self.add([(ROOT, "a.png")], name="lora grids")["collection"]
renamed = self.db.rename_collection(made["id"], "LoRA grids ")
self.assertEqual(renamed, {"id": made["id"], "name": "LoRA grids"})
self.assertEqual(self.db.list_collections(), [renamed])
other = self.db.rename_collection(made["id"], "Portraits")
self.assertEqual(other["name"], "Portraits")
def test_delete_unknown_raises_missing(self):
self.require()
made = self.add([(ROOT, "a.png")], name="Doomed")["collection"]
self.assertTrue(self.db.delete_collection(made["id"]))
self.assertEqual(self.db.root_collection_members(ROOT), {})
with self.assertRaises(CollectionMissing):
self.db.delete_collection(made["id"])
with self.assertRaises(CollectionMissing):
self.db.delete_collection(999)
def test_list_is_in_name_key_order(self):
self.require()
for name in ("beta", "Alpha", "été", "Zeta", "alpha two"):
self.add([(ROOT, "a.png")], name=name)
self.assertEqual(
[row["name"] for row in self.db.list_collections()],
["Alpha", "alpha two", "beta", "Zeta", "été"],
)
def test_names_are_trimmed_and_collapsed(self):
self.require()
made = self.add([(ROOT, "a.png")], name=" LoRa \t testing\n ")["collection"]
self.assertEqual(made["name"], "LoRa testing")
for bad in ("", " ", "x" * 101, "bell\x07", "c1\x81", "del\x7f", 5, None):
with self.subTest(name=bad):
with self.assertRaises(CollectionInvalid):
self.add([(ROOT, "a.png")], name=bad)
self.assertEqual(len(self.db.list_collections()), 1)
def test_targets_and_items_are_validated(self):
self.require()
made = self.add([(ROOT, "a.png")], name="Set")["collection"]
for target in (
{},
{"name": "x", "collection_id": made["id"]},
{"collection_id": True},
{"collection_id": 0},
{"collection_id": "1"},
):
with self.subTest(target=target):
with self.assertRaises(CollectionInvalid):
self.add([(ROOT, "a.png")], **target)
for items in ([("root",)], [(ROOT, 5)], "a.png", [(ROOT, "x" * 4097)]):
with self.subTest(items=items):
with self.assertRaises(CollectionInvalid):
self.add(items, collection_id=made["id"])
def test_a_failure_after_the_membership_move_rolls_it_back(self):
self.require()
made = self.add([(ROOT, "b.png")], name="Grids")["collection"]
move = {
"removed": [],
"kept": [],
"targets": {},
"kept_path": "/real/output/a.png",
"favorites": [(ROOT, "b.png", ROOT, "a.png")],
"removed_locator": "/real/output/b.png",
"kept_locator": "/real/output/a.png",
"sha256": "0" * 64,
"kept_size": 1,
"kept_mtime_ns": 1,
}
with patch.object(
type(self.db), "_move_copy_tags", side_effect=RuntimeError("boom")
):
with self.assertRaises(RuntimeError):
self.db.move_copy_records(move)
self.assertEqual(self.db.file_collections(ROOT, "b.png"), [made["id"]])
self.assertEqual(self.db.file_collections(ROOT, "a.png"), [])
# Without the failure the same move succeeds.
self.assertEqual(self.db.move_copy_records(move)["collections"], 1)
self.assertEqual(self.db.file_collections(ROOT, "a.png"), [made["id"]])
def test_count_snapshot_reads_in_one_transaction(self):
self.require()
made = self.add([(ROOT, "a.png"), (OTHER, "b.png")], name="Set")["collection"]
statements = []
conn = self.db.model.get_connection()
conn.set_trace_callback(statements.append)
try:
members, stored = self.db.collection_count_snapshot([ROOT, OTHER, ROOT])
finally:
conn.set_trace_callback(None)
self.assertEqual(
members, {ROOT: {made["id"]: {"a.png"}}, OTHER: {made["id"]: {"b.png"}}}
)
self.assertEqual(stored, {made["id"]: 2})
self.assertEqual(statements[0], "BEGIN")
self.assertEqual(statements[-1], "ROLLBACK")
self.assertFalse(conn.in_transaction)
def test_reads_use_an_index(self):
self.require()
self.add([(ROOT, "a.png")], name="Set")
conn = sqlite3.connect(self.db.model.db_path)
try:
for sql in READ_SQL:
plan = " ".join(
row[3]
for row in conn.execute(
"EXPLAIN QUERY PLAN " + sql, [None] * sql.count("?")
)
)
with self.subTest(sql=sql):
self.assertNotIn("SCAN image_collection_items", plan)
if "?" in sql: # a lookup, not the whole list
self.assertNotIn("SCAN image_collections", plan)
finally:
conn.close()
if __name__ == "__main__":
unittest.main()
-521
View File
@@ -1,521 +0,0 @@
"""Collections schema (#331): install, validation and restore safety."""
import re
import sqlite3
import tempfile
import unittest
import unicodedata
from contextlib import closing
from pathlib import Path
from unittest.mock import patch
try:
from tests.open_handles import assert_closed
from tests.test_annotation_schema import LEGACY_TABLES, table_rows
except ImportError: # discovered with tests/ as the top-level directory
from open_handles import assert_closed
from test_annotation_schema import LEGACY_TABLES, table_rows
from database.models import PromptModel
try: # guarded so a missing module fails assertions, not test loading
from database import image_collections
from database.image_collections import (
image_collections_schema_error,
migrate_image_collections,
)
except ImportError:
image_collections = None
image_collections_schema_error = None
migrate_image_collections = None
COLLECTIONS = "image_collections"
ITEMS = "image_collection_items"
INDEX = "idx_image_collection_items_file"
TABLES = (COLLECTIONS, ITEMS)
DDL = {
COLLECTIONS: (
f"CREATE TABLE {COLLECTIONS} (id INTEGER PRIMARY KEY AUTOINCREMENT, "
"name TEXT NOT NULL, name_key TEXT NOT NULL UNIQUE, "
"created_at TEXT NOT NULL)"
),
ITEMS: (
f"CREATE TABLE {ITEMS} (collection_id INTEGER NOT NULL REFERENCES "
f"{COLLECTIONS}(id) ON DELETE CASCADE, root_key TEXT NOT NULL, "
"relative_path TEXT NOT NULL, added_at TEXT NOT NULL, "
"PRIMARY KEY (collection_id, root_key, relative_path))"
),
}
INDEX_DDL = f"CREATE INDEX {INDEX} ON {ITEMS}(root_key, relative_path)"
NOW = "2026-10-27T00:00:00.000000+00:00"
IMAGES = 23_000
def _compact(sql):
return re.sub(r"\s+", "", sql or "").lower()
def expected_catalog():
return {
("table", COLLECTIONS, _compact(DDL[COLLECTIONS])),
("table", ITEMS, _compact(DDL[ITEMS])),
("index", INDEX, _compact(INDEX_DDL)),
("index", f"sqlite_autoindex_{COLLECTIONS}_1", ""),
("index", f"sqlite_autoindex_{ITEMS}_1", ""),
}
def catalog(conn):
return {
(kind, name, _compact(sql))
for kind, name, sql in conn.execute(
"SELECT type,name,sql FROM sqlite_master "
"WHERE name IN (?,?,?) OR tbl_name IN (?,?)",
(*TABLES, INDEX, *TABLES),
)
}
def key(name):
return unicodedata.normalize("NFC", unicodedata.normalize("NFC", name).casefold())
def schema(conn, exclude=()):
return sorted(
tuple(row)
for row in conn.execute(
"SELECT type,name,tbl_name,sql FROM sqlite_master ORDER BY name"
)
if row[1] not in exclude and row[2] not in exclude
)
def row_counts(conn):
tables = [
name
for (name,) in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
)
]
return {
table: conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # nosec
for table in tables
}
class ImageCollectionsSchemaTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.addCleanup(assert_closed, self, self.temp.name)
self.addCleanup(PromptModel.reset_schema_cache)
self.models = []
self.addCleanup(self.close_models)
self.path = str(Path(self.temp.name) / "live.db")
def close_models(self):
for model in self.models:
model.close_all()
def model(self, path=None):
model = PromptModel(path or self.path)
self.models.append(model)
return model
def require_module(self):
self.assertIsNotNone(
image_collections_schema_error,
"database.image_collections is not implemented",
)
def closed_current(self, name="current.db", *statements):
"""A closed current database, altered afterwards by *statements*."""
path = str(Path(self.temp.name) / name)
model = self.model(path)
model.close_all()
PromptModel.reset_schema_cache()
with closing(sqlite3.connect(path)) as raw, raw:
raw.execute("PRAGMA foreign_keys=OFF")
for statement in statements:
if isinstance(statement, tuple):
raw.execute(*statement)
else:
raw.execute(statement)
return path
@staticmethod
def drop_group(path):
with closing(sqlite3.connect(path)) as raw, raw:
raw.execute(f"DROP TABLE {ITEMS}")
raw.execute(f"DROP TABLE {COLLECTIONS}")
def test_migration_creates_tables_and_index(self):
self.require_module()
model = self.model()
with model.get_connection() as conn:
self.assertEqual(catalog(conn), expected_catalog())
self.assertIsNone(image_collections_schema_error(conn, allow_missing=False))
self.assertEqual(
conn.execute(f"SELECT COUNT(*) FROM {COLLECTIONS}").fetchone()[0], 0
)
self.assertEqual(
conn.execute(f"SELECT COUNT(*) FROM {ITEMS}").fetchone()[0], 0
)
self.assertEqual(
PromptModel.verify_database_file(self.path, require_library_state=True),
(True, "ok"),
)
def test_migration_is_idempotent(self):
self.require_module()
model = self.model()
with model.get_connection() as conn:
conn.execute(
f"INSERT INTO {COLLECTIONS} (name, name_key, created_at) "
"VALUES ('Kept', 'kept', ?)",
(NOW,),
)
conn.execute(f"INSERT INTO {ITEMS} VALUES (1, 'root', 'a.png', ?)", (NOW,))
before = schema(conn)
model.close_all()
for _ in range(2):
PromptModel.reset_schema_cache()
model = self.model()
with model.get_connection() as conn:
migrate_image_collections(conn)
self.assertEqual(schema(conn), before)
self.assertEqual(
[tuple(r) for r in conn.execute(f"SELECT * FROM {ITEMS}")],
[(1, "root", "a.png", NOW)],
)
model.close_all()
def test_migration_reads_only_its_own_tables(self):
self.require_module()
model = self.model()
with model.get_connection() as conn:
conn.execute("INSERT INTO prompts(id,text,hash) VALUES(1,'positive','p')")
conn.executemany(
"INSERT INTO generated_images(prompt_id,image_path,filename,"
"file_path) VALUES(1,?,?,?)",
((f"/out/{n}.png", f"{n}.png", f"/out/{n}.png") for n in range(IMAGES)),
)
conn.execute("INSERT INTO image_favorites VALUES('root','a.png',?)", (NOW,))
model.close_all()
self.drop_group(self.path)
with closing(sqlite3.connect(self.path)) as raw:
raw.execute("PRAGMA foreign_keys=ON")
counts_before = row_counts(raw)
schema_before = schema(raw)
legacy_before = table_rows(raw, LEGACY_TABLES)
self.assertEqual(counts_before["generated_images"], IMAGES)
touched = []
def record(action, arg1, arg2, _db, _trigger):
touched.append((action, arg1, arg2))
return sqlite3.SQLITE_OK
raw.set_authorizer(record)
try:
migrate_image_collections(raw)
finally:
raw.set_authorizer(None)
raw.commit()
reads = {
arg1 for action, arg1, _arg2 in touched if action == sqlite3.SQLITE_READ
}
self.assertTrue(reads)
self.assertLessEqual(reads, {"sqlite_master", COLLECTIONS, ITEMS})
writes = {
arg1
for action, arg1, _arg2 in touched
if action
in (sqlite3.SQLITE_INSERT, sqlite3.SQLITE_UPDATE, sqlite3.SQLITE_DELETE)
}
self.assertLessEqual(writes, {"sqlite_master"})
self.assertEqual(catalog(raw), expected_catalog())
counts_after = row_counts(raw)
self.assertEqual(counts_after.pop(COLLECTIONS), 0)
self.assertEqual(counts_after.pop(ITEMS), 0)
self.assertEqual(counts_after, counts_before)
self.assertEqual(schema(raw, exclude=TABLES), schema_before)
self.assertEqual(table_rows(raw, LEGACY_TABLES), legacy_before)
# A second run is a no-op.
after_first = schema(raw)
changes = raw.total_changes
migrate_image_collections(raw)
raw.commit()
self.assertEqual(schema(raw), after_first)
self.assertEqual(raw.total_changes, changes)
def test_migration_refuses_a_foreign_object_of_the_same_name(self):
self.require_module()
for statements in (
(f"CREATE TABLE {COLLECTIONS} (x)",),
(f"CREATE VIEW {COLLECTIONS} AS SELECT 1",),
(f"CREATE INDEX {INDEX} ON prompts(text)",),
):
with self.subTest(statements=statements):
path = str(Path(self.temp.name) / f"foreign{len(self.models)}.db")
self.model(path).close_all()
PromptModel.reset_schema_cache()
self.drop_group(path)
with closing(sqlite3.connect(path)) as raw, raw:
for statement in statements:
raw.execute(statement)
with closing(sqlite3.connect(path)) as raw:
before = schema(raw)
self.assertIsNotNone(image_collections_schema_error(raw))
with self.assertRaises(sqlite3.DatabaseError):
self.model(path).get_connection()
PromptModel.close_all_instances()
PromptModel.reset_schema_cache()
with closing(sqlite3.connect(path)) as raw:
self.assertEqual(schema(raw), before)
self.assertIsNone(
raw.execute(
"SELECT 1 FROM sqlite_master WHERE name=?", (ITEMS,)
).fetchone()
)
self.assertFalse(PromptModel.verify_database_file(path)[0])
def test_migration_rolls_back_on_error(self):
self.require_module()
self.model().close_all()
PromptModel.reset_schema_cache()
self.drop_group(self.path)
with closing(sqlite3.connect(self.path)) as raw:
with patch.object(
image_collections,
"image_collections_schema_error",
side_effect=[None, "Image collections went wrong"],
):
with self.assertRaisesRegex(sqlite3.DatabaseError, "went wrong"):
migrate_image_collections(raw)
raw.commit()
self.assertEqual(catalog(raw), set())
def test_startup_accepts_a_value_saved_under_older_rules(self):
self.require_module()
path = self.closed_current(
"older.db",
(
f"INSERT INTO {COLLECTIONS} (name, name_key, created_at) "
"VALUES (?, 'not the key', ?)",
(" " + "x" * 150 + " ", NOW),
),
)
model = self.model(path)
with model.get_connection() as conn:
self.assertIsNone(image_collections_schema_error(conn, check_values=False))
self.assertIsNotNone(image_collections_schema_error(conn))
self.assertEqual(
conn.execute(f"SELECT COUNT(*) FROM {COLLECTIONS}").fetchone()[0], 1
)
model.close_all()
self.assertFalse(
PromptModel.verify_database_file(path, require_library_state=True)[0]
)
def test_restore_accepts_an_old_backup_without_collections(self):
self.require_module()
path = self.closed_current("old.db")
self.drop_group(path)
with closing(sqlite3.connect(path)) as raw:
self.assertIsNone(image_collections_schema_error(raw))
self.assertEqual(
image_collections_schema_error(raw, allow_missing=False),
"Missing image collections schema",
)
self.assertEqual(PromptModel.verify_database_file(path), (True, "ok"))
model = self.model(path)
with model.get_connection() as conn:
self.assertEqual(catalog(conn), expected_catalog())
model.close_all()
self.assertEqual(
PromptModel.verify_database_file(path, require_library_state=True),
(True, "ok"),
)
def test_restore_requires_collections_on_a_current_stage(self):
self.require_module()
path = self.closed_current("stage.db")
self.drop_group(path)
self.assertEqual(
PromptModel.verify_database_file(path, require_library_state=True),
(False, "Missing image collections schema"),
)
# One table without the other is never acceptable.
partial = self.closed_current("partial.db", f"DROP TABLE {ITEMS}")
self.assertFalse(PromptModel.verify_database_file(partial)[0])
def assert_refused(self, name, *statements):
self.require_module()
path = self.closed_current(
f"{name}.db",
(
f"INSERT INTO {COLLECTIONS} (id, name, name_key, created_at) "
"VALUES (1, 'Good', 'good', ?)",
(NOW,),
),
(f"INSERT INTO {ITEMS} VALUES (1, 'root', 'a.png', ?)", (NOW,)),
*statements,
)
with closing(sqlite3.connect(path)) as raw:
self.assertIsNone(image_collections_schema_error(raw, check_values=False))
error = image_collections_schema_error(raw)
self.assertIsNotNone(error)
self.assertIn("collections", error.lower())
ok, reason = PromptModel.verify_database_file(path)
self.assertFalse(ok)
self.assertEqual(reason, error)
self.assertFalse(
PromptModel.verify_database_file(path, require_library_state=True)[0]
)
def test_a_valid_current_database_is_accepted(self):
self.require_module()
path = self.closed_current(
"valid.db",
(
f"INSERT INTO {COLLECTIONS} (id, name, name_key, created_at) "
"VALUES (1, ?, ?, ?)",
("LoRa testing", key("LoRa testing"), NOW),
),
(f"INSERT INTO {ITEMS} VALUES (1, 'root', ?, ?)", ("é" * 4096, NOW)),
)
self.assertEqual(
PromptModel.verify_database_file(path, require_library_state=True),
(True, "ok"),
)
def test_restore_refuses_a_non_text_name(self):
# Column affinity turns a number into text; a blob stays a blob.
self.assert_refused("blob-name", f"UPDATE {COLLECTIONS} SET name = x'61'")
def test_restore_refuses_an_empty_name(self):
self.assert_refused("empty", f"UPDATE {COLLECTIONS} SET name = '', name_key=''")
def test_restore_refuses_a_name_over_100_characters(self):
name = "é" * 101
self.assert_refused(
"long",
(f"UPDATE {COLLECTIONS} SET name = ?, name_key = ?", (name, key(name))),
)
def test_restore_refuses_a_control_character(self):
name = "a\x07b"
self.assert_refused(
"control",
(f"UPDATE {COLLECTIONS} SET name = ?, name_key = ?", (name, key(name))),
)
def test_restore_refuses_a_leading_or_trailing_space(self):
for name in (" Good", "Good "):
with self.subTest(name=name):
self.assert_refused(
f"padded{len(name)}{name[0] == ' '}",
(
f"UPDATE {COLLECTIONS} SET name = ?, name_key = ?",
(name, key(name)),
),
)
def test_restore_accepts_a_name_key_from_other_case_tables(self):
# Python versions fold some characters differently; a key written
# by another version must never make a backup unrestorable.
self.require_module()
path = self.closed_current(
"other-key.db",
(
f"INSERT INTO {COLLECTIONS} (id, name, name_key, created_at) "
"VALUES (1, 'Good', 'Good', ?)",
(NOW,),
),
)
self.assertEqual(
PromptModel.verify_database_file(path, require_library_state=True),
(True, "ok"),
)
def test_restore_refuses_an_empty_or_non_text_name_key(self):
self.assert_refused("empty-key", f"UPDATE {COLLECTIONS} SET name_key = ''")
self.assert_refused(
"blob-name-key", f"UPDATE {COLLECTIONS} SET name_key = x'61'"
)
self.assert_refused(
"long-name-key",
(f"UPDATE {COLLECTIONS} SET name_key = ?", ("k" * 1201,)),
)
def test_restore_refuses_a_c1_control_character(self):
for name in ("a\x81b", "a\x7fb"):
with self.subTest(name=repr(name)):
self.assert_refused(
f"c1-{ord(name[1])}",
(
f"UPDATE {COLLECTIONS} SET name = ?, name_key = ?",
(name, key(name)),
),
)
def test_restore_refuses_a_non_text_key(self):
# Column affinity turns a number into text; a blob stays a blob.
self.assert_refused("blob-key", f"UPDATE {ITEMS} SET relative_path = x'61'")
def test_restore_refuses_a_key_over_4096_characters(self):
self.assert_refused(
"long-key", (f"UPDATE {ITEMS} SET root_key = ?", ("é" * 4097,))
)
def test_restore_refuses_a_timestamp_over_64_bytes(self):
self.assert_refused(
"stamp", (f"UPDATE {COLLECTIONS} SET created_at = ?", ("t" * 65,))
)
self.assert_refused(
"item-stamp", (f"UPDATE {ITEMS} SET added_at = ?", ("t" * 65,))
)
def test_restore_refuses_an_item_of_a_missing_collection(self):
self.assert_refused("orphan", f"INSERT INTO {ITEMS} VALUES (9, 'r', 'b', 'n')")
def test_restore_refuses_a_non_integer_collection_id(self):
self.assert_refused("text-id", f"UPDATE {ITEMS} SET collection_id = 'one'")
def test_restore_refuses_a_trigger_or_extra_index(self):
self.require_module()
for name, statement in (
(
"trigger",
f"CREATE TRIGGER t AFTER INSERT ON {ITEMS} BEGIN SELECT 1; END",
),
("extra", f"CREATE INDEX extra ON {COLLECTIONS}(name)"),
):
with self.subTest(name=name):
path = self.closed_current(f"{name}.db", statement)
with closing(sqlite3.connect(path)) as raw:
self.assertIsNotNone(image_collections_schema_error(raw))
self.assertFalse(PromptModel.verify_database_file(path)[0])
def test_deleting_a_collection_deletes_its_members(self):
self.require_module()
model = self.model()
with model.get_connection() as conn:
conn.execute(
f"INSERT INTO {COLLECTIONS} (id, name, name_key, created_at) "
"VALUES (1, 'A', 'a', ?), (2, 'B', 'b', ?)",
(NOW, NOW),
)
conn.executemany(
f"INSERT INTO {ITEMS} VALUES (?, 'root', ?, ?)",
((1, "a.png", NOW), (1, "b.png", NOW), (2, "a.png", NOW)),
)
conn.execute(f"DELETE FROM {COLLECTIONS} WHERE id = 1")
self.assertEqual(
[tuple(r) for r in conn.execute(f"SELECT collection_id FROM {ITEMS}")],
[(2,)],
)
if __name__ == "__main__":
unittest.main()

Some files were not shown because too many files have changed in this diff Show More