Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9d4620e2bb | ||
|
|
049a4ec91c | ||
|
|
f060a39ace | ||
|
|
d73288b37d | ||
|
|
ff98f33079 |
@@ -1,9 +1,9 @@
|
||||
name: Windows
|
||||
name: Windows and macOS
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Non-blocking Windows runs (#420; macOS dropped from CI in #571): never on pull requests or in the merge queue, and no context
|
||||
# Non-blocking portability runs (#420): never on pull requests or in the merge queue, and no context
|
||||
# here is a required check. A red run is fixed at its cause before the workspace cutover (#334).
|
||||
on:
|
||||
push:
|
||||
@@ -25,10 +25,10 @@ env:
|
||||
# run counts (tests/vendor/ci-workflow.test.mjs recomputes it, #493).
|
||||
# Measured 2026-10-29 on main runs 40935337099 (dd247e3), 40938407252 (dd247e3), 40941240624 (dd247e3):
|
||||
# slowest leg: 7m24s (py3.12 windows) / 7m31s (py3.13 windows) / 7m32s (py3.14 windows) -> 15 min
|
||||
# (windows 4m56s-7m32s)
|
||||
# native-trash-linux (#543) has no green main runs yet: it installs the dev requirements and runs
|
||||
# one test module, about 1 minute by the same estimate, so ceil(1.5 x 2) + 3 gives it 6 minutes.
|
||||
# Recalibrate after three green runs.
|
||||
# (windows 4m56s-7m32s, macos 4m39s-6m14s)
|
||||
# native-trash (#499) has no green main runs yet: its probe, run 40949370943 (2026-10-29), took
|
||||
# about 1 minute per macOS leg, so ceil(1.5 x 2) + 3 gives it 6 minutes (step 4). Recalibrate
|
||||
# after three green runs.
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [windows-latest]
|
||||
os: [windows-latest, macos-latest]
|
||||
python-version: ["3.12", "3.13", "3.14"]
|
||||
|
||||
steps:
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
|
||||
# The runner's TEMP is on C:, the OS disk, where a flush took about 6 ms
|
||||
# and a test database's create-to-delete about 111 ms; on the D: work
|
||||
# disk the same took 0.5 ms and 23 ms. Tests write their
|
||||
# disk the same took 0.5 ms and 23 ms (macOS: 19 ms). Tests write their
|
||||
# databases under TEMP, so it moves to a folder on D: (measured in #487,
|
||||
# run 40915866444). The runner's TEMP was an 8.3 short name
|
||||
# (C:\Users\RUNNER~1\...), so the new one is spelled by its short name too
|
||||
@@ -80,12 +80,16 @@ jobs:
|
||||
PM_TEST_SLOWEST: "40"
|
||||
run: python -m tests.run_suite -v
|
||||
|
||||
# The same tests as the fake mount table, on a real ext4 mount used as the output root (#543). A
|
||||
# runner can mount with sudo; the tests run as the normal user, so owner and mode checks are real.
|
||||
native-trash-linux:
|
||||
name: native trash on a real mount
|
||||
runs-on: ubuntu-latest
|
||||
# The real Trash on both macOS CPUs (#499 design §10.2). It runs apart from the suite, so a red
|
||||
# suite never skips the native proof.
|
||||
native-trash:
|
||||
name: native trash on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 6
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [macos-latest, macos-15-intel]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
@@ -94,24 +98,9 @@ jobs:
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
cache-dependency-path: requirements-dev.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Mount an ext4 image
|
||||
run: |
|
||||
truncate -s 64M "$RUNNER_TEMP/drive.img"
|
||||
mkfs.ext4 -q "$RUNNER_TEMP/drive.img"
|
||||
mkdir "$RUNNER_TEMP/drive"
|
||||
sudo mount -o loop "$RUNNER_TEMP/drive.img" "$RUNNER_TEMP/drive"
|
||||
sudo rmdir "$RUNNER_TEMP/drive/lost+found"
|
||||
sudo chown "$(id -u):$(id -g)" "$RUNNER_TEMP/drive"
|
||||
findmnt "$RUNNER_TEMP/drive"
|
||||
|
||||
- name: Native trash proof
|
||||
timeout-minutes: 4
|
||||
run: PM_TRASH_MOUNT="$RUNNER_TEMP/drive" python -m unittest tests.test_trash_mount_root -v
|
||||
env:
|
||||
PM_NATIVE_TRASH: macos
|
||||
run: python -m unittest tests.test_trash_macos_native tests.test_trash_macos -v
|
||||
|
||||
@@ -28,7 +28,7 @@ env:
|
||||
# py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
# 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs.
|
||||
# Windows moved to portability.yml (#420; macOS left CI in #571), which has its own measured limit (#493);
|
||||
# Windows and macOS moved to portability.yml (#420), which has its own measured limit (#493);
|
||||
# this Linux-only matrix keeps 27 minutes until it is re-measured.
|
||||
# real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min
|
||||
# coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min
|
||||
@@ -39,7 +39,7 @@ env:
|
||||
# Pull requests run a fast subset (#420): Python 3.12 on ubuntu-latest, coverage, lint and frontend
|
||||
# modules. The merge queue (merge_group), pushes to main and manual runs add Python 3.13 and 3.14 on
|
||||
# Linux and the real ComfyUI job. "all tests" is the required gate that holds the queue to that set.
|
||||
# Windows runs in portability.yml (push to main, nightly, on demand) and never gate a merge.
|
||||
# Windows and macOS run in portability.yml (push to main, nightly, on demand) and never gate a merge.
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
|
||||
@@ -41,13 +41,9 @@ concurrency:
|
||||
# 7.3m-12.2m), because --shard splits by test count and the slow concept specs sort into shard 1. The page
|
||||
# workflows now run in three shards per browser with the limits unchanged; the per-shard times are
|
||||
# re-measured after three green main runs.
|
||||
# #563: coverage shards reached their 18-minute step limit three times on 2026-10-31 with no failing
|
||||
# test (PR #560 shard 2/3: 814 passed in 17.9m; queue run 41206495451 shard 1/3: 827 passed; PR #564
|
||||
# shard 2/3 twice), against recent green shards of 11m-17m. Coverage now runs in four shards with the
|
||||
# limits unchanged; the per-shard times are re-measured after three green main runs.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/4)
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -63,7 +59,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4]
|
||||
shard: [1, 2, 3]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -92,13 +88,13 @@ jobs:
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
# Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests
|
||||
# run on the lightest shard instead.
|
||||
- if: matrix.shard == 4
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor check
|
||||
- if: matrix.shard == 4
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor test
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/4
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
|
||||
timeout-minutes: 18
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/4
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
|
||||
- name: Upload raw coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
@@ -140,13 +136,13 @@ jobs:
|
||||
with:
|
||||
pattern: coverage-raw-*
|
||||
path: ${{ runner.temp }}/coverage-raw
|
||||
# coverage:merge only checks that raw files exist; merging three of four shards would report
|
||||
# coverage:merge only checks that raw files exist; merging two of three shards would report
|
||||
# partial coverage, so a missing or empty shard artifact fails here.
|
||||
- name: Verify every shard's raw coverage arrived
|
||||
shell: bash
|
||||
env:
|
||||
STAGED_RAW: ${{ runner.temp }}/coverage-raw
|
||||
EXPECTED_SHARDS: 4
|
||||
EXPECTED_SHARDS: 3
|
||||
run: |
|
||||
found=0
|
||||
for shard in $(seq 1 "$EXPECTED_SHARDS"); do
|
||||
|
||||
@@ -74,14 +74,14 @@ V8 files it writes under `coverage/js/raw`:
|
||||
|
||||
```sh
|
||||
PM_TEST_WORKERS=8 npm run --prefix tools/vendor test:coverage # whole run, one report
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 1/4 # node units, viewer suite, workflows 1/4
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 2/4 # workflows 2/4 (likewise 3/4, 4/4)
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 1/3 # node units, viewer suite, workflows 1/3
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 2/3 # workflows 2/3 (likewise 3/3)
|
||||
npm run --prefix tools/vendor coverage:merge # report and baseline check of every raw file
|
||||
npm run --prefix tools/vendor test:workflows -- --project webkit --shard 2/2
|
||||
```
|
||||
|
||||
`coverage:merge` refuses an empty raw directory; it does not count shards, so run
|
||||
every shard first (CI's merge job checks all four arrived). Measured on a 32-CPU
|
||||
every shard first (CI's merge job checks all three arrived). Measured on a 32-CPU
|
||||
machine in the pinned image (2026-10-21): `test:coverage` takes about 5 minutes
|
||||
with 8 workers and 24 minutes with 1; all four projects, 8 workers, each test 3
|
||||
times (6132 tests), 24 minutes. A CI-sized run (`docker run --cpus=4`,
|
||||
@@ -189,14 +189,14 @@ check, so run it before pushing a new or deleted frontend file instead of
|
||||
waiting for the `frontend coverage` jobs.
|
||||
|
||||
The `viewerjs` check runs on every PR, including backend-only changes. It is an
|
||||
aggregate job. It requires the four `frontend coverage (shard N/4)` jobs and
|
||||
aggregate job. It requires the three `frontend coverage (shard N/3)` jobs and
|
||||
`frontend coverage (merge)` to succeed on every event. The four
|
||||
`page workflows (firefox|webkit, shard N/2)` jobs run only in the merge queue,
|
||||
on `main` and on `workflow_dispatch` (see "Pull requests and the merge queue").
|
||||
`viewerjs` requires them to be skipped on a pull request and to succeed
|
||||
everywhere else. Any other failed, cancelled or skipped job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
|
||||
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/4`
|
||||
(shard 1 also runs the node units and the viewer suite, shard 4 the vendor pins
|
||||
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3`
|
||||
(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins
|
||||
and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3
|
||||
days). The merge job downloads them, fails unless all three arrived with raw
|
||||
files and without clashing names, then runs `coverage:merge` (report plus
|
||||
@@ -261,7 +261,7 @@ the process exits non-zero.
|
||||
job fails anyway and the dump is what matters.
|
||||
- `PM_TEST_SLOWEST=N` (off by default) lists the N slowest tests after the
|
||||
summary, each timed from its `start:` line to the next one like the hang
|
||||
limit. The Windows jobs (`portability.yml`) set it to 40, so every
|
||||
limit. The Windows and macOS jobs (`portability.yml`) set it to 40, so every
|
||||
run shows where its time went (#487). The list prints only when the run
|
||||
ends: a job GitHub cancels or a watchdog exit prints none, so read the
|
||||
timestamps between `start:` lines instead.
|
||||
@@ -327,7 +327,7 @@ Python 3.13/3.14, real ComfyUI and Firefox/WebKit failures first show up in the
|
||||
merge queue, on the merge-group commit rather than the
|
||||
pushed one. Find that run with `gh run list --event merge_group --workflow test.yml`
|
||||
(or `viewerjs-check.yml`) and match its head branch
|
||||
(`gh-readonly-queue/main/pr-<N>-...`). Windows failures show up only
|
||||
(`gh-readonly-queue/main/pr-<N>-...`). Windows and macOS failures show up only
|
||||
after the merge, in `gh run list --workflow portability.yml`.
|
||||
|
||||
Start fixing as soon as a failed log shows the cause, but push the fix only after
|
||||
@@ -345,16 +345,11 @@ and the time left. The steps, in order:
|
||||
monitor's callbacks (Needs review recorder, preview queue, Images listing
|
||||
listener).
|
||||
2. `ImageMonitor.stop_monitoring` works against one 5 s deadline. Closing the
|
||||
image handler joins the image worker; watchdog's `observer.stop()` and the
|
||||
observer join get what is left, which can be 0 s. `stop()` joins watchdog's
|
||||
emitters without a limit, so it runs on a daemon thread and is waited for
|
||||
only until the deadline; a later stop waits for that same call (#561).
|
||||
3. `close_auto_thumbnails(final=True)` stops the preview worker (30 s),
|
||||
kills ffmpeg and shuts the interactive image pool down.
|
||||
4. `shutdown_job_pool` (`utils/parallel.py`) cancels the long-lived job pool's
|
||||
queued items and joins its threads (5 s; part `workers`, only when a job
|
||||
used the pool, #523).
|
||||
5. `PromptTracker.shutdown` stops the prompt cleanup thread (2 s).
|
||||
image handler joins the image worker; watchdog's `observer.stop()` (not
|
||||
bounded here) and the observer join get what is left, which can be 0 s.
|
||||
3. `close_auto_thumbnails(final=True)` stops the preview worker (30 s) and
|
||||
kills ffmpeg.
|
||||
4. `PromptTracker.shutdown` stops the prompt cleanup thread (2 s).
|
||||
|
||||
A failing step, or a second Ctrl+C, is logged with its traceback and does not
|
||||
skip the later steps. A clean run logs one INFO line to PromptManager's log:
|
||||
@@ -366,15 +361,6 @@ stored, with the parts not reached False, and the `atexit` callback is
|
||||
removed: the later steps are skipped, no summary line is logged, and a later
|
||||
call returns that result and runs nothing.
|
||||
|
||||
The observer is started from a short-lived daemon thread, so every thread it
|
||||
creates is a daemon (a thread inherits that flag from the thread that creates
|
||||
it). watchdog 7 on Windows starts a plain `threading.Thread` per watch for its
|
||||
`ReadDirectoryChangesW` reader; created from ComfyUI's main thread it would not
|
||||
be a daemon, and the interpreter waits for every such thread before it runs any
|
||||
`atexit` callback, so the teardown would never start (#561).
|
||||
`tests/test_package_exit.py` runs a child whose observer does both (a plain
|
||||
reader thread, a `stop()` that never returns) and requires its exit.
|
||||
|
||||
ComfyUI never runs aiohttp's `on_cleanup` hooks, because it never calls
|
||||
`runner.cleanup()`. So `atexit` is the live host's path. It runs on Ctrl+C
|
||||
(`main.py` catches `KeyboardInterrupt` and returns) and on a normal exit.
|
||||
@@ -459,44 +445,12 @@ A hot SQLite read that returns many rows aggregates them into one step
|
||||
(`json_group_array`) instead of stepping once per row, so it does not wait for
|
||||
the GIL once per row while ComfyUI generates (#354, `get_image_link_matches`).
|
||||
|
||||
A metadata read never decodes pixels (#523). Pillow's `PngImageFile.text` loads
|
||||
every pixel to reach text chunks after IDAT; the scan (`_extract_comfyui_metadata`)
|
||||
and the image monitor (`ComfyUIMetadataExtractor.extract_metadata`) read PNG text
|
||||
through `utils/png_text.py`, which runs Pillow's own chunk handlers after the
|
||||
first IDAT and seeks over pixel data. The legacy `.text` read is the floor:
|
||||
`tests/test_png_text.py` compares both readers on every image under
|
||||
`tests/fixtures`, a curated corpus (text after IDAT, zTXt, iTXt, split IDAT, bad
|
||||
checksums, truncations, oversized text, APNG) and a seeded fuzz, and asserts that
|
||||
neither reader decodes pixels by patching `ImageFile.load`. The only reads that
|
||||
differ are files whose pixel data is undecodable or cut short: legacy found
|
||||
nothing, the walk keeps the text before the pixels. Animated PNGs still use
|
||||
`.text`.
|
||||
|
||||
Image work runs on PromptManager's own threads (#523). glibc keeps each thread's
|
||||
malloc arena at its high-water mark, so image buffers spread over the default
|
||||
executor's threads stay resident. Interactive image work (view previews behind their
|
||||
gate, workspace card facts, both inspector routes) runs on the API's fixed
|
||||
`pm-image` pool (`IMAGE_POOL_SIZE` 3, `_run_image`, `_map_image`; card facts already
|
||||
cached answer without a trip; while the gate's 3 preview writes hold every thread, a
|
||||
card or inspector read waits for one of them, by design of the merged pool; after the
|
||||
package teardown the pool is not recreated and late calls use the default executor). Jobs (scan metadata and file facts, thumbnail and
|
||||
convert jobs, the legacy gallery generation, capture previews) run through
|
||||
`map_parallel` on one long-lived `pm-worker` pool of Worker threads, reused between
|
||||
jobs; a new Worker threads value replaces it (the teardown also joins a replaced pool
|
||||
still finishing), and a call made from a job thread runs inline. After a scan, thumbnail or convert job, or the legacy gallery generation,
|
||||
`release_free_memory` (`utils/native_memory.py`) calls glibc's `malloc_trim(0)` once;
|
||||
it does nothing on Windows, macOS and musl and never raises. No allocator setting
|
||||
changes. `tests/test_image_pools.py` asserts the thread of each read and write and the
|
||||
trim; `tests/test_utils_parallel.py` the pool's reuse, resize, nesting and shutdown.
|
||||
`tools/perf/memory_drive.py` (manual, Linux, never in CI) measures RSS, heap, threads
|
||||
and arenas under the #523 load against the disposable browser test server.
|
||||
|
||||
The `perf budget` job in `.github/workflows/test.yml` is the reference
|
||||
measurement: `ubuntu-latest`, one Python, no coverage, only the wall-clock tests.
|
||||
It runs on every push to `main` and on `workflow_dispatch`, never on pull
|
||||
requests, because shared-runner timing must not block a merge. Dispatch it on a
|
||||
branch to measure before merging.
|
||||
Shared Windows runners do not run them; the structural tests cover
|
||||
Shared Windows and macOS runners do not run them; the structural tests cover
|
||||
portability. Run it locally with
|
||||
`PROMPTMANAGER_PERF_BUDGET=1 python -m unittest tests.test_api_workspace_images.TestWorkspaceImages.test_filter_budget`
|
||||
and quote the printed p95 in the PR body.
|
||||
@@ -692,10 +646,10 @@ Before changing paths, SQLite, streams, restore or cleanup:
|
||||
ordinary busy timeout; cross-thread handoffs wait on events with generous
|
||||
ceilings, never a short wall clock (#242).
|
||||
- Run native Windows CI early for these changes. Pull requests and the merge
|
||||
queue run only Linux, so start Windows on the branch with
|
||||
queue run only Linux, so start Windows and macOS on the branch with
|
||||
`gh workflow run portability.yml --ref <branch>`.
|
||||
Include a Windows impact and evidence note in the PR. All three Windows
|
||||
legs in that run, every Linux leg and every applicable
|
||||
Include a Windows impact and evidence note in the PR. All six Windows/macOS
|
||||
combinations in that run, every Linux leg and every applicable
|
||||
exact-head check must pass before merge. Portable `ntpath`/`PureWindowsPath`
|
||||
controls do not establish native filesystem, locking or cleanup success.
|
||||
|
||||
@@ -710,11 +664,11 @@ what runs:
|
||||
| `py3.13 on ubuntu-latest`, `py3.14 on ubuntu-latest` | not created | run |
|
||||
| `real ComfyUI compatibility` | skipped | runs |
|
||||
| `coverage gate (85%)`, `black and flake8`, `frontend modules`, `bandit (high severity)` | run | run |
|
||||
| `frontend coverage (shard N/4)`, `frontend coverage (merge)` (vendor pins and tooling tests on shard 4) | run | run |
|
||||
| `frontend coverage (shard N/3)`, `frontend coverage (merge)` (vendor pins and tooling tests on shard 3) | run | run |
|
||||
| `page workflows (firefox\|webkit, shard N/2)` | skipped | run |
|
||||
| `all tests`, `viewerjs` (gates) | run | run |
|
||||
| `perf budget` | skipped | push and dispatch only (not in the queue) |
|
||||
| `py3.12-3.14 on windows-latest` (`portability.yml`) | not run | push to `main`, nightly, `workflow_dispatch` only; never required |
|
||||
| `py3.12-3.14 on windows-latest/macos-latest` (`portability.yml`) | not run | push to `main`, nightly, `workflow_dispatch` only; never required |
|
||||
|
||||
GitHub has one required set for pull requests and the queue. A job skipped by
|
||||
its `if:` reports Success. A dropped matrix leg never reports, so it would stay
|
||||
@@ -727,9 +681,8 @@ guarantee:
|
||||
So a merge group cannot turn green unless all three Linux Python legs, real
|
||||
ComfyUI and the four browser shards passed.
|
||||
|
||||
Windows (Python 3.12-3.14) is out of the blocking path for now
|
||||
(maintainer, 2026-10-28). macOS is not tested in CI (maintainer, 2026-10-31, #571);
|
||||
its code stays and support is best-effort. They run in `portability.yml` on every push to `main`,
|
||||
Windows and macOS (Python 3.12-3.14) are out of the blocking path for now
|
||||
(maintainer, 2026-10-28). They run in `portability.yml` on every push to `main`,
|
||||
nightly and on demand, and no context there is required. A red portability run
|
||||
is still a failure: fix it at its cause, not by rerunning, and have every one
|
||||
green before the workspace cutover (#334). `tests/vendor/ci-workflow.test.mjs` pins the
|
||||
@@ -810,7 +763,7 @@ filter Images, D21), the six facts, and a footer with "Send to ComfyUI" and
|
||||
"Prompt history" only when prompt roles resolved or a link is saved; the sticky panel
|
||||
scrolls inside the window. The viewer's details (`#viewer-details`) repeat that
|
||||
content without the cover, then "File & provenance" (the honest file facts, the
|
||||
`Prompt evidence` line, Download original and, for a listed file, D38's
|
||||
`Prompt evidence` line, Download original and, for a listed file, D38's local-only
|
||||
"Show in folder" after the Path) and "Workflow & raw metadata" (raw records and their
|
||||
two downloads), then the pair send and Prompt history buttons. D22 words an empty
|
||||
positive explicitly and makes confirmed prompt text focusable.
|
||||
@@ -1449,15 +1402,6 @@ seeds two seedless files (`seedless_images` on `/__test__/concept/reset`, with
|
||||
Tag) and asserts that every row reserves the tallest footer and that their own footer
|
||||
is shorter than the row it reserves.
|
||||
|
||||
### Overview counts (#547)
|
||||
|
||||
Each Overview card is the count of the page it names: the shell's Library, Images and Tags sidebar counts, one source,
|
||||
so a card and its badge cannot differ. `/prompt_manager/stats` is for the legacy page and counts raw rows. A visit re-reads
|
||||
the three unless a read is in flight. An unread or failed card shows a dash; Retry re-reads only the failed counts.
|
||||
System › Overview's Images metric reads the same shell count (#555), so it, the Overview card and the Images badge show
|
||||
one number; its Statistics dialog labels the legacy row counts as "Database records".
|
||||
Tests: `tests/js/workspace-state.test.js`, `tests/browser/workspace.spec.mjs`, `tests/comfyui/workspace.spec.mjs`.
|
||||
|
||||
### Images page controls and recorded deviations (#249 PR C)
|
||||
|
||||
The Images page renders the concept's DOM with real data. Its controls, top to bottom:
|
||||
@@ -1622,27 +1566,20 @@ Recorded deviations on this page (spec §8):
|
||||
|
||||
| Result | Toast |
|
||||
|---|---|
|
||||
| Success | "Opened the folder on the computer running ComfyUI." |
|
||||
| 403 `not-same-origin` | "The folder was not opened: this page's address does not match the server's." |
|
||||
| 501 `unavailable` | "No file manager is available on the computer running ComfyUI." |
|
||||
| Success | "Opened the folder on this computer." |
|
||||
| 403 `not-local` | "Show in folder works only in a browser on the computer running ComfyUI." |
|
||||
| 501 `unavailable` | "No file manager is available on this computer." |
|
||||
| 429 `busy` | "The folder is already opening." |
|
||||
| Anything else (415, 404, 409, 500, offline) | "The folder could not be opened. Try again." |
|
||||
|
||||
**D176 (#533, beyond the concept):** the viewer is a modal, so `showWorkspaceToast`
|
||||
moves `#workspace-toast` into the topmost open modal (by hit test when several are
|
||||
open), keeping its fixed position, and back when that modal closes. A modal opened
|
||||
after a toast covers it until the next toast. Otherwise every toast raised from a modal is
|
||||
covered and inert while `toHaveText` still passes: assert `toBeVisible` and the
|
||||
hit test as `full-inspector.spec.mjs` does.
|
||||
|
||||
The route opens the folder on the computer running ComfyUI wherever the browser is,
|
||||
through a proxy too (maintainer decision, #533, amending the original local-only
|
||||
rule; accepted risk: anyone who can reach the server can open file windows on it).
|
||||
It answers 403 `not-same-origin` unless `Origin` is present and matches: its host
|
||||
and port against `X-Forwarded-Host` when sent, else `Host`, with the scheme's default
|
||||
port filled in; its scheme against `X-Forwarded-Proto` when sent, else the socket's
|
||||
(an https page on an http socket passes, TLS having ended at a proxy); and, when
|
||||
sent, `Sec-Fetch-Site: same-origin`. A body that is not
|
||||
The route is local-only. It answers 403 `not-local` unless all of these hold: no
|
||||
forwarding-style header (`Forwarded`, `Via`, `X-Real-IP`, `CF-Connecting-IP`,
|
||||
`True-Client-IP`, `X-Client-IP`, `X-Cluster-Client-IP`, `Fastly-Client-IP`, and any
|
||||
`X-Forwarded-*`, `X-Original-*` or `*-Client-IP`); a loopback `request.remote`; a
|
||||
loopback `Host` (`localhost`, `*.localhost` or a loopback address, against DNS
|
||||
rebinding); an `Origin` equal to the request's own scheme, host and port (absent,
|
||||
`null`, another port or another scheme is refused; browsers send `Origin` on every
|
||||
fetch POST); and, when sent, `Sec-Fetch-Site: same-origin`. A body that is not
|
||||
`application/json` is 415 `unsupported-media-type`, so any cross-origin request
|
||||
needs a CORS preflight. The body is strict (an int root, a 64-hex scope, a bounded
|
||||
relative path; absolute, drive and `..` paths are 400), the scope goes through
|
||||
@@ -1659,9 +1596,7 @@ Recorded deviations on this page (spec §8):
|
||||
reach it), falling back to `os.startfile(folder, "explore")`. macOS runs
|
||||
`/usr/bin/open -R`; Linux asks FileManager1 `ShowItems` with `dbus-send
|
||||
--print-reply --reply-timeout=2000` (a real reply or a fallback) and else runs
|
||||
`xdg-open` on the folder, or answers 501 without `DISPLAY` and `WAYLAND_DISPLAY`.
|
||||
`open` and `xdg-open` get `OPENER_WAIT` (1 s): a nonzero exit is the 500 above, and
|
||||
one still running counts as started and goes to the reaper (#533). Those openers are argv lists resolved only on a fixed
|
||||
`xdg-open` on the folder. Those openers are argv lists resolved only on a fixed
|
||||
system PATH, with no shell, DEVNULL streams, `close_fds`, their own session, a
|
||||
minimal environment (no venv PATH, PYTHONPATH or API keys) and a daemon-thread
|
||||
reaper. `tests/test_image_reveal.py` mocks `Popen`, `sys.platform`, `shutil.which`,
|
||||
@@ -1669,11 +1604,12 @@ Recorded deviations on this page (spec §8):
|
||||
(`POST`/`GET /__test__/reveal`) with no cooldown. The inspector comparison reads
|
||||
closed disclosures as hidden, so the button needs no mapping there.
|
||||
|
||||
Limits of that check: a proxy that rewrites `Host` and sends no `X-Forwarded-Host` is
|
||||
refused with that toast until it sends one. A non-browser client can forge every
|
||||
header. The check stops a plain cross-site POST, not every other site: a DNS-rebinding
|
||||
page has a matching `Host` and `Origin`, so it can open a folder of a listed file, at
|
||||
most once a second. The path check confines it, one check ahead of every opener.
|
||||
Residual risk, accepted: a reverse proxy or tunnel that adds no forwarding header
|
||||
(a bare nginx `proxy_pass` to 127.0.0.1, `ssh -R`, `socat`) cannot be told apart
|
||||
from a local client, and a non-browser client behind it can forge a matching
|
||||
`Host` and `Origin`. Such a client can open a folder of a listed file on the host's
|
||||
screen, at most once a second, and learn whether that file exists (which the
|
||||
listing routes already tell it). Operators who expose ComfyUI that way accept this.
|
||||
- **D39** (ruled 2026-10-21: fallback): a recipe-linked image shows its recipe's own
|
||||
category when set, otherwise the linked positive prompt's, and its recipe's own tags
|
||||
when it has any, otherwise the positive's; each field falls back on its own and the
|
||||
@@ -1886,8 +1822,7 @@ Inspect plan keeps D60–D66 (plan D60 → D83, D61 → D84, D65 → D88, D68
|
||||
engines in the mock's order with legacy labels and sizes (`WD14 SwinV2 · ~400
|
||||
MB`, `WD14 ViT · ~400 MB`, `GGUF JoyCaption (Recommended) · ~4.5 GB` in the
|
||||
mock's Ollama slot, `HuggingFace JoyCaption · ~16 GB`); the instructions start
|
||||
from the chosen engine's default saved instructions (#431, D204; before, the
|
||||
server's `default_prompt`).
|
||||
from the server's `default_prompt`.
|
||||
- **D91** (plan D68, ruling T3-bulk): a **Suggestions** number field (1–100) for
|
||||
every model, prefilled from `autotag.defaults.suggestion_limit` (10); a model
|
||||
change keeps it.
|
||||
@@ -2554,28 +2489,7 @@ files to the system trash. **No code path unlinks or overwrites a media file.**
|
||||
(`_collect_media_files`), the legacy auto-tag listing (`_collect_scan_files`) and the
|
||||
delete review (`trash-folder`), and the image monitor, which tests the folders below the
|
||||
watched folder holding the new file (a source named like a trash folder is still watched). Tests: `tests/test_media_folders.py`,
|
||||
`tests/test_image_removal_gaps.py`. Media routes answer not found for a path in such a
|
||||
folder, resolved first so a link or another spelling cannot pass (#543):
|
||||
`_media_access_error` is the one gate (by-id, inspect, reveal, link, rescope as
|
||||
`file-missing`, a prompt's media listing); `/images/serve` and the autotag target ask
|
||||
`in_trash_below_roots` too, `media_target` refuses by name (annotations, collections,
|
||||
whose capture store denies links and aliases), and previews are neither made nor served, the fallback past the preview gate's cap
|
||||
included (`_quick_source`). A History output in one answers `missing`, checked at the
|
||||
candidate and again at the open (a preview can be a link). The legacy duplicate delete
|
||||
removes exactly the folder's real path plus the name (a link itself, never its target)
|
||||
and refuses if either spelling is in a trash folder; `file_problem` refuses too. The delete
|
||||
and Return flows never use this gate. Windows spellings (trailing dots and spaces, case,
|
||||
8.3) run for real on the Windows job in `tests/test_trash_gate_routes.py`.
|
||||
- **An output folder that is a mount point** (#543): the mount's own `.Trash-$uid` and
|
||||
`.Trash/$uid` are allowed only for a root that is exactly that mount, with the spec's
|
||||
checks intact. A root above the mount, a mount nested under the root, a source that is or
|
||||
sits in the trash folder, and a link in place of the trash folder stay refused. Tests:
|
||||
`tests/test_trash_mount_root.py`, on a fake mount table in the suite and, with
|
||||
`PM_TRASH_MOUNT=<real mount>`, in portability.yml's `native-trash-linux` job (loop-mounted
|
||||
ext4). Locally, without root: `unshare -Urm` over a tmpfs.
|
||||
- **D190** (#543, beyond the concept): a trash review where nothing can move shows a primary
|
||||
that clearly cannot act: `aria-disabled`, no danger colors (also on hover), and a press does
|
||||
nothing. Tests: `tests/browser/workspace-images-delete.spec.mjs`.
|
||||
`tests/test_image_removal_gaps.py`.
|
||||
- **Trash backends** (`utils/trash.py`, no dependency): Linux and BSD implement the
|
||||
freedesktop Trash specification (home trash on the same mount, else the mount's
|
||||
`.Trash/$uid` or `.Trash-$uid`), reserve the name with `O_EXCL` and move with
|
||||
@@ -2599,8 +2513,8 @@ files to the system trash. **No code path unlinks or overwrites a media file.**
|
||||
("Moving files to the Recycle Bin isn't available on Windows: files larger than the bin's
|
||||
capacity would be deleted permanently."). Library-only removal works everywhere. Tests use `tests/fake_trash.py`; the
|
||||
browser test server installs it, so no test touches a real trash. The native proof,
|
||||
`tests/test_trash_macos_native.py`, runs only with `PM_NATIVE_TRASH=macos` on a Mac, by hand
|
||||
(CI dropped its `native-trash` macOS job in #571; it last passed on `macos-latest` and `macos-15-intel`): the startup
|
||||
`tests/test_trash_macos_native.py`, runs only with `PM_NATIVE_TRASH=macos` in
|
||||
portability.yml's `native-trash` job (`macos-latest` and `macos-15-intel`): the startup
|
||||
volume, a name clash, fresh APFS and FAT32 disk images, an image used as an output root,
|
||||
a read-only image and an unwritable folder (design and probe:
|
||||
`docs/egm/499-macos-trash/`, local).
|
||||
@@ -2974,15 +2888,6 @@ with its own negative, a self-pair, each drift, validation); `tests/test_library
|
||||
copy, the server after, archive-only, reset to Archive) and `library-delete-concept.spec.mjs`. A Python
|
||||
change: ComfyUI needs a restart.
|
||||
|
||||
### Pair node gives a new prompt its category and tags (#546)
|
||||
|
||||
A Pair save, by a run or the node's Save pair button, writes the category and tags to the positive prompt only when that
|
||||
save creates the prompt row (maintainer ruling). An existing prompt never changes on a run: empty, cleared or set by a person, including one first seen as
|
||||
another pair's negative. The negative stays empty, the saved pair keeps its own metadata, and `save_prompt_pair` writes
|
||||
no prompt metadata unless called with `fill_new_positive`. No backfill on a run; the opt-in repair for prompts older
|
||||
Pair runs saved is System › Maintenance › "Pair category and tags" (#550, "Pair category and tags repair" below).
|
||||
Tests: `tests/test_prompt_pair_node.py`, `tests/test_api_pairs.py`, `tests/comfyui/pair-node.spec.mjs`. A Python change: ComfyUI needs a restart.
|
||||
|
||||
### Delete… on the archived shelf (#492)
|
||||
|
||||
Extends **D160** (coordinator ruling 2026-10-29, cutover item): the archived shelf's bar reads "N selected · Restore ·
|
||||
@@ -3514,20 +3419,15 @@ document title "Review" (the concept's), heading "Metadata review" with its line
|
||||
concept's sentence, then `.gallery-layout`: the queue's concept media cards
|
||||
(`image-card.mjs`, shared with Images; the image opens the shared viewer, Info selects
|
||||
the item, Tag opens the shared Tag image dialog with the review scope note) and
|
||||
`section.surface` "Resolve the association" (filename, Tag image, Link to a prompt…,
|
||||
Sampler context, the chosen path's prompt blocks, Leave for later, Leave unresolved and
|
||||
primary Apply association). The
|
||||
`section.surface` "Resolve the association" (filename, Tag image, Sampler context,
|
||||
the chosen path's prompt blocks, Leave for later and primary Apply association). The
|
||||
first item is selected on load, as the concept's first image is. Sampler context lists
|
||||
"Sampler <id> · <positive> / <negative>" per candidate (the server's label), disables
|
||||
unusable ones with the server's reason and preselects the first workable one
|
||||
(rulings R2/R3). Apply
|
||||
unusable ones, preselects the first workable one and ends with "Leave unresolved"
|
||||
(rulings R2/R3); "Leave unresolved" shows the ambiguous status in both blocks. Apply
|
||||
takes a fresh capture (`observe`) and posts the decision; Leave unresolved on a file
|
||||
under no current root posts without a capture. Each decision and Leave for later opens
|
||||
the next item and the page stays (#536 overturns ruling R7); Leave for later writes
|
||||
nothing. Link to a prompt… (#536) records a Library prompt the reviewer chose,
|
||||
`POST /prompt_manager/review/items/{id}/link`: the capture guards of Apply, the same
|
||||
one-prompt-per-file rules, and link provenance `kind: "manual-link"` with no prompt
|
||||
sides, so it never reads as file evidence. A recorded decision re-reads the row's count, as does every metadata scan that ends. A
|
||||
under no current root posts without a capture. Both actions return to Images (R7); a
|
||||
recorded decision re-reads the row's count, as does every metadata scan that ends. A
|
||||
refused decision keeps the selection and shows the server's sentence (the
|
||||
association conflict's explanation included) as `p.warning-note` in the panel; a
|
||||
restored storage or a changed queue reads the queue again; a request with no answer
|
||||
@@ -3542,14 +3442,6 @@ pending count (`GET /prompt_manager/review/items?limit=1`, its `total`).
|
||||
carries the concept's **Review metadata** (app.js a738 `go-review`): in its Activity
|
||||
item after the item's line, and in its job view's heading after Download report. The
|
||||
item's chip still names its state (D54), where the concept's says "Needs attention".
|
||||
- **D179** (#536, maintainer decision 2026-10-30): "Leave unresolved" is a button beside
|
||||
Leave for later, not the Sampler context select's last option; a disabled option names
|
||||
the server's reason.
|
||||
- **D180** (#536): with nothing to apply, the select, the prompt blocks and Apply are
|
||||
hidden; the panel gives the path count and each reason, and Link to a prompt… is primary.
|
||||
- **D181** (#536): the concept has no picker. Link to a prompt… opens the shared content
|
||||
dialog: Library search, the prompt's recipes as Negative prompt (one preselected,
|
||||
several must be chosen, always "No negative"), Cancel and Link image.
|
||||
- **D119**: the queue cards have no selection checkbox and
|
||||
no star (the page has no selection bar and the queue carries no favorite state); a
|
||||
file under no current root has an empty, `aria-disabled` image button and Tag; more
|
||||
@@ -3895,8 +3787,8 @@ workspace never asks; previews are a cache the server keeps:
|
||||
- `GET /images/serve/thumbnails/<rel>_thumb<ext>` for a missing preview is refused as
|
||||
text first (traversal, a backslash, a drive or stream colon, empty or dot segments, so
|
||||
`D:/x`, `\\server\share` and a leading `/` never reach a joined path), then admitted
|
||||
on the event loop (`_PreviewGate`): 3 previews are written at once on the API's
|
||||
interactive `pm-image` pool (never the default executor; #523), one flight runs per file (the history
|
||||
on the event loop (`_PreviewGate`): 3 previews are written at once on the API's own
|
||||
`pm-preview` pool (never the default executor), one flight runs per file (the history
|
||||
route shares the flight key and the admission), and at most 64 requests wait; one
|
||||
more gets its fallback at once without listing a folder. Inside the admission the
|
||||
preview maps back to its media file under the same root (exact names, then one folder
|
||||
@@ -3929,7 +3821,7 @@ workspace never asks; previews are a cache the server keeps:
|
||||
`exclusive` job runs (`_auto_thumbnail_blocker`), checked when a write starts (after
|
||||
its slot) and again just before its rename.
|
||||
- Shutdown is bounded and logged: the worker stops at a file boundary, ffmpeg is
|
||||
killed, an idle worker leaves (the next output starts a fresh one), the image pool
|
||||
killed, an idle worker leaves (the next output starts a fresh one), the view pool
|
||||
shuts down, and after the app's cleanup a late monitor event
|
||||
starts nothing. A package reload stops the workers the previous import left.
|
||||
- Cards and strips load previews only: Images in every layout and size, Library cards,
|
||||
@@ -4057,11 +3949,7 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
"Save preferences" is `aria-disabled` (opacity .55, `not-allowed`) while clean or
|
||||
saving; a setting the last save could not save gets a red outline. CSS under D92.
|
||||
- **D93** (Overview): real values. The metrics are the Library's eligible positives
|
||||
and active saved negatives and the Images count (#555, amending Q2's indexed
|
||||
`generated_images` rows): the shell's Images sidebar count, the Images page's own
|
||||
total, which also lists files no row links, as the concept's `images.length` is
|
||||
both. A visit re-reads it unless a read is in flight; "–" until read or after a
|
||||
failed read. `images.indexed` stays in the API and is not shown. "Database
|
||||
and active saved negatives and the indexed `generated_images` rows (Q2). "Database
|
||||
integrity" is "Not checked" until a check ran this session, then "N issues" (checks
|
||||
that found issues, even beside a check that could not run), "Passed" or "Incomplete".
|
||||
The review row is "N ambiguous image links" (the Needs review queue's pending count,
|
||||
@@ -4099,11 +3987,10 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
- **D96** (Statistics; amended by D153): "Usage statistics" without the sample note: the
|
||||
concept's "Runs by category" table without its Avg. rating column (Category / Runs)
|
||||
with the real runs of eligible parents per category ("Uncategorized"), then a second
|
||||
table "Database records" (Total / Count: Prompt records, Tags on prompts, Image
|
||||
records: the raw row counts of the legacy statistics, labelled as rows so they never
|
||||
read as the Library or Images page counts, #555; no Rated images). The API still reports `average_rating` and
|
||||
table "Library totals" (Total / Count: Prompts, Tags, Images from the legacy
|
||||
statistics; no Rated images). The API still reports `average_rating` and
|
||||
`rated_images`; the workspace does not show them.
|
||||
The Overview row says "Runs by category, with database record counts." because
|
||||
The Overview row says "Runs by category, with library totals." because
|
||||
that is what the dialog shows (the concept's own dialog shows only "Runs by
|
||||
category"); per-prompt run counts, prompt tags and dated runs are recorded, but no
|
||||
table of them exists yet. A failed read toasts "Statistics could not
|
||||
@@ -4128,23 +4015,6 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
defaults" ("Restore this browser’s display defaults, then save. Other settings are
|
||||
kept.", button "Reset"), which drafts the gallery's defaults for the browser rows only.
|
||||
The node search timeout shows the server's real value, not the sample 30. CSS under D97.
|
||||
- **D170** (#532, maintainer request 2026-10-30, beyond the concept): every viewer has
|
||||
`button#viewer-keys.icon-btn` right before the D156 view switch (accessible name and
|
||||
`title` "Keyboard shortcuts", a `command` outline icon, hidden at ≤700px with the
|
||||
switch). It opens `dialog#viewer-keys-dialog`, a modal built on demand inside the
|
||||
viewer, so a closed overlay adds nothing to the concept comparisons but the button:
|
||||
`.dialog-header` with × and the keys grouped as Move (← → Space), Image (+ - 0 R H),
|
||||
View (F I ? Esc) and File (D, Delete where the viewer has Delete…), one column at
|
||||
390px. Esc or × closes only the overlay and focus returns to the button; opening it
|
||||
stops the slideshow. ← and → navigate from anywhere in the viewer, on `document` as the
|
||||
legacy gallery does, unless a modifier is held, text is selected, another dialog is on
|
||||
top, or focus is in a video, form field, `pre`, `summary`, or anything in the details
|
||||
but a button or link. The other keys share that gate but allow Shift, and each clicks
|
||||
the toolbar's own control, so an aria-disabled, missing or hidden control does nothing
|
||||
(Rotate on a video, I when Centered, F at ≤700px, Delete without Delete…). `?` opens
|
||||
the overlay directly. Delete only opens the review. Space on a focused button or link
|
||||
activates it instead, and only + and - repeat. `workspace-viewer-keys.spec.mjs` pins
|
||||
this.
|
||||
- **D156** (#450, maintainer request 2026-10-28, beyond the concept): every viewer (the
|
||||
shared `dialog#viewer-dialog` of Images, Library, History, collections and Inspect) has
|
||||
`button#viewer-mode.icon-btn` in `.dialog-header`, right before ✕ (a direct child, so
|
||||
@@ -4309,12 +4179,6 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
primary stays the concept's plain button; the plan it opens has the primary.
|
||||
`duplicates dialog matches the concept` and `duplicates comparison detects mutations`
|
||||
(system-maintenance-concept.spec) pin the mapping.
|
||||
- **D192** (Maintenance, #550, beyond the concept): Database adds "Pair category and
|
||||
tags" after "Unused prompt components", a plain Review plan row (no danger mark: it
|
||||
deletes nothing) with the line "Fill an empty category or tags on prompts older Pair
|
||||
node runs saved, from their saved pairs. Only empty fields of never-edited prompts."
|
||||
`system-maintenance-concept.spec.mjs` maps it (`mapPairRow`) as a copy of Optimize
|
||||
database's row.
|
||||
- **D102** (Maintenance, S7, Q5, #222): Metadata cache clears only the browser entries
|
||||
Prompt Manager owns as a cache (`web/js/cache-ownership.js`). The inventory of every
|
||||
storage writer has none (see "Browser cache ownership" below), so the row's line adds
|
||||
@@ -4394,41 +4258,6 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
makes the page see a restore run and finish deterministically. A refused file toasts "This file can't be restored: <validator reason>.". A
|
||||
review value may wrap anywhere (`#modal-body > dl.meta-grid dd`, CSS under D105), so a
|
||||
long file name keeps the concept's two equal columns at phone width.
|
||||
- **D173** (#535, amends D106; maintainer 2026-10-30): "always asks" now means asks when
|
||||
there is a choice. One eligible window is selected, and a selected window with exactly
|
||||
one eligible node selects that node; two or more nodes are never selected. A node whose
|
||||
requested field is linked is not offered (`linkedRoles` on each destination), and the
|
||||
status line says which case applies: linked, wrong roles, no node, or no window. It
|
||||
counts the other nodes that cannot receive, and the review names the node and window.
|
||||
- **D174** (#535): the window option shows the tab's current title, read at each discovery
|
||||
(a getter on the host options, which the Toolkit reads per discover), without the old
|
||||
"PromptManager —" prefix. Same-named windows get " (window 2)", numbered by host id so the
|
||||
number is stable while the tabs stay loaded. Every node option reads "#<id> <title>", so
|
||||
identical nodes can be told apart. Discovery waits 300 ms, as System does.
|
||||
- **D175** (#535, maintainer 2026-10-30): the concept README previews before applying;
|
||||
production makes Preview optional. Apply is enabled once a window and node are selected
|
||||
and requests the ticket itself, in one click. It stays closed after "Sent to ComfyUI.",
|
||||
so a double click sends once, and after an unknown outcome until a new preview or
|
||||
choice. The host's checks still refuse a linked field or a changed workflow, and the
|
||||
sender also refuses if the destination's `revision` (node object plus graph version)
|
||||
moved since discovery; it is undefined on an older tab, which skips the comparison. A host error reads as a refusal, not "may have applied".
|
||||
- **D191** (#542, beyond the concept; maintainer 2026-10-31, Q542-1 A to 5 A): a window with two or more
|
||||
eligible nodes adds a last Destination node option "All N eligible nodes" (`ALL_NODES` in
|
||||
`prompt-sender.mjs`), never selected for you (D173). Eligible is D173's filter; the nodes it leaves out
|
||||
stay counted in the discovery sentence. One window only. Preview stays optional (D175): Preview changes
|
||||
shows one `section.prompt-sender-node` per node, headed "#<id> <title>", in send order (node id,
|
||||
numeric). Apply previews every node first and writes nothing if any refuses ("#<id> <title>: <reason>
|
||||
Nothing was sent."). Then it sends one node after another. Only the first node of a pass taken in that
|
||||
click keeps its ticket; every other node, and every node of a held Preview (host tickets expire after
|
||||
30 s), takes a fresh ticket that is refused unless its Before texts and composition equal the reviewed
|
||||
ones. That fresh preview checks the discovered revision until something has been posted, and not after,
|
||||
since our own write may move the graph version. Refusals are best effort: "Sent to 2 of 3 nodes in
|
||||
<window>. #12 <title>: <host's refusal>."; Apply stays available and a retry sends to every node again
|
||||
(rewriting the same text). A lost reply stops the run, names that node as unknown and lists the nodes
|
||||
not sent. Closing the dialog stops after the current node. The review sections are labelled by their
|
||||
`h4` node heading. Unit tests in `tests/js/prompt-sender.test.js`, a fake three-node host in
|
||||
`tests/browser/prompt-inspector.spec.mjs`, real Classic and Nodes 2.0 widgets in
|
||||
`tests/comfyui/prompt-sender.spec.mjs`.
|
||||
- **D106** (Integrations, S10, Q8): the send bridge always asks for the ComfyUI window
|
||||
and node, and the nodes always follow ComfyUI's own renderer, so "Default send
|
||||
destination" offers only "Ask each time" and "Host compatibility" only "Follow host
|
||||
@@ -4614,9 +4443,9 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
engine and policy (read-only) and opens "Review tag output images" in the concept's
|
||||
record-changing review (its `maintenancePlan` shell): `h3` "Tag output images", the
|
||||
`dl.meta-grid` Model, Existing tags, Images, Already auto-tagged, Images to tag
|
||||
("about N"), Time, Settings ("General <g> · Character <c>", then " · up to L tags
|
||||
per image" with the preview's saved `suggestion_limit`; for a language model, since
|
||||
D205, an "Instructions" row and "Up to L tags per image") and Backup, the estimate sentence ("Nothing is written until you
|
||||
("about N"), Time, Settings ("General <g> · Character <c>" or "Template of N
|
||||
characters", then " · up to L tags per image" with the preview's saved
|
||||
`suggestion_limit`) and Backup, the estimate sentence ("Nothing is written until you
|
||||
start. The number to tag is an estimate: images changed since they were tagged are
|
||||
tagged again, and missing files are skipped. Activity reports what was tagged.") in the
|
||||
`p.small`, the D55 backup box and Cancel / "Start tagging", which starts the
|
||||
@@ -4638,71 +4467,6 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
Cancel-only review matches the concept review shell` (system-tagging-concept.spec) and
|
||||
the batch tests of workspace-system-tagging.spec; the mutation check renames each grid
|
||||
label in turn.
|
||||
- **D202** (Auto-tagging, #431, beyond the concept): a "Skipped tags" block
|
||||
(`.setting-block`, `data-setting="tag-skipped"`, `data-key="autotag_skipped_tags"`)
|
||||
follows "Suggestions per image": the label and description in the concept's
|
||||
setting-row tokens ("Suggested tags that contain one of these words are hidden.
|
||||
Words are separated by spaces, _ or -; other characters such as ( ) belong to the
|
||||
word."), an "Add a word or words" input (`#tag-skipped-input`) with Add (Enter adds
|
||||
too), the entries as removable chips in the Tag image dialog's D83 chip markup (each
|
||||
with its count, title "Hides N tags", or muted "none" with "Hides no known tag"), and
|
||||
a `role="status"` preview line: "**Hides N tags from suggestions**: <up to 12>[ and K
|
||||
more]. Tags already on images are not changed." ("Checked against library tags only;
|
||||
no WD14 label list is downloaded." before the last sentence without a label list;
|
||||
"No tags are skipped." for an empty list; "Checking…" while waiting; "The preview
|
||||
could not be read." on failure). The client refuses an entry with "Enter a word.",
|
||||
"No commas or line breaks.", "At most 50 characters.", "Already in the list." or "The
|
||||
list is full (200)." under the input. The Tag image dialog's chip hint adds "N skipped
|
||||
tags were not suggested." when the server hid any (the static mock's copy). Pinned by
|
||||
the comparison's mapping (system-tagging-concept.spec: the block's tokens are the
|
||||
concept row's), `skipped tags add, preview, remove, save and reload (D202)`, `skipped
|
||||
tags preview checks library tags only …` (workspace-system-tagging.spec) and
|
||||
`suggestions omit skipped tags and say how many were hidden` (workspace-tagging.spec).
|
||||
- **D203** (Auto-tagging, #431, beyond the concept): "Saved instructions" replaces the
|
||||
concept's "LLM tag template" field (the mock's state 2): a `.setting-block`
|
||||
(`data-setting="tag-instructions"`, `data-key="autotag_instructions"`) with the label,
|
||||
"Instructions for the language-model engines. Built-in can’t be changed; duplicate it
|
||||
to make your own. N of 20 saved · Built‑in is not counted.", a `role="listbox"` list
|
||||
(Built-in first with a "Read-only" chip, then the saved entries; "Default for GGUF" /
|
||||
"Default for HuggingFace" chips), "+ New" ("New instructions", "(2)" if taken),
|
||||
"Duplicate" ("<name> copy"), "Delete" (`.danger`, disabled for Built-in), "At most 20
|
||||
saved instructions." at the cap, and the editor beside the list (under it below 700px):
|
||||
"Name" ("N of 60") and "Text" ("N of 4,000 characters"), read-only for Built-in with
|
||||
"Built-in instructions can’t be changed. Duplicate them to make your own.", else the
|
||||
entry's problem ("The name “X” is already used.", "The text must not be empty."…).
|
||||
Then two concept select rows, "Default for GGUF JoyCaption" and "Default for
|
||||
HuggingFace JoyCaption" ("The Tag image dialog and batches start from this.",
|
||||
`data-key="autotag_default_instructions"`), and the note a deleted default leaves
|
||||
("GGUF now uses Built-in."; nothing is written until Save preferences). Pinned by the
|
||||
comparison's mapping (system-tagging-concept.spec: the block in the concept row's
|
||||
tokens, the two rows on the concept's select row) and `saved instructions: new, edit,
|
||||
per-engine default, delete, save and reload (D203)`, `a refused instructions save names
|
||||
the entry and highlights the block (D203)` (workspace-system-tagging.spec).
|
||||
- **D204** (Tag image and Auto-tag selected dialogs, #431, beyond the concept): for a
|
||||
language-model engine a "Saved instructions" select (`#image-tag-instructions-pick`,
|
||||
`#bulk-tag-instructions-pick`) sits above "Tagging instructions" (the mock's state 3),
|
||||
on the concept's select field: Built-in, then the saved entries (`GET /autotag/models`
|
||||
`instructions`, full text), the engine's default marked "<name> · default". It starts
|
||||
at that default; a pick replaces the text. The text stays editable for the run: while
|
||||
it differs from the picked entry the select shows a disabled "Edited for this run" and
|
||||
a muted line under the textarea reads "Edits apply to this run only. Manage saved
|
||||
instructions in System › Auto-tagging." Choosing an entry again replaces the edit.
|
||||
Switching between GGUF and HF starts from the new engine's default and drops an edit;
|
||||
WD14 hides both. A preset (Preview tagging review, the bulk review) shows the entry
|
||||
with its text, else "Edited for this run". When the read fails the select holds only
|
||||
Built-in with the client's copy (`BUILTIN_INSTRUCTIONS`, pinned equal to
|
||||
`DEFAULT_TAG_TEMPLATE` by workspace-image-tagging-state.test). Request bodies are
|
||||
unchanged (`options.instructions` is the textarea's text); there is no "Save as new".
|
||||
The workspace no longer reads `default_prompt`. Pinned by the tagging comparison's
|
||||
mapping (tagging-canonical-comparison.mjs: the select before the instructions),
|
||||
`the saved instructions picker fills, edits one-off and resets per engine (D204)`
|
||||
(workspace-tagging.spec) and `Auto-tag selected picks saved instructions and sends the
|
||||
text shown (D204)` (workspace-tagging-bulk.spec).
|
||||
- **D205** (Auto-tagging, #431; amends D134): for a language-model engine the batch
|
||||
review adds a read-only "Instructions" row with the drafted engine default's name
|
||||
before "Settings", which reads "Up to L tags per image"; the batch and Preview tagging
|
||||
review send that entry's text. WD14 is unchanged. Pinned by `the batch review names the
|
||||
language model's default instructions (D205)` (workspace-system-tagging.spec).
|
||||
- **D158** (Sources, #247 D-Q4): the concept has no switch for prompts from workflows
|
||||
without a PromptManager node; the maintainer chose a visible switch over a config key.
|
||||
A `.setting-row` in the concept's toggle markup follows "Watch for new outputs":
|
||||
@@ -4934,47 +4698,6 @@ replaced the database since the review; the plan is dropped). The job:
|
||||
prompt ids only), logs the backup by its file name, and prunes safety backups to the
|
||||
newest 10. With no backup the log and the status line say "No safety backup".
|
||||
|
||||
#### Pair category and tags repair (#550)
|
||||
|
||||
`pair_metadata` ("Pair category and tags", D192; `database/pair_metadata_repair.py`) fills
|
||||
the empty category and the empty tag set of prompts that Pair node runs before #546
|
||||
saved, from their saved pairs. It is reviewed and applied like the operations above and
|
||||
follows the safety backup policy like them (no forced backup). Rules
|
||||
(`docs/egm/post-3.4.1/550-design.md`):
|
||||
|
||||
- Considered: prompts the Pair node created as a positive: their earliest pair (lowest
|
||||
id, either role, archived included) has them as the positive and its `created_at` is
|
||||
within 2 s of theirs. A prompt first saved by the PromptManager node, or first as a
|
||||
negative, is out of scope and not counted.
|
||||
- Sources: the prompt's non-archived pairs where it is the positive; pairs with a blank
|
||||
category or no tags are ignored for that field. Category values are trimmed and
|
||||
case-sensitive, tag sets compared by tag id; a field fills when exactly one value is
|
||||
left. A whitespace-only category counts as empty.
|
||||
- Kept, per field, in this order: archived prompt, edited prompt (`updated_at` not within
|
||||
2 s of `created_at`; NULL or unreadable counts as edited), pairs that disagree, a tag
|
||||
set over 20 tags. A prompt can fill one field and keep the other.
|
||||
- The plan lists protected rows first, then fills, then a "Record" row, within the 50
|
||||
detail rows; at most 20,000 prompts per plan ("review again for the rest").
|
||||
- The fingerprint covers each considered prompt's computed result (values, tag ids and
|
||||
names, source pair ids), never a pair's title, notes, rating, run count or use
|
||||
stamps, so a Pair run between review and apply still applies.
|
||||
- The apply's writes are guarded (`category` still blank, no `prompt_tags` row) and a
|
||||
guard that hits nothing rolls the whole transaction back as `plan-changed`. It never
|
||||
writes `prompt_pairs`, `prompt_pair_tags`, `updated_at`, a tag or Library state.
|
||||
- The job lists 200 changed rows ("Prompt 12: category “night”; tags a, b"); its
|
||||
Download report lists every one (`truncated: false`, `Job.record_counted(complete=True)`).
|
||||
Protected lists one row per kept prompt in the review's wording and order ("Prompt 7:
|
||||
kept, pairs disagree on category: Adult, asian"), also complete in the report.
|
||||
- Activity shows a row with no path as a record: its detail as plain text, never an open
|
||||
or locate action, and the filter reads the detail. A maintenance job of records (every
|
||||
operation but Missing-file links) says "No records in this group.", "No listed record
|
||||
matches this filter." and "Filter records…" (`groupCopy`); file jobs keep "files".
|
||||
|
||||
Tests: `tests/test_maintenance_pair_metadata.py`, `tests/test_api_maintenance_pair_metadata.py`,
|
||||
`tests/test_jobs.py`, `tests/js/workspace-system-maintenance.test.js`,
|
||||
`tests/browser/workspace-system-maintenance.spec.mjs` (hook `/__test__/system/pair-metadata`).
|
||||
A Python change: ComfyUI needs a restart.
|
||||
|
||||
The legacy restore (`POST /prompt_manager/restore`) takes a backup of any size the disks
|
||||
have room for, so a real library's safety backup restores: the upload, its staged copy
|
||||
and the swap (the upload twice in the temporary folder, once more plus the current
|
||||
@@ -5329,16 +5052,16 @@ server-side tagging configuration, `autotag.defaults` in config.json: `model_typ
|
||||
(`wd14-swinv2`, `wd14-vit`, `gguf`, `hf`), `existing` (`skip`, `merge`,
|
||||
`replace_after_review`), `general_threshold` and `character_threshold` (0–1 in
|
||||
steps of 0.05, #369),
|
||||
`suggestion_limit` (1–100), and since #431 `skipped_tags`, `instructions` and
|
||||
`default_instructions` (below). They are host drafts (`autotag_model_type`, `autotag_existing`,
|
||||
`autotag_general_threshold`, `autotag_character_threshold`, `autotag_suggestion_limit`,
|
||||
`autotag_skipped_tags`, `autotag_instructions`, `autotag_default_instructions`) saved by the one Save row through `POST
|
||||
`template` (non-blank, at most 4,000 characters and 16 KiB) and `suggestion_limit`
|
||||
(1–100). They are host drafts (`autotag_model_type`, `autotag_existing`,
|
||||
`autotag_general_threshold`, `autotag_character_threshold`, `autotag_template`,
|
||||
`autotag_suggestion_limit`) saved by the one Save row through `POST
|
||||
/prompt_manager/settings` with `expected_revision`; a bad value is refused with a 400
|
||||
naming the setting by its row label ("General-tag confidence must be between 0 and
|
||||
1."), nothing is written, the Save row shows that sentence (`invalid-setting`, set by
|
||||
`settingRefusal` in system.mjs only for a save that carried `autotag_*` keys; every
|
||||
other section's plain 400 keeps the generic status) and the control is highlighted.
|
||||
Instruction text limits count code points, as Python's `len` does. A hand-edited bad value
|
||||
The template's limit counts code points, as Python's `len` does. A hand-edited bad value
|
||||
falls back to its default on load, one key at a time. Every page shows the thresholds in
|
||||
steps of 0.05 (the System rows, the Tag image dialogs, the legacy sliders), so a saved
|
||||
threshold off that grid is refused ("General-tag confidence must be a multiple of
|
||||
@@ -5349,70 +5072,8 @@ stored exactly on the grid (#369). A per-request threshold keeps its own check:
|
||||
`validate_options` and starts the `autotag` job titled "Tag output images (legacy page)";
|
||||
the legacy `/autotag/single` only requires a number. `GET /autotag/models` reports the
|
||||
saved values (`wd14_general_threshold`, `wd14_character_threshold`, `suggestion_limit`,
|
||||
`default_prompt`, `model_type`, `existing`, and since #431 `instructions` and
|
||||
`default_instructions`), so the Tag image and Auto-tag selected dialogs start from them;
|
||||
a preset (the bulk review, Preview tagging review) wins.
|
||||
|
||||
**Saved instructions** (#431, D203, D205): `autotag.defaults.instructions` holds the
|
||||
saved entries `[{id, name, text}]` in order (at most 20; `id` 12 lowercase hex made by
|
||||
the client, unique, `builtin` reserved; `name` trimmed, 1–60 characters, no control
|
||||
character, unique case-folded, not "Built-in"; `text` as `validate_template`), and
|
||||
`autotag.defaults.default_instructions` is `{gguf, hf}`, each `builtin` or a saved id.
|
||||
Built-in (`{id: "builtin", name: "Built-in", text: DEFAULT_TAG_TEMPLATE}`) is never
|
||||
stored; readers put it first (`AutotagConfig.all_instructions`), so a later release's
|
||||
built-in text reaches everyone who uses it. Refusals (400, the Save row shows them and
|
||||
highlights the editor): "Saved instructions: at most 20 entries besides Built-in.",
|
||||
"Saved instructions: entry N has an invalid id.", "… repeats the id of entry M.", "…
|
||||
needs a name.", "… name must be at most 60 characters.", "… name must not contain a line
|
||||
break.", "… name “X” is already used.", "Saved instructions: the name “Built-in” is
|
||||
reserved.", "Saved instructions: entry N text must be …", "Default for GGUF JoyCaption
|
||||
must be one of the saved instructions." (and HuggingFace). A save carrying either key
|
||||
without `expected_revision` is refused ("Reload the page and try again."); a save
|
||||
carrying one is paired with the other from the running configuration and both are
|
||||
written, plus the old `template` key as the GGUF default's text (deleted when that
|
||||
default is Built-in), so an older build reads that default as its one template. `POST`
|
||||
with `autotag_template` is refused ("LLM tag template was replaced by Saved instructions.
|
||||
Reload the page."); `GET` keeps returning it (the GGUF default's text, read-only) for one
|
||||
release and adds `autotag_builtin_instructions` (Built-in's text, read-only). Load: each
|
||||
key alone (a damaged list → Built-in only, a default naming nothing → Built-in, each with
|
||||
one warning), then the in-memory migration of the old `template`: with no `instructions`
|
||||
key, a template other than Built-in becomes "Custom template" (id = first 12 hex of its
|
||||
SHA-256), the default of both engines unless `default_instructions` is stored; with the
|
||||
key, a template matching no entry (edited on an older build) is appended as "Custom
|
||||
template (n)" with one INFO line (a full list: one WARNING, nothing imported). Nothing is
|
||||
written at load; the next save that carries either key writes the list.
|
||||
`AutotagConfig.default_instructions_text(engine)` feeds the legacy batch and single
|
||||
blank-prompt fallbacks (by the request's engine) and `default_prompt`. Tests:
|
||||
`tests/test_saved_instructions.py`, `tests/test_saved_instructions_routes.py`,
|
||||
`tests/js/workspace-system-instructions.test.js`.
|
||||
|
||||
**Skipped tags** (#431, D202): `autotag.defaults.skipped_tags`, host draft
|
||||
`autotag_skipped_tags`, a list of at most 200 entries, each trimmed, 1–50 code points,
|
||||
no comma or control character, at least one word, no two with the same words (refused
|
||||
with "Skipped tags: at most 200 entries.", "Skipped tags: entry N is empty.", "… must be
|
||||
at most 50 characters.", "… must not contain a comma or a line break.", "… repeats entry
|
||||
M."); a save carrying it without `expected_revision` is refused ("Reload the page and try
|
||||
again."). A damaged saved list falls back to none with one warning. Words are the
|
||||
NFC, case-folded text split on runs of space, `_` and `-` (`tag_words`); an entry hides
|
||||
a suggested tag whose words contain the entry's words consecutively (`SkipMatcher`:
|
||||
`blurry` hides `very_blurry` and `blurry_background`, not `unblurry`; `(fate)` hides
|
||||
`saber_(fate)`, `fate` does not). `generate_tag_suggestions(skip=)` drops a match after
|
||||
the name check and before the dedupe and the limit, so skipped tags use up no slot
|
||||
(WD14 fills from lower scores; a language model may return fewer). Every path filters:
|
||||
`/image-annotations/suggest` (compiled per request; the answer adds `skipped`, the
|
||||
number of distinct names it hid), `/autotag/single`, and the `image-tags` (Auto-tag
|
||||
selected) and `autotag` (System batch, legacy `/autotag/start`) jobs, which snapshot the
|
||||
list at their start and log "Skipped tags: N entries" or "Skipped tags: none"; a save
|
||||
during a run does not change it. Hand-typed and stored tags are never filtered; the
|
||||
language models' hard-coded exclude list (`_parse_tags`) runs first. `POST
|
||||
/prompt_manager/autotag/skipped-tags/preview {entries}` (validated as a save) answers
|
||||
`count`, `sample` (12, library tags first, each part sorted case-folded),
|
||||
`sample_limit`, `per_entry [{entry, count}]` and `sources {library, wd14}` over the
|
||||
library's tags plus the category 0/4 names of each downloaded WD14 `selected_tags.csv`
|
||||
(read as UTF-8, cached by mtime and size; a missing list is skipped). It loads no model,
|
||||
opens no image, writes nothing and is `409 busy` while an exclusive job runs. Tests:
|
||||
`tests/test_skipped_tags.py`, `tests/test_skipped_tags_routes.py`, the settings and
|
||||
config tests, `tests/js/workspace-system-skipped-tags.test.js`.
|
||||
`default_prompt`, `model_type`, `existing`), so the Tag image and Auto-tag selected
|
||||
dialogs start from them; a preset (the bulk review, Preview tagging review) wins.
|
||||
|
||||
`GET /prompt_manager/autotag/models` reads the disk in the executor, never loads or
|
||||
downloads a model and carries no absolute path: per engine `name`, `label`,
|
||||
@@ -5660,7 +5321,7 @@ list stay unscoped: any row's control proves them.
|
||||
| Worker threads | `admin.html:437–445` | S4 Sources | covered | `sources` · slider · "Worker threads" | worker range max is cpu_count (workspace-system-sources.spec.mjs) |
|
||||
| LoraManager badge, enable, path, trigger words, CivitAI key, clear key | `admin.html:448–490`; `admin.js:1052–1156` | S10 Integrations | covered | `integrations` · switch · "LoraManager integration" | LoRA rows show detected / enabled / cache-loaded separately (workspace-system-integrations.spec.mjs) |
|
||||
| LoRA data import with progress | `admin.html:492–498,517–541`; `admin.js:1157` | S10 Integrations | covered | `integrations` · button · "Review import" | Review import shows replacement, blocked items and network effects with the backup box; Start opens the job (D107) (workspace-system-integrations.spec.mjs) |
|
||||
| Statistics | `admin.html:159`; `admin.js:1627` | S2 Overview | covered | `overview` · button · "Explore" | statistics dialog lists runs by category and database record counts (workspace-system.spec.mjs) |
|
||||
| Statistics | `admin.html:159`; `admin.js:1627` | S2 Overview | covered | `overview` · button · "Explore" | statistics dialog lists runs by category and library totals (workspace-system.spec.mjs) |
|
||||
| Export components JSON `/export` | `admin.html:163`; `admin.js:1607`; `py/api/prompts.py:238` | S9 Backup & export | covered | `backups` · button · "Export all prompts JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Saved pairs export | `py/api/pairs.py:84` | S9 Backup & export | covered | `backups` · button · "Export saved pairs JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Library export | `py/api/library.py:223` | S9 Backup & export | covered | `backups` · button · "Export JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
@@ -5988,23 +5649,6 @@ npm run --prefix tools/vendor test:workflows -- workspace-system-backups.spec.mj
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
```
|
||||
|
||||
## Sticky chrome (#531)
|
||||
|
||||
Beyond the concept, from the maintainer's test of #528; the concept is unchanged. `workspace-sticky-chrome.spec.mjs` holds these.
|
||||
|
||||
- **D186:** the selection bars (Images `#bulkbar`, Library `#library-bulkbar`; there are no others) are sticky: under the
|
||||
top bar at the top of the page, and at the top of the window once that non-sticky top bar has scrolled away (one rule
|
||||
in workspace.css). While the bar's flow place is above the window, its appearing, resizing or leaving does not move the cards
|
||||
(`keepingPlace`). Focused card controls and the Images panel keep clear of it (`--selection-bar-block`). At 700px and below
|
||||
the bar is one row that scrolls sideways. After a bulk action finishes the page scrolls to the top and refreshes the
|
||||
list; Set category, Delete… and Return to library clear the selection as before, while tagging and collections keep
|
||||
it (D85, D138). Export metadata changes nothing, so it stays put. Single-image actions from the inspector and viewer stay put.
|
||||
- **D187:** Back to top: one shell chip past 200px of scroll, above the toast's spot; it rises only for a toast taller than two lines. It never animates, because the
|
||||
pages' height corrections would cancel a smooth scroll, and focus moves to `#workspace-main`. D42's band grows to 140px.
|
||||
- **D188:** the Needs review panel is sticky like the Images panel, with its own scroll on a window under 720px tall, and static at 700px and below.
|
||||
- **D189:** where a panel is one column (static): Info brings the panel into view. On
|
||||
Images the panel follows the whole list, past infinite scroll, so loading waits for the reader's next move (wheel, touch or key) rather than pushing the panel away.
|
||||
|
||||
## Workspace data fields for the concept pages (#249)
|
||||
|
||||
These fields are additive. The legacy `/prompt_manager/images/output` listing (no
|
||||
@@ -7044,7 +6688,7 @@ checkout or point this harness at a live ComfyUI checkout.
|
||||
|
||||
The package-qualified config import also corrects #211: the configured database
|
||||
path is honored when PromptManager runs as a ComfyUI custom node. Follow the
|
||||
[database path compatibility and safe backup sequence](README-legacy.md#database-path-compatibility-211)
|
||||
[database path compatibility and safe backup sequence](README.md#database-path-compatibility-211)
|
||||
before restarting an existing deployment.
|
||||
|
||||
The ledger records only new dequeued jobs and observed outcomes. Existing
|
||||
|
||||
@@ -120,7 +120,6 @@ except Exception as e:
|
||||
logger = get_logger("prompt_manager.init")
|
||||
logger.error(f"Failed to register API routes: {e}")
|
||||
except Exception:
|
||||
# logging failed too; a report must never stop the node loading
|
||||
pass
|
||||
|
||||
# Shared database, set up with the monitoring system below. Stays None when that
|
||||
@@ -280,11 +279,11 @@ def _teardown_logger():
|
||||
|
||||
|
||||
def _stop_background_work():
|
||||
"""Stop the image monitor, the preview worker, the job worker threads
|
||||
and the prompt cleanup thread against one TEARDOWN_SECONDS deadline, once.
|
||||
"""Stop the image monitor, the preview worker and the prompt cleanup
|
||||
thread against one TEARDOWN_SECONDS deadline, once.
|
||||
|
||||
Returns ``{"monitor", "thumbnails", "workers", "tracker"}`` booleans
|
||||
(True: it stopped), leaving out parts that never started. Later calls return a
|
||||
Returns ``{"monitor", "thumbnails", "tracker"}`` booleans (True: it
|
||||
stopped), leaving out parts that never started. Later calls return a
|
||||
copy of the first result. A Ctrl+C inside a step is logged and the
|
||||
step is False; one between steps is re-raised after the result (parts
|
||||
not reached are False) is stored and the ``atexit`` entry removed.
|
||||
@@ -298,14 +297,11 @@ def _stop_background_work():
|
||||
api,
|
||||
_global_prompt_tracker,
|
||||
)
|
||||
from .utils import parallel
|
||||
|
||||
# A part whose global is set has started; False until its step says
|
||||
# otherwise, so an interrupt between steps leaves an honest result.
|
||||
started = (
|
||||
("monitor", monitor),
|
||||
("thumbnails", thumbnails),
|
||||
("workers", parallel.job_pool_started() or None),
|
||||
("tracker", tracker),
|
||||
)
|
||||
result = {name: False for name, part in started if part is not None}
|
||||
@@ -348,16 +344,6 @@ def _stop_background_work():
|
||||
result["thumbnails"] = _teardown_step(
|
||||
"the preview worker", close_previews, logger
|
||||
)
|
||||
if "workers" in result:
|
||||
|
||||
def stop_workers():
|
||||
from .utils.parallel import JOB_POOL_JOIN_SECONDS
|
||||
|
||||
return parallel.shutdown_job_pool(left(JOB_POOL_JOIN_SECONDS))
|
||||
|
||||
result["workers"] = _teardown_step(
|
||||
"the job worker threads", stop_workers, logger
|
||||
)
|
||||
if tracker is not None:
|
||||
|
||||
def stop_tracker():
|
||||
@@ -425,7 +411,6 @@ except Exception as e:
|
||||
f"Failed to register the exit signal: {e}"
|
||||
)
|
||||
except Exception:
|
||||
# logging failed too; a report must never stop the node loading
|
||||
pass
|
||||
|
||||
try:
|
||||
@@ -443,14 +428,16 @@ except Exception as e:
|
||||
f"Failed to register the background work teardown: {e}"
|
||||
)
|
||||
except Exception:
|
||||
# logging failed too; a report must never stop the node loading
|
||||
pass
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
||||
|
||||
# One startup line. ASCII only: a Windows console using a legacy code page
|
||||
# cannot print emoji, and a UnicodeEncodeError here would abort loading the node.
|
||||
print(
|
||||
f"\033[94m[ComfyUI-PromptManager] Version:\033[0m {get_version()}"
|
||||
f" - {len(NODE_CLASS_MAPPINGS)} tools loaded"
|
||||
)
|
||||
# Print startup message with loaded tools
|
||||
print()
|
||||
print(f"\033[94m[ComfyUI-PromptManager] Version:\033[0m {get_version()}")
|
||||
for node_key, display_name in NODE_DISPLAY_NAME_MAPPINGS.items():
|
||||
# ASCII only: a Windows console using a legacy code page cannot print emoji,
|
||||
# and a UnicodeEncodeError here would abort loading the whole node.
|
||||
print(f" \033[94mLoaded:\033[0m {display_name}")
|
||||
print(f"\033[94mTotal: {len(NODE_CLASS_MAPPINGS)} tools loaded\033[0m")
|
||||
print()
|
||||
|
||||
|
Before Width: | Height: | Size: 35 KiB |
|
Before Width: | Height: | Size: 88 KiB |
|
Before Width: | Height: | Size: 153 KiB |
|
Before Width: | Height: | Size: 82 KiB |
|
Before Width: | Height: | Size: 72 KiB |
|
Before Width: | Height: | Size: 156 KiB |
|
Before Width: | Height: | Size: 159 KiB |
|
Before Width: | Height: | Size: 96 KiB |
|
Before Width: | Height: | Size: 165 KiB |
|
Before Width: | Height: | Size: 60 KiB |
|
Before Width: | Height: | Size: 38 KiB |
|
Before Width: | Height: | Size: 41 KiB |
|
Before Width: | Height: | Size: 42 KiB |
|
Before Width: | Height: | Size: 39 KiB |
|
Before Width: | Height: | Size: 42 KiB |
|
Before Width: | Height: | Size: 40 KiB |
|
Before Width: | Height: | Size: 41 KiB |
|
Before Width: | Height: | Size: 43 KiB |
|
Before Width: | Height: | Size: 54 KiB |
|
Before Width: | Height: | Size: 237 KiB |
|
Before Width: | Height: | Size: 66 KiB |
|
Before Width: | Height: | Size: 53 KiB |
|
Before Width: | Height: | Size: 54 KiB |
|
Before Width: | Height: | Size: 181 KiB |
|
Before Width: | Height: | Size: 51 KiB |
|
Before Width: | Height: | Size: 110 KiB |
@@ -590,7 +590,6 @@ class _Move:
|
||||
try:
|
||||
os.rmdir(self.plan.target_dir)
|
||||
except OSError:
|
||||
# not empty or already gone: the undo leaves it alone
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
|
||||
Four operations, each the native owner of a legacy ``/maintenance`` one and
|
||||
built on the same queries, so a preview counts what the legacy cleanup would
|
||||
change (``workflow_storage`` and #550's ``pair_metadata``, which fills old
|
||||
Pair prompts' empty category and tags, live in their own modules):
|
||||
change:
|
||||
|
||||
- ``duplicate_prompts``: prompts whose text is equal ignoring case and outer
|
||||
spaces (``DUPLICATE_PROMPTS_SQL``, legacy ``cleanup_duplicates``); each
|
||||
@@ -61,9 +60,6 @@ MAINTENANCE_OPERATIONS = (
|
||||
"missing_links",
|
||||
# Applied as its own batched job (database/workflow_storage.py).
|
||||
"workflow_storage",
|
||||
# #550: fill old Pair prompts' empty category and tags
|
||||
# (database/pair_metadata_repair.py).
|
||||
"pair_metadata",
|
||||
)
|
||||
MAX_DETAILS = 50 # detail rows a plan lists
|
||||
MAX_RESULT_ROWS = 200 # changed records an apply describes (the job lists 200)
|
||||
@@ -537,10 +533,6 @@ class MaintenanceOperations:
|
||||
lambda conn: (self._link_rows(conn), _mark_rows(conn)),
|
||||
)
|
||||
return self._plan_missing_links(rows, roots, _stale_marks(marks, roots))
|
||||
if operation == "pair_metadata":
|
||||
from .pair_metadata_repair import plan_pair_metadata
|
||||
|
||||
return _read_snapshot(self.model.db_path, plan_pair_metadata)
|
||||
reader = (
|
||||
self._plan_duplicate_prompts
|
||||
if operation == "duplicate_prompts"
|
||||
@@ -808,8 +800,7 @@ class MaintenanceOperations:
|
||||
records changed and up to ``MAX_RESULT_ROWS`` descriptions of them
|
||||
(``{prompt_id[, kept_id]}`` or ``{image_path}``). ``marks`` are a
|
||||
missing-links plan's stale removal marks; that apply also returns
|
||||
``marks``, how many it forgot. ``pair_metadata`` returns every filled
|
||||
prompt, uncapped (its job keeps the complete record, #550).
|
||||
``marks``, how many it forgot.
|
||||
"""
|
||||
if operation not in MAINTENANCE_OPERATIONS or operation == "workflow_storage":
|
||||
raise ValueError(f"Unknown maintenance operation {operation!r}")
|
||||
@@ -824,10 +815,6 @@ class MaintenanceOperations:
|
||||
"unused_components": self._apply_unused_components,
|
||||
}
|
||||
with self._pair_transaction() as conn:
|
||||
if operation == "pair_metadata":
|
||||
from .pair_metadata_repair import apply_pair_metadata
|
||||
|
||||
return apply_pair_metadata(conn, fingerprint, targets)
|
||||
if operation == "missing_links":
|
||||
return self._apply_missing_links(
|
||||
conn, fingerprint, targets, roots, list(marks)
|
||||
|
||||
@@ -35,51 +35,38 @@ from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import operator
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
from collections import deque
|
||||
from typing import Any, Callable, Iterable, Iterator, List, Optional
|
||||
from typing import Iterable, Iterator, List, Optional
|
||||
|
||||
import sqlite3
|
||||
|
||||
CLOSED_MESSAGE = "Cannot operate on a closed database."
|
||||
# Rows fetched per step when a cursor is iterated: the connection is marked
|
||||
# in use once per chunk instead of once per row (iteration costs about what
|
||||
# a plain cursor costs). Prefetched rows belong to the cursor so iteration
|
||||
# and explicit fetches always advance the same position.
|
||||
# a plain cursor costs). A loop that stops early leaves up to this many rows
|
||||
# fetched; ``fetchone``/``fetchmany`` after a partial loop skip them.
|
||||
ITER_CHUNK = 128
|
||||
|
||||
|
||||
class ManagedCursor(sqlite3.Cursor):
|
||||
"""A cursor whose calls into SQLite count as its connection being in use."""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._pending = deque()
|
||||
self._closed = False
|
||||
|
||||
def execute(self, sql, parameters=(), /):
|
||||
return self._run_statement(super().execute, sql, parameters)
|
||||
|
||||
def executemany(self, sql, seq_of_parameters, /):
|
||||
return self._run_statement(super().executemany, sql, seq_of_parameters)
|
||||
|
||||
def _run_statement(self, operation: Callable, *args: Any) -> ManagedCursor:
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
description = self.description
|
||||
completed = False
|
||||
try:
|
||||
result = operation(*args)
|
||||
completed = True
|
||||
return result
|
||||
return super().execute(sql, parameters)
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def executemany(self, sql, seq_of_parameters, /):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().executemany(sql, seq_of_parameters)
|
||||
finally:
|
||||
# Argument parsing can fail before SQLite replaces the old statement.
|
||||
if completed or self.description is not description:
|
||||
self._pending.clear()
|
||||
conn._leave()
|
||||
|
||||
def executescript(self, sql_script, /):
|
||||
@@ -94,8 +81,6 @@ class ManagedCursor(sqlite3.Cursor):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
if self._pending and not self._closed:
|
||||
return self._pending.popleft()
|
||||
return super().fetchone()
|
||||
finally:
|
||||
conn._leave()
|
||||
@@ -104,20 +89,7 @@ class ManagedCursor(sqlite3.Cursor):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
# Let SQLite validate the original arguments, including version-specific
|
||||
# handling of zero/negative sizes, before touching prefetched rows.
|
||||
rows = super().fetchmany(*args, **kwargs)
|
||||
if not self._pending:
|
||||
return rows
|
||||
size = operator.index(
|
||||
kwargs.get("size", args[0] if args else self.arraysize)
|
||||
)
|
||||
# Python 3.12 reads all rows for size=0; 3.13+ reads none.
|
||||
if size == 0 and sys.version_info >= (3, 13):
|
||||
return rows
|
||||
self._pending.extend(rows)
|
||||
count = len(self._pending) if size <= 0 else min(size, len(self._pending))
|
||||
return [self._pending.popleft() for _ in range(count)]
|
||||
return super().fetchmany(*args, **kwargs)
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
@@ -125,31 +97,25 @@ class ManagedCursor(sqlite3.Cursor):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
rows = super().fetchall()
|
||||
pending = list(self._pending)
|
||||
self._pending.clear()
|
||||
return pending + rows
|
||||
return super().fetchall()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def __iter__(self) -> Iterator:
|
||||
return self
|
||||
return self._chunks()
|
||||
|
||||
def _chunks(self) -> Iterator:
|
||||
while True:
|
||||
rows = self.fetchmany(ITER_CHUNK)
|
||||
if not rows:
|
||||
return
|
||||
yield from rows
|
||||
|
||||
def __next__(self):
|
||||
conn = self.connection
|
||||
pending = self._pending
|
||||
# Buffered Python rows need no SQLite lock, but retirement and cursor
|
||||
# close must still stop an iterator obtained before either operation.
|
||||
if conn._retired or self._closed:
|
||||
raise sqlite3.ProgrammingError(CLOSED_MESSAGE)
|
||||
if pending:
|
||||
return pending.popleft()
|
||||
conn._enter()
|
||||
try:
|
||||
pending.extend(super().fetchmany(ITER_CHUNK))
|
||||
if not pending:
|
||||
raise StopIteration
|
||||
return pending.popleft()
|
||||
return super().__next__()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
@@ -160,10 +126,7 @@ class ManagedCursor(sqlite3.Cursor):
|
||||
except sqlite3.ProgrammingError:
|
||||
return super().close() # the connection is closed or closing
|
||||
try:
|
||||
result = super().close()
|
||||
self._pending = deque()
|
||||
self._closed = True
|
||||
return result
|
||||
return super().close()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
@@ -330,9 +293,6 @@ class ManagedConnection(sqlite3.Connection):
|
||||
cursors = list(self._cursors)
|
||||
try:
|
||||
for cursor in cursors:
|
||||
if isinstance(cursor, ManagedCursor):
|
||||
cursor._pending = deque()
|
||||
cursor._closed = True
|
||||
with contextlib.suppress(sqlite3.Error):
|
||||
sqlite3.Cursor.close(cursor)
|
||||
super().close()
|
||||
|
||||
@@ -34,7 +34,7 @@ from .history_reads import HistoryReadOperations
|
||||
from .health import HealthOperations, iter_consistency_issues
|
||||
from .duplicate_copies import DuplicateCopyOperations
|
||||
from .workflow_storage import WorkflowStorageOperations
|
||||
from .category_import import CategoryImportOperations
|
||||
from .category_import import CategoryImportOperations, EmptyReplacement # noqa: F401
|
||||
from .image_categories import ImageCategoryOperations
|
||||
from .workflow_documents import (
|
||||
DocumentReader,
|
||||
|
||||
@@ -1,452 +0,0 @@
|
||||
"""Pair category and tags repair (#550): the ``pair_metadata`` maintenance plan.
|
||||
|
||||
Before #546 a Pair node run saved its category and tags on the saved pair
|
||||
only; the positive prompt it created stayed Uncategorized and untagged. This
|
||||
plan fills the **empty** category and the **empty** tag set of those prompts
|
||||
from their saved pairs (``docs/egm/post-3.4.1/550-design.md``):
|
||||
|
||||
- Considered: prompts the Pair node created as a positive (their earliest
|
||||
pair, in either role, has them as the positive and its ``created_at`` is
|
||||
within ``PAIR_CREATED_TOLERANCE`` of theirs) whose category is blank
|
||||
and/or that have no tags.
|
||||
- Sources: the prompt's non-archived pairs where it is the positive. Pairs
|
||||
with a blank category or no tags are ignored for that field. A field fills
|
||||
when exactly one value remains (category: trimmed, case-sensitive; tags:
|
||||
the tag-id set).
|
||||
- Kept (protected), in this order: archived prompt, edited prompt
|
||||
(``updated_at`` not within the tolerance of ``created_at``), pairs that
|
||||
disagree, a tag set over ``MAX_LIBRARY_TAGS``.
|
||||
|
||||
The apply runs in the caller's ``BEGIN IMMEDIATE`` transaction, recomputes
|
||||
the fingerprint, then writes guarded statements (the field must still be
|
||||
empty) that never touch a pair, a negative or ``updated_at``. A guard that
|
||||
hits nothing raises ``PlanChanged`` so the whole transaction rolls back.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
|
||||
from . import maintenance as _maintenance
|
||||
from .library_commands import MAX_LIBRARY_TAGS
|
||||
from .maintenance import (
|
||||
MAX_DETAILS,
|
||||
MaintenanceDraft,
|
||||
PlanChanged,
|
||||
_excerpt,
|
||||
_Fingerprint,
|
||||
_marks,
|
||||
_plural,
|
||||
)
|
||||
|
||||
OPERATION = "pair_metadata"
|
||||
# Seconds: the Pair save stamps its new prompt and its pair with separate
|
||||
# utc_now_iso() calls in one transaction (microseconds apart).
|
||||
PAIR_CREATED_TOLERANCE = 2.0
|
||||
# Fill prompts one plan applies; the rest are applied by the next review.
|
||||
# Bounds the complete record a job keeps in memory (about 100 bytes a row).
|
||||
MAX_PLAN_FILLS = 20000
|
||||
DETAIL_VALUES = 3
|
||||
DETAIL_TAGS = 5
|
||||
MEDIA_LABEL = "No image file or saved pair is changed."
|
||||
EMPTY_LABEL = "No Pair prompts are missing a category or tags"
|
||||
RECORD_NOTE = (
|
||||
"The job's report lists every filled field. Download it to keep it: "
|
||||
"Activity forgets jobs after a restart."
|
||||
)
|
||||
# Protected classes in precedence order: (key, detail, reason one, many).
|
||||
EDITED_DETAIL = (
|
||||
"Kept: edited after it was saved; its empty category or tags may be intentional"
|
||||
)
|
||||
PROTECTED = (
|
||||
(
|
||||
"archived",
|
||||
"Kept: archived in the Library",
|
||||
"archived in the Library",
|
||||
"archived in the Library",
|
||||
),
|
||||
(
|
||||
"edited",
|
||||
EDITED_DETAIL,
|
||||
"edited after it was saved",
|
||||
"edited after they were saved",
|
||||
),
|
||||
("conflict", None, "where pairs disagree", "where pairs disagree"),
|
||||
(
|
||||
"too-many-tags",
|
||||
f"Kept: its pair has more than {MAX_LIBRARY_TAGS} tags",
|
||||
f"whose pair has more than {MAX_LIBRARY_TAGS} tags",
|
||||
f"whose pair has more than {MAX_LIBRARY_TAGS} tags",
|
||||
),
|
||||
)
|
||||
_PROTECTED_ORDER = [key for key, *_rest in PROTECTED]
|
||||
# Detail rows: what the user can act on (a conflict) first.
|
||||
DETAIL_ORDER = ("conflict", "edited", "archived", "too-many-tags")
|
||||
|
||||
_CANDIDATES_SQL = """
|
||||
SELECT p.id, p.hash, p.text, p.created_at, p.updated_at,
|
||||
COALESCE(TRIM(p.category), '') = '' AS blank_category,
|
||||
NOT EXISTS (SELECT 1 FROM prompt_tags pt WHERE pt.prompt_id = p.id)
|
||||
AS untagged,
|
||||
EXISTS (SELECT 1 FROM prompt_archive pa WHERE pa.prompt_id = p.id)
|
||||
AS archived,
|
||||
(SELECT CASE WHEN pp.positive_prompt_id = p.id
|
||||
THEN pp.created_at END
|
||||
FROM prompt_pairs pp
|
||||
WHERE pp.positive_prompt_id = p.id OR pp.negative_prompt_id = p.id
|
||||
ORDER BY pp.id LIMIT 1) AS first_pair_at
|
||||
FROM prompts p
|
||||
WHERE EXISTS (SELECT 1 FROM prompt_pairs pp WHERE pp.positive_prompt_id = p.id)
|
||||
AND (COALESCE(TRIM(p.category), '') = ''
|
||||
OR NOT EXISTS (SELECT 1 FROM prompt_tags pt WHERE pt.prompt_id = p.id))
|
||||
ORDER BY p.id
|
||||
"""
|
||||
_SOURCES_SQL = (
|
||||
"SELECT id, positive_prompt_id, TRIM(COALESCE(category, '')) FROM prompt_pairs "
|
||||
"WHERE archived = 0 AND positive_prompt_id IN ({marks}) ORDER BY id"
|
||||
)
|
||||
_PAIR_TAGS_SQL = (
|
||||
"SELECT ppt.pair_id, t.id, t.name FROM prompt_pair_tags ppt "
|
||||
"JOIN tags t ON t.id = ppt.tag_id WHERE ppt.pair_id IN ({marks}) "
|
||||
"ORDER BY ppt.pair_id, t.id"
|
||||
)
|
||||
|
||||
|
||||
def parse_stamp(value):
|
||||
"""A stored timestamp as an aware UTC datetime, or None when unreadable."""
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
try:
|
||||
stamp = datetime.datetime.fromisoformat(value.strip())
|
||||
except ValueError:
|
||||
return None
|
||||
if stamp.tzinfo is None:
|
||||
stamp = stamp.replace(tzinfo=datetime.timezone.utc)
|
||||
return stamp
|
||||
|
||||
|
||||
def _close(first, second):
|
||||
"""Whether two stored stamps parse and lie within the tolerance."""
|
||||
a, b = parse_stamp(first), parse_stamp(second)
|
||||
if a is None or b is None:
|
||||
return False
|
||||
return abs((a - b).total_seconds()) <= PAIR_CREATED_TOLERANCE
|
||||
|
||||
|
||||
def _batches(ids):
|
||||
size = _maintenance.ID_BATCH
|
||||
for start in range(0, len(ids), size):
|
||||
yield ids[start : start + size]
|
||||
|
||||
|
||||
def _sources(conn, ids):
|
||||
"""``{prompt_id: [(pair_id, category, ((tag_id, name), ...)), ...]}``."""
|
||||
pairs = {}
|
||||
for batch in _batches(ids):
|
||||
for pair_id, prompt_id, category in conn.execute(
|
||||
_SOURCES_SQL.format(marks=_marks(batch)), batch
|
||||
):
|
||||
pairs.setdefault(prompt_id, []).append([pair_id, category, []])
|
||||
by_pair = {pair[0]: pair for group in pairs.values() for pair in group}
|
||||
pair_ids = sorted(by_pair)
|
||||
for batch in _batches(pair_ids):
|
||||
for pair_id, tag_id, name in conn.execute(
|
||||
_PAIR_TAGS_SQL.format(marks=_marks(batch)), batch
|
||||
):
|
||||
by_pair[pair_id][2].append((tag_id, name))
|
||||
return {
|
||||
prompt_id: [(pid, category, tuple(tags)) for pid, category, tags in group]
|
||||
for prompt_id, group in pairs.items()
|
||||
}
|
||||
|
||||
|
||||
def _field(values, guard):
|
||||
"""``(class, value)`` of one field from its distinct values, or None."""
|
||||
if not values:
|
||||
return None
|
||||
if guard:
|
||||
return (guard, None)
|
||||
if len(values) > 1:
|
||||
return ("conflict", sorted(values))
|
||||
return ("fill", next(iter(values)))
|
||||
|
||||
|
||||
def _classify(row, sources):
|
||||
"""The result of one considered prompt, or None when nothing applies."""
|
||||
guard = None
|
||||
if row["archived"]:
|
||||
guard = "archived"
|
||||
elif not _close(row["updated_at"], row["created_at"]):
|
||||
guard = "edited"
|
||||
category = tags = None
|
||||
tag_pair = None
|
||||
if row["blank_category"]:
|
||||
category = _field({c for _p, c, _t in sources if c}, guard)
|
||||
if row["untagged"]:
|
||||
sets = {}
|
||||
for pair_id, _category, pair_tags in sources:
|
||||
if pair_tags:
|
||||
sets.setdefault(frozenset(pair_tags), pair_id)
|
||||
tags = _field(set(sets), guard)
|
||||
if tags and tags[0] == "fill":
|
||||
tag_pair = sets[tags[1]]
|
||||
if len(tags[1]) > MAX_LIBRARY_TAGS:
|
||||
tags = ("too-many-tags", None)
|
||||
else:
|
||||
tags = ("fill", tuple(sorted(tags[1])))
|
||||
elif tags and tags[0] == "conflict":
|
||||
tags = ("conflict", [sorted(s) for s in sorted(sets, key=sorted)])
|
||||
if category is None and tags is None:
|
||||
return None
|
||||
return {
|
||||
"id": row["id"],
|
||||
"hash": row["hash"],
|
||||
"text": row["text"],
|
||||
"category": category,
|
||||
"tags": tags,
|
||||
"tag_pair": tag_pair if tags and tags[0] == "fill" else None,
|
||||
"pairs": [pair_id for pair_id, _c, _t in sources],
|
||||
}
|
||||
|
||||
|
||||
def _state(conn):
|
||||
"""``(results, fingerprint)``: every considered prompt's result, by id."""
|
||||
considered = []
|
||||
for row in conn.execute(_CANDIDATES_SQL):
|
||||
row = dict(
|
||||
zip(
|
||||
(
|
||||
"id",
|
||||
"hash",
|
||||
"text",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"blank_category",
|
||||
"untagged",
|
||||
"archived",
|
||||
"first_pair_at",
|
||||
),
|
||||
tuple(row),
|
||||
)
|
||||
)
|
||||
if _close(row["created_at"], row["first_pair_at"]):
|
||||
considered.append(row)
|
||||
sources = _sources(conn, [row["id"] for row in considered])
|
||||
fingerprint = _Fingerprint(OPERATION)
|
||||
results = []
|
||||
for row in considered:
|
||||
result = _classify(row, sources.get(row["id"], []))
|
||||
if result is None:
|
||||
continue
|
||||
results.append(result)
|
||||
fingerprint.add(
|
||||
[
|
||||
result["id"],
|
||||
result["hash"],
|
||||
result["category"],
|
||||
result["tags"],
|
||||
result["tag_pair"],
|
||||
result["pairs"],
|
||||
]
|
||||
)
|
||||
return results, fingerprint.hexdigest()
|
||||
|
||||
|
||||
def _fills(result):
|
||||
return (
|
||||
bool(result["category"] and result["category"][0] == "fill"),
|
||||
bool(result["tags"] and result["tags"][0] == "fill"),
|
||||
)
|
||||
|
||||
|
||||
def _targets(results):
|
||||
targets = []
|
||||
for result in results:
|
||||
category, tags = _fills(result)
|
||||
if category or tags:
|
||||
targets.append([result["id"], category, tags])
|
||||
return targets[:MAX_PLAN_FILLS]
|
||||
|
||||
|
||||
def _protected_classes(result):
|
||||
"""The protected classes of a result's fields, in precedence order."""
|
||||
found = {
|
||||
field[0]
|
||||
for field in (result["category"], result["tags"])
|
||||
if field and field[0] != "fill"
|
||||
}
|
||||
return [key for key in _PROTECTED_ORDER if key in found]
|
||||
|
||||
|
||||
def _more(values, limit):
|
||||
shown = ", ".join(values[:limit])
|
||||
return shown + (", …" if len(values) > limit else "")
|
||||
|
||||
|
||||
def _fill_detail(result):
|
||||
parts = []
|
||||
category, tags = _fills(result)
|
||||
if category:
|
||||
parts.append(f"category “{result['category'][1]}”")
|
||||
if tags:
|
||||
names = sorted(name for _id, name in result["tags"][1])
|
||||
noun = "tag" if len(names) == 1 else "tags"
|
||||
parts.append(f"{len(names)} {noun}: {_more(names, DETAIL_TAGS)}")
|
||||
pairs = [str(pair_id) for pair_id in result["pairs"]]
|
||||
label = "pair" if len(pairs) == 1 else "pairs"
|
||||
return f"Fills {' and '.join(parts)} ({label} {_more(pairs, DETAIL_VALUES)})"
|
||||
|
||||
|
||||
def _protected_detail(result, key):
|
||||
if key != "conflict":
|
||||
return next(detail for k, detail, *_r in PROTECTED if k == key)
|
||||
if result["category"] and result["category"][0] == "conflict":
|
||||
values = result["category"][1]
|
||||
return f"Kept: pairs disagree on category: {_more(values, DETAIL_VALUES)}"
|
||||
return "Kept: pairs disagree on tags"
|
||||
|
||||
|
||||
def _kept(results):
|
||||
"""One row per kept prompt, its review reasons without "Kept: " (#550)."""
|
||||
rows = []
|
||||
for result in results:
|
||||
classes = set(_protected_classes(result))
|
||||
if classes:
|
||||
rows.append(
|
||||
{
|
||||
"prompt_id": result["id"],
|
||||
"reasons": [
|
||||
_protected_detail(result, key).removeprefix("Kept: ")
|
||||
for key in DETAIL_ORDER
|
||||
if key in classes
|
||||
],
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _details(results, targets):
|
||||
rows = []
|
||||
for key in DETAIL_ORDER:
|
||||
for result in results:
|
||||
if key in _protected_classes(result):
|
||||
rows.append(
|
||||
{
|
||||
"label": _excerpt(result["text"]),
|
||||
"detail": _protected_detail(result, key),
|
||||
}
|
||||
)
|
||||
planned = {target[0] for target in targets}
|
||||
for result in results:
|
||||
if result["id"] in planned:
|
||||
rows.append(
|
||||
{"label": _excerpt(result["text"]), "detail": _fill_detail(result)}
|
||||
)
|
||||
if not targets:
|
||||
return rows[:MAX_DETAILS]
|
||||
return rows[: MAX_DETAILS - 1] + [{"label": "Record", "detail": RECORD_NOTE}]
|
||||
|
||||
|
||||
def _scope_label(targets, total):
|
||||
if not targets:
|
||||
return EMPTY_LABEL
|
||||
categories = sum(1 for _id, category, _tags in targets if category)
|
||||
tag_sets = sum(1 for _id, _category, tags in targets if tags)
|
||||
label = (
|
||||
f"{_plural(len(targets), 'prompt')} to fill: "
|
||||
f"{_plural(categories, 'category', 'categories')}, "
|
||||
f"{_plural(tag_sets, 'tag set')}"
|
||||
)
|
||||
if total > len(targets):
|
||||
label += (
|
||||
f" (the first {len(targets):,} of {total:,}; review again for the rest)"
|
||||
)
|
||||
return label
|
||||
|
||||
|
||||
def plan_pair_metadata(conn):
|
||||
"""The reviewed ``pair_metadata`` plan on ``conn``'s snapshot."""
|
||||
results, fingerprint = _state(conn)
|
||||
targets = _targets(results)
|
||||
total = sum(1 for result in results if any(_fills(result)))
|
||||
counts = {key: 0 for key in _PROTECTED_ORDER}
|
||||
held = 0
|
||||
for result in results:
|
||||
classes = _protected_classes(result)
|
||||
held += bool(classes)
|
||||
for key in classes:
|
||||
counts[key] += 1
|
||||
reasons = [
|
||||
f"{counts[key]:,} {one if counts[key] == 1 else many}"
|
||||
for key, _detail, one, many in PROTECTED
|
||||
if counts[key]
|
||||
]
|
||||
return MaintenanceDraft(
|
||||
operation=OPERATION,
|
||||
fingerprint=fingerprint,
|
||||
targets=targets,
|
||||
scope={"count": len(targets), "label": _scope_label(targets, total)},
|
||||
protected={"count": held, "reasons": reasons},
|
||||
media={"deletes_files": False, "label": MEDIA_LABEL},
|
||||
details=_details(results, targets),
|
||||
)
|
||||
|
||||
|
||||
def _fill_categories(conn, fills):
|
||||
"""Set each blank category; raises PlanChanged when one is no longer blank."""
|
||||
cursor = conn.executemany(
|
||||
"UPDATE prompts SET category = ? WHERE id = ? "
|
||||
"AND COALESCE(TRIM(category), '') = ''",
|
||||
fills,
|
||||
)
|
||||
if cursor.rowcount != len(fills):
|
||||
raise PlanChanged(OPERATION)
|
||||
|
||||
|
||||
def _fill_tags(conn, fills, expected):
|
||||
"""Copy each source pair's tags onto its untagged prompt."""
|
||||
cursor = conn.executemany(
|
||||
"INSERT INTO prompt_tags(prompt_id, tag_id) "
|
||||
"SELECT ?, ppt.tag_id FROM prompt_pair_tags ppt "
|
||||
"JOIN tags t ON t.id = ppt.tag_id WHERE ppt.pair_id = ? "
|
||||
"AND NOT EXISTS (SELECT 1 FROM prompt_tags WHERE prompt_id = ?)",
|
||||
fills,
|
||||
)
|
||||
if cursor.rowcount != expected:
|
||||
raise PlanChanged(OPERATION)
|
||||
|
||||
|
||||
def apply_pair_metadata(conn, fingerprint, targets):
|
||||
"""Apply the reviewed fills on ``conn`` (inside BEGIN IMMEDIATE).
|
||||
|
||||
Returns ``{changed, categories, tag_sets, rows, kept}``; ``rows`` lists
|
||||
every filled prompt (``{prompt_id, category?, tags?}``) and ``kept`` every
|
||||
prompt the plan keeps (``{prompt_id, reasons}``, in the review's wording
|
||||
and order), both never capped.
|
||||
"""
|
||||
results, current = _state(conn)
|
||||
if current != fingerprint or _targets(results) != targets:
|
||||
raise PlanChanged(OPERATION)
|
||||
by_id = {result["id"]: result for result in results}
|
||||
categories, tags, rows = [], [], []
|
||||
tag_count = 0
|
||||
for prompt_id, fill_category, fill_tags in targets:
|
||||
result = by_id[prompt_id]
|
||||
row = {"prompt_id": prompt_id}
|
||||
if fill_category:
|
||||
row["category"] = result["category"][1]
|
||||
categories.append((row["category"], prompt_id))
|
||||
if fill_tags:
|
||||
row["tags"] = sorted(name for _id, name in result["tags"][1])
|
||||
tags.append((prompt_id, result["tag_pair"], prompt_id))
|
||||
tag_count += len(result["tags"][1])
|
||||
rows.append(row)
|
||||
if categories:
|
||||
_fill_categories(conn, categories)
|
||||
if tags:
|
||||
_fill_tags(conn, tags, tag_count)
|
||||
return {
|
||||
"changed": len(rows),
|
||||
"categories": len(categories),
|
||||
"tag_sets": len(tags),
|
||||
"rows": rows,
|
||||
"kept": _kept(results),
|
||||
}
|
||||
@@ -112,14 +112,8 @@ class PairOperations:
|
||||
tags: Optional[List[str]] = None,
|
||||
rating: Optional[int] = None,
|
||||
notes: Optional[str] = None,
|
||||
fill_new_positive: bool = False,
|
||||
) -> int:
|
||||
"""Save an ordered pair atomically; repeat saves preserve its metadata.
|
||||
|
||||
``fill_new_positive`` also gives the positive prompt the category and
|
||||
tags, only if this save inserted its row (#546). An existing prompt
|
||||
never changes, so a cleared category stays cleared.
|
||||
"""
|
||||
"""Save an ordered pair atomically; repeat saves preserve its metadata."""
|
||||
with self._pair_transaction() as conn:
|
||||
return self._save_pair(
|
||||
conn,
|
||||
@@ -130,7 +124,6 @@ class PairOperations:
|
||||
tags=tags,
|
||||
rating=rating,
|
||||
notes=notes,
|
||||
fill_new_positive=fill_new_positive,
|
||||
)
|
||||
|
||||
def _save_pair(
|
||||
@@ -144,7 +137,6 @@ class PairOperations:
|
||||
tags: Optional[List[str]] = None,
|
||||
rating: Optional[int] = None,
|
||||
notes: Optional[str] = None,
|
||||
fill_new_positive: bool = False,
|
||||
) -> int:
|
||||
"""Save an ordered pair atomically; repeat saves preserve its metadata."""
|
||||
if not isinstance(positive_text, str) or not positive_text.strip():
|
||||
@@ -152,13 +144,8 @@ class PairOperations:
|
||||
if not isinstance(negative_text, str):
|
||||
raise ValueError("Negative prompt text must be a string; use '' for empty")
|
||||
# Validate metadata before any component insertion.
|
||||
tags = _validate_metadata(title, category, tags, rating, notes)
|
||||
positive, created = self._pair_component_outcome(conn, positive_text)
|
||||
if created and fill_new_positive:
|
||||
conn.execute(
|
||||
"UPDATE prompts SET category = ? WHERE id = ?", (category, positive)
|
||||
)
|
||||
self._sync_prompt_tags(conn, positive, tags or [])
|
||||
_validate_metadata(title, category, tags, rating, notes)
|
||||
positive = self._pair_component(conn, positive_text)
|
||||
negative = (
|
||||
self._pair_component(conn, negative_text) if negative_text.strip() else None
|
||||
)
|
||||
|
||||
@@ -5,12 +5,8 @@ A decision is one row of ``media_review_decisions`` for one item revision.
|
||||
ingester's compatible-only rules (``_ingest_resolved_pair_in``) in the same
|
||||
``_pair_transaction`` as the decision row, the item's new state and revision
|
||||
and the review index revision; any conflict or later failure rolls all of it
|
||||
back. ``manual`` (#536) is the same but for a Library prompt the reviewer chose:
|
||||
its link's provenance names a manual choice and carries no prompt sides, so it
|
||||
cannot pose as file evidence, and the decision's ``candidate_hash`` (the table
|
||||
requires one) is the digest of that choice. ``unresolved`` records the decision
|
||||
and links nothing. No decision counts a run, touches usage or calls an execution
|
||||
writer.
|
||||
back. ``unresolved`` records the decision and links nothing. No decision
|
||||
counts a run, touches usage or calls an execution writer.
|
||||
|
||||
Callers verify the file first (the capture must be the item's bytes and the
|
||||
current file must still hash to them); nothing here reads the filesystem.
|
||||
@@ -21,7 +17,6 @@ import sqlite3
|
||||
|
||||
from .annotation_schema import BYTES_TABLE
|
||||
from .annotations import _new_id
|
||||
from .library import _ELIGIBLE, _FROM, SHELVED
|
||||
from .library_commands import LibraryCommandError
|
||||
from .models import utc_now_iso
|
||||
from .review import (
|
||||
@@ -35,28 +30,9 @@ from .review import (
|
||||
same_name_rows,
|
||||
)
|
||||
from .review_changes import notify_review_change
|
||||
from .review_schema import (
|
||||
DECISIONS_TABLE,
|
||||
ITEMS_TABLE,
|
||||
canonical_evidence,
|
||||
evidence_digest,
|
||||
)
|
||||
from .review_schema import DECISIONS_TABLE, ITEMS_TABLE, canonical_evidence
|
||||
|
||||
DECISION_KIND = "user-selection"
|
||||
MANUAL_KIND = "manual-link"
|
||||
_MANUAL_POSITIVE = (
|
||||
f"SELECT p.id,p.text FROM {_FROM} " # nosec B608
|
||||
f"WHERE p.id=? AND {_ELIGIBLE} AND {SHELVED}"
|
||||
)
|
||||
_MANUAL_PAIR = (
|
||||
"SELECT COALESCE(n.text,'') AS negative FROM prompt_pairs pp "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
|
||||
"WHERE pp.id=? AND pp.positive_prompt_id=? AND pp.archived=0"
|
||||
)
|
||||
_ARCHIVED_BARE_PAIR = (
|
||||
"SELECT 1 FROM prompt_pairs WHERE positive_prompt_id=? "
|
||||
"AND negative_prompt_id IS NULL AND archived=1"
|
||||
)
|
||||
_INSERT_DECISION = (
|
||||
f"INSERT INTO {DECISIONS_TABLE}(id,item_id,decision_revision," # nosec B608
|
||||
"created_at,outcome,candidate_hash,evidence,pair_id,image_id) "
|
||||
@@ -83,8 +59,6 @@ class ReviewCommandError(LibraryCommandError):
|
||||
``fingerprint`` the current dependency fingerprint, when known;
|
||||
``changed`` marks an association conflict caused by links that changed
|
||||
since the reviewer read them (rather than an incompatible choice).
|
||||
``prompt-missing``: the Library prompt or recipe of a manual link is gone,
|
||||
archived or not that prompt's.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -119,7 +93,6 @@ class ReviewDecisionOperations:
|
||||
sha256: str,
|
||||
outcome: str,
|
||||
candidate: dict | None = None,
|
||||
manual: dict | None = None,
|
||||
image_path: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
expected_fingerprint: str | None = None,
|
||||
@@ -128,9 +101,7 @@ class ReviewDecisionOperations:
|
||||
) -> dict:
|
||||
"""Record the decision; return ``{item, total, pair}``.
|
||||
|
||||
``selected`` needs the evidence ``candidate`` (with its ``hash``) or
|
||||
``manual`` (``{positive_id, pair_id}``; a null ``pair_id`` is no
|
||||
negative), the
|
||||
``selected`` needs the evidence ``candidate`` (with its ``hash``), the
|
||||
``image_path`` the scan and the monitor link under, the captured
|
||||
``metadata`` and the ``expected_fingerprint`` the reviewer saw.
|
||||
``aliases`` are the other spellings legacy rows of the same physical
|
||||
@@ -144,12 +115,9 @@ class ReviewDecisionOperations:
|
||||
if outcome not in ("selected", "unresolved"):
|
||||
raise ValueError("outcome must be selected or unresolved")
|
||||
if outcome == "selected" and (
|
||||
(not isinstance(candidate, dict)) == (not isinstance(manual, dict))
|
||||
or not image_path
|
||||
not isinstance(candidate, dict) or not image_path
|
||||
):
|
||||
raise ValueError(
|
||||
"A selected decision needs one candidate or manual choice and a file"
|
||||
)
|
||||
raise ValueError("A selected decision needs a candidate and a file")
|
||||
try:
|
||||
with self._pair_transaction() as conn:
|
||||
decided = self._decide_in(
|
||||
@@ -160,7 +128,6 @@ class ReviewDecisionOperations:
|
||||
sha256=sha256,
|
||||
outcome=outcome,
|
||||
candidate=candidate,
|
||||
manual=manual,
|
||||
image_path=image_path,
|
||||
metadata=metadata or {},
|
||||
expected_fingerprint=expected_fingerprint,
|
||||
@@ -191,24 +158,13 @@ class ReviewDecisionOperations:
|
||||
raise _Rollback(ReviewCommandError("file-changed", epoch))
|
||||
decision_id = _new_id()
|
||||
pair = None
|
||||
selected = command["outcome"] == "selected"
|
||||
candidate = command["candidate"] if selected else None
|
||||
manual = command["manual"] if selected else None
|
||||
candidate_hash = candidate["hash"] if candidate is not None else None
|
||||
candidate = command["candidate"] if command["outcome"] == "selected" else None
|
||||
if candidate is not None:
|
||||
pair = self._apply_candidate(
|
||||
conn, epoch, current, decision_id, total, **command
|
||||
)
|
||||
elif manual is not None:
|
||||
pair = self._apply_manual(
|
||||
conn, epoch, current, decision_id, total, **command
|
||||
)
|
||||
manual = {"positive_id": manual["positive_id"], "pair_id": pair["id"]}
|
||||
candidate_hash = evidence_digest(
|
||||
canonical_evidence({"kind": MANUAL_KIND, **manual})
|
||||
)
|
||||
evidence = {
|
||||
"kind": MANUAL_KIND if manual is not None else DECISION_KIND,
|
||||
"kind": DECISION_KIND,
|
||||
"decision_id": decision_id,
|
||||
"outcome": command["outcome"],
|
||||
"reviewed_digest": command["sha256"],
|
||||
@@ -218,8 +174,6 @@ class ReviewDecisionOperations:
|
||||
"dependency_fingerprint": command["expected_fingerprint"],
|
||||
"pair": pair,
|
||||
}
|
||||
if manual is not None:
|
||||
evidence["manual"] = manual
|
||||
now = utc_now_iso()
|
||||
self._insert_review_decision(
|
||||
conn,
|
||||
@@ -229,7 +183,7 @@ class ReviewDecisionOperations:
|
||||
current["revision"] + 1,
|
||||
now,
|
||||
command["outcome"],
|
||||
candidate_hash,
|
||||
candidate["hash"] if candidate is not None else None,
|
||||
canonical_evidence(evidence),
|
||||
pair["id"] if pair else None,
|
||||
pair["image_id"] if pair else None,
|
||||
@@ -250,77 +204,6 @@ class ReviewDecisionOperations:
|
||||
self, conn, epoch, current, decision_id, total, **command
|
||||
) -> dict:
|
||||
"""Link the candidate's pair, compatible-only; raise on any conflict."""
|
||||
candidate = command["candidate"]
|
||||
positive, negative = candidate["positive"], candidate["negative"]
|
||||
provenance = {
|
||||
"kind": DECISION_KIND,
|
||||
"decision_id": decision_id,
|
||||
"reviewed_digest": command["sha256"],
|
||||
"candidate_hash": candidate["hash"],
|
||||
"resolver_version": current["resolver_version"],
|
||||
"positive": positive,
|
||||
"negative": negative,
|
||||
}
|
||||
return self._link_compatible(
|
||||
conn, epoch, current, total, positive, negative, provenance, command
|
||||
)
|
||||
|
||||
def _apply_manual(self, conn, epoch, current, decision_id, total, **command):
|
||||
"""Link the chosen Library prompt, compatible-only; raise otherwise."""
|
||||
missing = _Rollback(
|
||||
ReviewCommandError("prompt-missing", epoch, item=current, total=total)
|
||||
)
|
||||
positive_id = command["manual"]["positive_id"]
|
||||
pair_id = command["manual"]["pair_id"]
|
||||
row = conn.execute(_MANUAL_POSITIVE, (positive_id,)).fetchone()
|
||||
if row is None:
|
||||
raise missing
|
||||
positive_text, negative_text = row[1], ""
|
||||
if pair_id is not None:
|
||||
chosen = conn.execute(_MANUAL_PAIR, (pair_id, positive_id)).fetchone()
|
||||
if chosen is None:
|
||||
raise missing
|
||||
negative_text = chosen[0]
|
||||
elif conn.execute(_ARCHIVED_BARE_PAIR, (positive_id,)).fetchone():
|
||||
raise missing
|
||||
provenance = {
|
||||
"kind": MANUAL_KIND,
|
||||
"decision_id": decision_id,
|
||||
"reviewed_digest": command["sha256"],
|
||||
"resolver_version": current["resolver_version"],
|
||||
"positive_id": positive_id,
|
||||
"pair_id": pair_id,
|
||||
}
|
||||
# The sides name no provenance: only the ingester reads them.
|
||||
pair = self._link_compatible(
|
||||
conn,
|
||||
epoch,
|
||||
current,
|
||||
total,
|
||||
{"text": positive_text},
|
||||
{"text": negative_text},
|
||||
provenance,
|
||||
command,
|
||||
)
|
||||
# A prompt whose stored hash is stale reaches another row: never link that.
|
||||
reached = conn.execute(
|
||||
"SELECT positive_prompt_id FROM prompt_pairs WHERE id=?", (pair["id"],)
|
||||
).fetchone()[0]
|
||||
if reached != positive_id or pair_id not in (None, pair["id"]):
|
||||
raise missing
|
||||
return pair
|
||||
|
||||
def _link_compatible(
|
||||
self,
|
||||
conn,
|
||||
epoch,
|
||||
current,
|
||||
total,
|
||||
positive,
|
||||
negative,
|
||||
provenance,
|
||||
command,
|
||||
) -> dict:
|
||||
# Imported here: pair_ingestion imports operations, which composes this.
|
||||
from .pair_ingestion import _ingest_resolved_pair_in
|
||||
|
||||
@@ -359,6 +242,17 @@ class ReviewDecisionOperations:
|
||||
# Enrich the one row of this file under the spelling it was
|
||||
# linked with, so the ingester's compatibility rules see it.
|
||||
image_path = rows[0][1]
|
||||
candidate = command["candidate"]
|
||||
positive, negative = candidate["positive"], candidate["negative"]
|
||||
provenance = {
|
||||
"kind": DECISION_KIND,
|
||||
"decision_id": decision_id,
|
||||
"reviewed_digest": command["sha256"],
|
||||
"candidate_hash": candidate["hash"],
|
||||
"resolver_version": current["resolver_version"],
|
||||
"positive": positive,
|
||||
"negative": negative,
|
||||
}
|
||||
result = _ingest_resolved_pair_in(
|
||||
conn,
|
||||
self,
|
||||
|
||||
@@ -46,7 +46,6 @@ def remove_quietly(*paths):
|
||||
if path and os.path.exists(path):
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
# cleanup after a failure must not replace that failure
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -66,6 +66,7 @@ def _tag_rows(page: str, order: str = "page.id") -> str:
|
||||
)
|
||||
|
||||
|
||||
_TAG_PAGE = _tag_rows("SELECT id,name FROM tags WHERE id>? ORDER BY id LIMIT ?")
|
||||
# #444 tag sorts: sort -> (key expression on tags/counted, direction). Ties and
|
||||
# "created" follow the id, so every order is total and keyset pages are exact.
|
||||
TAG_SORTS = {
|
||||
|
||||
@@ -18,13 +18,9 @@ protocol and peer checks reject unrelated messages. Same-origin scripts are
|
||||
trusted; this transport is not authentication against other scripts on that origin.
|
||||
Unsupported messaging is reported without breaking existing ComfyUI nodes.
|
||||
|
||||
Apply works in one click: without a preview, the sender requests the ticket itself
|
||||
and applies it. Preview remains optional, and shows the exact change first.
|
||||
|
||||
An apply timeout or abort after posting means the outcome is unknown. It cannot
|
||||
prove the host cancelled the write. The sender must inspect a fresh destination
|
||||
preview or choose the destination again before sending again; automatic retry must
|
||||
not overwrite edits. An error the host returned is a refusal: nothing was written.
|
||||
preview before deciding to send again; automatic retry must not overwrite edits.
|
||||
|
||||
## PromptManager adapter
|
||||
|
||||
@@ -35,9 +31,7 @@ field. Unknown/ambiguous metadata must not become an empty string.
|
||||
|
||||
Previews show original and requested text plus positive prepend/append composition.
|
||||
Connected composition is marked unknown rather than guessing upstream values.
|
||||
Connected destination fields reject writes. The sender compares the destination's revision
|
||||
(node object and graph version) from discovery with the one at the preview, and refuses if the workflow changed.
|
||||
Applying rechecks graph, node, widgets,
|
||||
Connected destination fields reject writes. Applying rechecks graph, node, widgets,
|
||||
values and connections. Ordinary serialization cache synchronization is accepted
|
||||
only when it preserves live values. Both requested roles are checked before writing.
|
||||
|
||||
|
||||
@@ -230,8 +230,6 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
negative_text,
|
||||
category=category.strip() or None,
|
||||
tags=self._parse_tags(tags),
|
||||
# The Library shows the prompt's own category and tags, not the pair's.
|
||||
fill_new_positive=True,
|
||||
)
|
||||
pair = self.db.get_prompt_pair(pair_id)
|
||||
prompt_id = pair["positive_prompt_id"]
|
||||
|
||||
@@ -85,8 +85,6 @@ try:
|
||||
from ...database.operations import PromptDatabase
|
||||
from ...utils.file_prompt import parse_comfyui_prompt
|
||||
from ...utils.logging_config import get_logger
|
||||
from ...utils.native_memory import release_free_memory
|
||||
from ...utils.png_text import png_text
|
||||
from ...utils.video_metadata import is_video_path, read_video_metadata
|
||||
except ImportError:
|
||||
import sys
|
||||
@@ -97,8 +95,6 @@ except ImportError:
|
||||
from database.operations import PromptDatabase
|
||||
from utils.file_prompt import parse_comfyui_prompt
|
||||
from utils.logging_config import get_logger
|
||||
from utils.native_memory import release_free_memory
|
||||
from utils.png_text import png_text
|
||||
from utils.video_metadata import is_video_path, read_video_metadata
|
||||
|
||||
|
||||
@@ -530,14 +526,6 @@ class PromptManagerAPI(
|
||||
return await loop.run_in_executor(None, call)
|
||||
return await loop.run_in_executor(None, func, *args)
|
||||
|
||||
async def _release_job_memory(self):
|
||||
"""After a scan or thumbnail job: glibc's free heap pages go back to
|
||||
the OS once (``malloc_trim(0)``; nothing elsewhere, #523)."""
|
||||
try:
|
||||
await self._run_in_executor(release_free_memory)
|
||||
except Exception: # best effort: the job's result already stands
|
||||
self.logger.debug("Releasing free memory failed", exc_info=True)
|
||||
|
||||
def _root_listing(self, root) -> RootListing:
|
||||
"""The shared listing record of one output root (created empty)."""
|
||||
return self._root_listings.setdefault(str(root), RootListing())
|
||||
@@ -1060,8 +1048,6 @@ class PromptManagerAPI(
|
||||
def _extract_comfyui_metadata(self, image_path):
|
||||
"""Embedded ComfyUI metadata of an image (PNG text chunks) or a video.
|
||||
|
||||
PNG text is read without decoding the pixels (``utils.png_text``, #523).
|
||||
|
||||
Videos go through ffprobe (see utils/video_metadata). Unreadable files
|
||||
are a warning, not an error: a scan over a large output tree meets
|
||||
truncated and foreign files, and it carries on past them.
|
||||
@@ -1070,7 +1056,7 @@ class PromptManagerAPI(
|
||||
return read_video_metadata(image_path)
|
||||
try:
|
||||
with Image.open(image_path) as img:
|
||||
return png_text(img)
|
||||
return dict(getattr(img, "text", None) or {})
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
f"Unreadable image {os.path.basename(str(image_path))}: {e}"
|
||||
|
||||
@@ -36,18 +36,10 @@ except ImportError:
|
||||
|
||||
try:
|
||||
from ...utils import exit_signal
|
||||
from ...utils.media_folders import (
|
||||
in_trash_below_roots,
|
||||
in_trash_folder,
|
||||
is_skipped_folder,
|
||||
)
|
||||
from ...utils.media_folders import is_skipped_folder
|
||||
except ImportError:
|
||||
from utils import exit_signal
|
||||
from utils.media_folders import (
|
||||
in_trash_below_roots,
|
||||
in_trash_folder,
|
||||
is_skipped_folder,
|
||||
)
|
||||
from utils.media_folders import is_skipped_folder
|
||||
|
||||
try:
|
||||
from ...database.image_removal import mark_hides
|
||||
@@ -176,7 +168,7 @@ def _physical_file_key(path, stat):
|
||||
volumes on Windows) falls back to the resolved, case-folded path.
|
||||
"""
|
||||
if stat.st_ino:
|
||||
return ("inode", (stat.st_dev, stat.st_ino))
|
||||
return ("inode", stat.st_dev, stat.st_ino)
|
||||
return ("path", os.path.normcase(os.path.realpath(str(path))))
|
||||
|
||||
|
||||
@@ -539,7 +531,6 @@ def _diagnose_output_dirs(configured_dirs, public_path=None):
|
||||
|
||||
candidates.append(folder_paths.get_output_directory())
|
||||
except ImportError:
|
||||
# outside ComfyUI there is no output folder to add
|
||||
pass
|
||||
|
||||
output_dirs = []
|
||||
@@ -1098,21 +1089,13 @@ class AdminRoutesMixin:
|
||||
return result
|
||||
|
||||
def _delete_one_duplicate(self, image_path, output_path):
|
||||
"""Delete one file and its thumbnail; return a failure reason or None.
|
||||
|
||||
What is removed is exactly the folder's real path plus the name: a
|
||||
link is removed itself, never its target, and the preview removed is
|
||||
that name's. Both the link's place and what it leads to are checked.
|
||||
"""
|
||||
"""Delete one file and its thumbnail; return a failure reason or None."""
|
||||
file_path = Path(image_path)
|
||||
if not file_path.is_absolute():
|
||||
file_path = output_path / file_path
|
||||
|
||||
try:
|
||||
real_root = output_path.resolve()
|
||||
rel_path = file_path.resolve().relative_to(real_root)
|
||||
exact = Path(os.path.realpath(file_path.parent)) / file_path.name
|
||||
exact_rel = exact.relative_to(real_root)
|
||||
rel_path = file_path.resolve().relative_to(output_path.resolve())
|
||||
except ValueError:
|
||||
self.logger.warning(
|
||||
f"Attempted to delete file outside output directory: {image_path}"
|
||||
@@ -1121,19 +1104,15 @@ class AdminRoutesMixin:
|
||||
|
||||
if file_path.suffix.lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||||
return "not a media file"
|
||||
if "thumbnails" in rel_path.parts or "thumbnails" in exact_rel.parts:
|
||||
if "thumbnails" in rel_path.parts:
|
||||
return "thumbnails are managed by the gallery"
|
||||
if in_trash_below_roots(file_path, [output_path]) or in_trash_folder(
|
||||
str(exact_rel)
|
||||
):
|
||||
return "it is in a trash folder" # never unlinked from a trash (#543)
|
||||
|
||||
if not file_path.is_file():
|
||||
return "file not found"
|
||||
|
||||
os.remove(exact)
|
||||
os.remove(file_path)
|
||||
self.logger.info(f"Deleted duplicate image: {image_path}")
|
||||
_remove_thumbnail(real_root, exact_rel, self.logger)
|
||||
_remove_thumbnail(output_path.resolve(), rel_path, self.logger)
|
||||
return None
|
||||
|
||||
async def get_statistics(self, request):
|
||||
@@ -1454,7 +1433,6 @@ class AdminRoutesMixin:
|
||||
verb = "cancelled" if state == "cancelled" else "finished"
|
||||
job.log_line(f"Scan {verb} · {_plural(job.processed, 'file')} processed")
|
||||
job.finish(state)
|
||||
await self._release_job_memory()
|
||||
if state == "finished":
|
||||
# After the scan, so the guarded thumbnail job is admitted (#249).
|
||||
await self._hand_off_scan_thumbnails(job, output_dirs)
|
||||
@@ -1669,10 +1647,9 @@ class AdminRoutesMixin:
|
||||
def _extract_batch_metadata_sync(self, file_batch):
|
||||
"""Extract metadata from a batch of files in one executor call.
|
||||
|
||||
Files are read on the long-lived job pool of
|
||||
``PromptManagerConfig.WORKER_THREADS`` threads (#523).
|
||||
PNG text is read chunk by chunk without decoding pixels (#523); file
|
||||
reads and ffprobe release the GIL.
|
||||
Files are read on ``PromptManagerConfig.WORKER_THREADS`` threads:
|
||||
Pillow decodes the whole image to reach trailing text chunks and
|
||||
releases the GIL while doing so.
|
||||
"""
|
||||
from ..config import PromptManagerConfig
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import math
|
||||
from ..tagging_defaults import ( # noqa: F401 - re-exported for the route
|
||||
LLM_MODEL_TYPES as LLM_MODELS,
|
||||
MAX_INSTRUCTION_BYTES,
|
||||
MAX_INSTRUCTION_CHARS,
|
||||
MODEL_TYPES,
|
||||
WD14_MODEL_TYPES as WD14_MODELS,
|
||||
validate_template,
|
||||
@@ -102,14 +103,11 @@ def model_refusal(status: dict, model_type: str, model_name: str):
|
||||
return None
|
||||
|
||||
|
||||
def run_suggestions(
|
||||
service, capture_path: str, model_type: str, options: dict, skip=None
|
||||
):
|
||||
def run_suggestions(service, capture_path: str, model_type: str, options: dict):
|
||||
"""Blocking: load (or swap to) *model_type*, then suggest for the capture.
|
||||
|
||||
The caller holds the model gate for the whole call; nothing is ever
|
||||
downloaded (``load_model`` refuses a model that is not on disk). ``skip``
|
||||
is the request's or job's compiled Skipped tags (#431).
|
||||
downloaded (``load_model`` refuses a model that is not on disk).
|
||||
"""
|
||||
if not service.is_model_loaded() or service.get_loaded_model_type() != model_type:
|
||||
service.load_model(model_type, options["use_gpu"])
|
||||
@@ -119,7 +117,6 @@ def run_suggestions(
|
||||
prompt=options.get("instructions"),
|
||||
general_threshold=options.get("general_threshold"),
|
||||
character_threshold=options.get("character_threshold"),
|
||||
skip=skip,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -35,7 +35,6 @@ from .media_target import media_target as _target
|
||||
from .images import _unavailable_link, _verified_title
|
||||
from .prompts import bad_request
|
||||
from .. import model_gate
|
||||
from ..tagging_defaults import SkipMatcher
|
||||
|
||||
try:
|
||||
from ...database.annotations import (
|
||||
@@ -641,8 +640,6 @@ class AnnotationRoutesMixin:
|
||||
owner = model_gate.new_owner("image-annotations")
|
||||
if not model_gate.try_acquire(owner):
|
||||
return model_busy_response()
|
||||
# #431: the Skipped tags as they are when the request is accepted.
|
||||
skip = SkipMatcher(AutotagConfig.SKIPPED_TAGS).counting()
|
||||
try:
|
||||
# From here the pin is the gate's to end: the worker ends it,
|
||||
# or ``on_skipped`` does when the worker never starts.
|
||||
@@ -656,7 +653,6 @@ class AnnotationRoutesMixin:
|
||||
capture,
|
||||
model_type,
|
||||
options,
|
||||
skip,
|
||||
on_skipped=functools.partial(store.release_use, token),
|
||||
)
|
||||
except MissingBackendPackages as exc:
|
||||
@@ -681,8 +677,6 @@ class AnnotationRoutesMixin:
|
||||
"model_type": model_type,
|
||||
"options": options,
|
||||
"suggestions": suggestion_rows(pairs),
|
||||
# How many distinct suggestions the Skipped tags hid (#431).
|
||||
"skipped": len(skip.hidden),
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
@@ -738,10 +732,10 @@ def _byte_range(request: web.Request, size: int):
|
||||
return start, stop
|
||||
|
||||
|
||||
def _pinned_suggestions(store, token, service, capture, model_type, options, skip=None):
|
||||
def _pinned_suggestions(store, token, service, capture, model_type, options):
|
||||
"""Worker thread: infer on the capture, then end its pin (even on error)."""
|
||||
try:
|
||||
return run_suggestions(service, capture.path, model_type, options, skip)
|
||||
return run_suggestions(service, capture.path, model_type, options)
|
||||
finally:
|
||||
store.release_use(token)
|
||||
|
||||
|
||||
@@ -24,9 +24,9 @@ from . import thumbnail_settings
|
||||
from .. import model_gate
|
||||
|
||||
try:
|
||||
from ...utils.media_folders import in_trash_below_roots, is_skipped_folder
|
||||
from ...utils.media_folders import is_skipped_folder
|
||||
except ImportError:
|
||||
from utils.media_folders import in_trash_below_roots, is_skipped_folder
|
||||
from utils.media_folders import is_skipped_folder
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -40,12 +40,6 @@ BATCH_PREVIEW_BODY_MAX = 1024
|
||||
APPLY_BODY_MAX = 64 * 1024 # POST /autotag/apply: one image and its tags
|
||||
_HEX64 = re.compile(r"[0-9a-f]{64}")
|
||||
BATCH_JOB_BODY_MAX = 32 * 1024
|
||||
# Skipped tags preview (#431): 200 entries of 50 characters, with room for JSON.
|
||||
SKIPPED_PREVIEW_BODY_MAX = 64 * 1024
|
||||
SKIPPED_PREVIEW_LEASE = "Skipped tags preview"
|
||||
SKIPPED_PREVIEW_SAMPLE = 12
|
||||
# WD14 label lists the preview found missing or unreadable (logged once each).
|
||||
_MISSING_LABEL_LISTS = set()
|
||||
REVIEW_REQUIRED = (
|
||||
"Replacing tags needs a review of each image. Use Preview tagging review, "
|
||||
"or choose Skip or Merge for a batch run."
|
||||
@@ -78,12 +72,6 @@ MODEL_DOWNLOAD_BODY_MAX = 1024
|
||||
_download_titles = {}
|
||||
|
||||
|
||||
def _instruction_engine(model_type: str) -> str:
|
||||
"""The language-model engine whose default instructions ``model_type``
|
||||
reads: itself, or GGUF for a WD14 engine (which takes none)."""
|
||||
return model_type if model_type in ("gguf", "hf") else "gguf"
|
||||
|
||||
|
||||
def _holder_label(holder: Optional[str]) -> str:
|
||||
if not holder:
|
||||
return ANOTHER_OPERATION
|
||||
@@ -259,10 +247,6 @@ class AutotagRoutesMixin:
|
||||
async def preview_autotag_batch_route(request):
|
||||
return await self.preview_autotag_batch(request)
|
||||
|
||||
@routes.post("/prompt_manager/autotag/skipped-tags/preview")
|
||||
async def preview_skipped_tags_route(request):
|
||||
return await self.preview_skipped_tags(request)
|
||||
|
||||
@routes.post("/prompt_manager/autotag/start")
|
||||
async def start_autotag_route(request):
|
||||
return await self.start_autotag(request)
|
||||
@@ -300,15 +284,7 @@ class AutotagRoutesMixin:
|
||||
"success": True,
|
||||
"models": public_models(status, loaded_type),
|
||||
# The saved tagging defaults (#328, System › Auto-tagging).
|
||||
# The saved engine's default instructions (GGUF for a
|
||||
# WD14 engine); kept one release for cached pages (#431).
|
||||
"default_prompt": AutotagConfig.default_instructions_text(
|
||||
_instruction_engine(AutotagConfig.MODEL_TYPE)
|
||||
),
|
||||
# #431: Built-in first, with every text, and each
|
||||
# language-model engine's default.
|
||||
"instructions": AutotagConfig.all_instructions(),
|
||||
"default_instructions": dict(AutotagConfig.DEFAULT_INSTRUCTIONS),
|
||||
"default_prompt": AutotagConfig.TEMPLATE,
|
||||
"model_loaded": loaded,
|
||||
"loaded_model_type": loaded_type,
|
||||
"wd14_general_threshold": AutotagConfig.GENERAL_THRESHOLD,
|
||||
@@ -448,7 +424,6 @@ class AutotagRoutesMixin:
|
||||
title = f"Tagging model download · {label}"
|
||||
_download_titles.clear()
|
||||
_download_titles[owner] = title
|
||||
admitted = False
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
MODEL_DOWNLOAD_JOB_KIND,
|
||||
@@ -464,13 +439,13 @@ class AutotagRoutesMixin:
|
||||
self._run_model_download, service, model_type, owner
|
||||
),
|
||||
)
|
||||
admitted = True
|
||||
except JobBusy as refused:
|
||||
return None, busy_response(refused)
|
||||
finally:
|
||||
if not admitted: # JobBusy, or anything else: give the gate back.
|
||||
model_gate.release(owner)
|
||||
_download_titles.pop(owner, None)
|
||||
except BaseException as refused:
|
||||
# JobBusy, or anything else the registry raised: give the gate back.
|
||||
model_gate.release(owner)
|
||||
_download_titles.pop(owner, None)
|
||||
if isinstance(refused, JobBusy):
|
||||
return None, busy_response(refused)
|
||||
raise
|
||||
|
||||
def ended(_task):
|
||||
model_gate.release(owner)
|
||||
@@ -563,10 +538,7 @@ class AutotagRoutesMixin:
|
||||
else:
|
||||
prompt = params.get("prompt")
|
||||
template = prompt if isinstance(prompt, str) and prompt.strip() else None
|
||||
options = {
|
||||
"instructions": template
|
||||
or AutotagConfig.default_instructions_text(model_type)
|
||||
}
|
||||
options = {"instructions": template or AutotagConfig.TEMPLATE}
|
||||
try:
|
||||
_model, options = validate_options(
|
||||
model_type,
|
||||
@@ -726,86 +698,6 @@ class AutotagRoutesMixin:
|
||||
return None, "existing must be one of " + ", ".join(EXISTING_POLICIES)
|
||||
return MODELS[model_type], None
|
||||
|
||||
async def preview_skipped_tags(self, request):
|
||||
"""Which known tags a Skipped tags list would hide (#431); no write.
|
||||
|
||||
``{entries}`` is checked as a save checks it (the same 400 sentence).
|
||||
The known tags are the library's plus the general and character names
|
||||
of each downloaded WD14 label list; no model is loaded and no image is
|
||||
opened. Refused (409 ``busy``) while an exclusive job runs.
|
||||
"""
|
||||
from ..tagging_defaults import validate_skipped_tags
|
||||
|
||||
data, error = await self._batch_body(
|
||||
request, ("entries",), SKIPPED_PREVIEW_BODY_MAX
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
try:
|
||||
entries = validate_skipped_tags(data["entries"])
|
||||
except ValueError as exc:
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=400)
|
||||
try:
|
||||
async with self._jobs.lease(SKIPPED_PREVIEW_LEASE):
|
||||
result = await self._run_in_executor(self._skipped_preview, entries)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
except Exception:
|
||||
self.logger.exception("The Skipped tags preview failed")
|
||||
return web.json_response(
|
||||
{"success": False, "error": _INTERNAL_ERROR}, status=500
|
||||
)
|
||||
return web.json_response({"success": True, **result})
|
||||
|
||||
def _skipped_preview(self, entries):
|
||||
"""Worker thread: the preview's counts over the known tags."""
|
||||
from ..autotag import MODELS, get_autotag_service
|
||||
from ..tagging_defaults import (
|
||||
WD14_MODEL_TYPES,
|
||||
SkipMatcher,
|
||||
wd14_label_names,
|
||||
)
|
||||
|
||||
library = self.db.get_all_tags()
|
||||
known = {} # case-folded -> (0 library / 1 label list, spelling)
|
||||
for name in library:
|
||||
known.setdefault(name.casefold(), (0, name))
|
||||
lists = []
|
||||
models_dir = getattr(get_autotag_service(), "models_dir", None)
|
||||
for model_type in sorted(WD14_MODEL_TYPES, key=lambda t: MODELS[t]["name"]):
|
||||
if models_dir is None:
|
||||
break
|
||||
path = Path(models_dir) / MODELS[model_type]["subdir"] / "selected_tags.csv"
|
||||
try:
|
||||
names = wd14_label_names(path)
|
||||
except (OSError, ValueError) as exc:
|
||||
if path not in _MISSING_LABEL_LISTS: # logged once per list
|
||||
_MISSING_LABEL_LISTS.add(path)
|
||||
self.logger.debug(f"No WD14 label list for {model_type}: {exc}")
|
||||
continue
|
||||
_MISSING_LABEL_LISTS.discard(path)
|
||||
lists.append(MODELS[model_type]["name"])
|
||||
for name in names:
|
||||
known.setdefault(name.casefold(), (1, name))
|
||||
matchers = [(entry, SkipMatcher([entry])) for entry in entries]
|
||||
whole = SkipMatcher(entries)
|
||||
hidden = [item for item in known.values() if whole.matches(item[1])]
|
||||
hidden.sort(key=lambda item: (item[0], item[1].casefold()))
|
||||
per_entry = [
|
||||
{
|
||||
"entry": entry,
|
||||
"count": sum(1 for _, name in hidden if matcher.matches(name)),
|
||||
}
|
||||
for entry, matcher in matchers
|
||||
]
|
||||
return {
|
||||
"count": len(hidden),
|
||||
"sample": [name for _, name in hidden[:SKIPPED_PREVIEW_SAMPLE]],
|
||||
"sample_limit": SKIPPED_PREVIEW_SAMPLE,
|
||||
"per_entry": per_entry,
|
||||
"sources": {"library": len(library), "wd14": lists},
|
||||
}
|
||||
|
||||
async def preview_autotag_batch(self, request):
|
||||
"""Review Tag output images: counts, model, backup and disk; no write.
|
||||
|
||||
@@ -957,12 +849,9 @@ class AutotagRoutesMixin:
|
||||
"""Return the real path when it is a media file inside an output directory."""
|
||||
if Path(str(raw_path)).suffix.lower() not in _MEDIA_EXTENSIONS:
|
||||
return None
|
||||
roots = list(self._get_all_output_dirs())
|
||||
for output_dir in roots:
|
||||
for output_dir in self._get_all_output_dirs():
|
||||
if path_is_within(raw_path, output_dir):
|
||||
real = os.path.realpath(raw_path)
|
||||
if in_trash_below_roots(real, roots):
|
||||
return None # thrown away, not an output (#543)
|
||||
return real if os.path.isfile(real) else None
|
||||
return None
|
||||
|
||||
@@ -1109,10 +998,7 @@ class AutotagRoutesMixin:
|
||||
else:
|
||||
prompt = data.get("prompt")
|
||||
template = prompt if isinstance(prompt, str) and prompt.strip() else None
|
||||
options = {
|
||||
"instructions": template
|
||||
or AutotagConfig.default_instructions_text(model_type)
|
||||
}
|
||||
options = {"instructions": template or AutotagConfig.TEMPLATE}
|
||||
use_gpu = data.get("use_gpu", True)
|
||||
options["use_gpu"] = use_gpu if isinstance(use_gpu, bool) else True
|
||||
return validate_options(
|
||||
@@ -1152,12 +1038,12 @@ class AutotagRoutesMixin:
|
||||
return self._bad_request('existing must be "skip" or "merge"')
|
||||
try:
|
||||
options = self._single_options(data, model_type)
|
||||
except OptionError as error:
|
||||
return self._bad_request(str(error))
|
||||
except (ValueError, TypeError):
|
||||
return self._bad_request(
|
||||
"general_threshold and character_threshold must be numeric"
|
||||
)
|
||||
except OptionError as error:
|
||||
return self._bad_request(str(error))
|
||||
target, error = await self._select_autotag_target(data)
|
||||
if error is not None:
|
||||
return error
|
||||
@@ -1206,8 +1092,6 @@ class AutotagRoutesMixin:
|
||||
|
||||
async def _suggest_single(self, target, model_type, options, base):
|
||||
from ..autotag import MODELS, MissingBackendPackages, get_autotag_service
|
||||
from ..config import AutotagConfig
|
||||
from ..tagging_defaults import SkipMatcher
|
||||
from .annotation_suggest import model_refusal
|
||||
from .annotations import _pinned_suggestions, _settled, _submit
|
||||
|
||||
@@ -1224,8 +1108,6 @@ class AutotagRoutesMixin:
|
||||
owner = model_gate.new_owner("legacy-single")
|
||||
if not model_gate.try_acquire(owner):
|
||||
return model_busy_response()
|
||||
# #431: the legacy review hides the Skipped tags too (19A).
|
||||
skip = SkipMatcher(AutotagConfig.SKIPPED_TAGS)
|
||||
store = self._capture_store()
|
||||
try:
|
||||
epoch = await self._run_in_executor(self.db.get_execution_epoch)
|
||||
@@ -1265,7 +1147,6 @@ class AutotagRoutesMixin:
|
||||
capture,
|
||||
model_type,
|
||||
options,
|
||||
skip,
|
||||
on_skipped=functools.partial(store.release_use, capture.token),
|
||||
)
|
||||
except MissingBackendPackages as exc:
|
||||
|
||||
@@ -971,7 +971,6 @@ class BackupRestoreRoutesMixin:
|
||||
line = restored_line(staged.summary)
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=line)
|
||||
return None
|
||||
|
||||
|
||||
async def _outlive_cancel(future):
|
||||
|
||||
@@ -574,13 +574,12 @@ class DuplicateFilesMixin:
|
||||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return None
|
||||
return
|
||||
try:
|
||||
await self._remove_duplicates(job, db, plan, backup)
|
||||
finally:
|
||||
if backup:
|
||||
await self._prune_backup_pool(job, db)
|
||||
return None
|
||||
|
||||
async def _remove_duplicates(self, job, db, plan, backup):
|
||||
targets = plan.payload["targets"]
|
||||
|
||||
@@ -16,11 +16,6 @@ from pathlib import Path
|
||||
from .admin import DELETABLE_MEDIA_EXTENSIONS
|
||||
from .maintenance import DUPLICATE_FILES
|
||||
|
||||
try:
|
||||
from ...utils.media_folders import in_trash_below_roots, is_trash_folder
|
||||
except ImportError:
|
||||
from utils.media_folders import in_trash_below_roots, is_trash_folder
|
||||
|
||||
# Keep entries one review takes; with paths of a few hundred characters the
|
||||
# list stays inside the preview body cap (DUPLICATE_PREVIEW_BODY_MAX, 1 MiB).
|
||||
MAX_KEEP = 2000
|
||||
@@ -202,8 +197,6 @@ def file_problem(
|
||||
parts = Path(os.path.relpath(lexical, base)).parts
|
||||
if "thumbnails" in parts or ".." in parts:
|
||||
return OUTSIDE_SOURCE
|
||||
if any(is_trash_folder(part) for part in parts[:-1]):
|
||||
return OUTSIDE_SOURCE # never removed from a trash folder (#543)
|
||||
if os.path.splitext(lexical)[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||||
return OUTSIDE_SOURCE
|
||||
try:
|
||||
@@ -219,8 +212,6 @@ def file_problem(
|
||||
try:
|
||||
if not _lexically_inside(_canonical_path(path), [_canonical_path(root)]):
|
||||
return OUTSIDE_SOURCE # a linked folder on the way
|
||||
if in_trash_below_roots(path, [root]):
|
||||
return OUTSIDE_SOURCE # a linked folder into a trash folder
|
||||
if keeper is not None:
|
||||
same = keeper_problem(path, root, info, keeper, keeper_root)
|
||||
if same:
|
||||
|
||||
@@ -27,7 +27,7 @@ def _identity(path):
|
||||
except OSError:
|
||||
return None
|
||||
if info.st_ino:
|
||||
return ("inode", (info.st_dev, info.st_ino))
|
||||
return ("inode", info.st_dev, info.st_ino)
|
||||
return ("path", os.path.normcase(os.path.realpath(str(path))))
|
||||
|
||||
|
||||
|
||||
@@ -843,10 +843,7 @@ class HistoryRoutesMixin:
|
||||
raise
|
||||
|
||||
async def execute():
|
||||
# An inspection reads the file's header and text: the
|
||||
# interactive image pool (#523); media stays on the default.
|
||||
run = self._run_image if kind == "inspection" else self._run_in_executor
|
||||
pending = asyncio.create_task(run(project))
|
||||
pending = asyncio.create_task(self._run_in_executor(project))
|
||||
try:
|
||||
return await asyncio.shield(pending)
|
||||
except asyncio.CancelledError:
|
||||
|
||||
@@ -17,7 +17,6 @@ from aiohttp import web
|
||||
|
||||
from . import thumbnail_settings
|
||||
from .images import _canonical, _public_path, SERVABLE_EXTENSIONS, VIDEO_EXTENSIONS
|
||||
from .images import in_trash_below_roots
|
||||
from .images import existing_media_thumbnail, media_thumbnail_rel
|
||||
|
||||
try:
|
||||
@@ -199,9 +198,6 @@ def resolve_current_candidate(
|
||||
if namespace is None or not namespace.is_dir():
|
||||
return _unavailable("root_unavailable")
|
||||
target = _canonical(lexical)
|
||||
if in_trash_below_roots(target, [namespace]):
|
||||
# Every kind and the outputs listing come through here (#543).
|
||||
return _unavailable("missing")
|
||||
if not _authorized(target, namespace, scope):
|
||||
return _unavailable("not_authorized")
|
||||
if target.suffix.lower() not in SERVABLE_EXTENSIONS:
|
||||
@@ -252,10 +248,6 @@ def _open_current_target(
|
||||
A *preview* (a file under ``thumbnails/``) may also be JPEG XL (#317)."""
|
||||
if _canonical(target) != target:
|
||||
raise CurrentLocationUnavailable("not_authorized", stale=True)
|
||||
if in_trash_below_roots(target, [namespace]):
|
||||
# Also at the open: the cover chooses its preview after the candidate
|
||||
# check, and a preview can be a link into a trash folder (#543).
|
||||
raise CurrentLocationUnavailable("missing")
|
||||
if not _authorized(target, namespace, scope):
|
||||
raise CurrentLocationUnavailable("not_authorized")
|
||||
suffix = target.suffix.lower()
|
||||
|
||||
@@ -330,13 +330,12 @@ class ImageDeleteMixin:
|
||||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return None
|
||||
return
|
||||
try:
|
||||
await self._delete_images(job, db, targets, mode, backup)
|
||||
finally:
|
||||
if backup:
|
||||
await self._prune_backup_pool(job, db)
|
||||
return None
|
||||
|
||||
async def _delete_images(self, job, db, targets, mode, backup):
|
||||
job.update(total=len(targets), status_line=TITLES[mode])
|
||||
|
||||
@@ -39,7 +39,6 @@ from typing import NamedTuple
|
||||
from aiohttp import web
|
||||
|
||||
from .. import model_gate
|
||||
from ..tagging_defaults import SkipMatcher
|
||||
from .admin import _plural, _time_left
|
||||
from .annotation_suggest import (
|
||||
OptionError,
|
||||
@@ -171,8 +170,6 @@ class _Run:
|
||||
owner: object
|
||||
service: object = None
|
||||
roots: dict = field(default_factory=dict)
|
||||
# The job's Skipped tags, compiled once at its start (#431).
|
||||
skip: object = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -186,9 +183,6 @@ class _Spec:
|
||||
options: dict | None = None
|
||||
# A listing filter the start resolves into ``targets`` (#392).
|
||||
filter: dict | None = None
|
||||
# The Skipped tags when the job was requested (#431); a later save does
|
||||
# not change a running job.
|
||||
skip: tuple = ()
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
@@ -244,11 +238,7 @@ def _parse_auto(targets, data):
|
||||
)
|
||||
except OptionError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
skip = tuple(AutotagConfig.SKIPPED_TAGS)
|
||||
return (
|
||||
_Spec("auto", targets, model_type=model_type, options=options, skip=skip),
|
||||
None,
|
||||
)
|
||||
return _Spec("auto", targets, model_type=model_type, options=options), None
|
||||
|
||||
|
||||
def parse_image_tag_request(data: dict):
|
||||
@@ -339,8 +329,6 @@ class AutotagRun:
|
||||
keep_in_memory: bool
|
||||
backup_on: bool
|
||||
owner: object
|
||||
# The Skipped tags when the job was started (#431).
|
||||
skip: tuple = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -412,14 +400,6 @@ def _load_model(service, model_type: str, use_gpu: bool) -> None:
|
||||
raise RuntimeError(f"{model_type} could not be loaded")
|
||||
|
||||
|
||||
def _skipped_tags_line(entries: tuple) -> str:
|
||||
"""The job log's one line about the Skipped tags it runs with (#431)."""
|
||||
if not entries:
|
||||
return "Skipped tags: none"
|
||||
count = len(entries)
|
||||
return f"Skipped tags: {count} {'entry' if count == 1 else 'entries'}"
|
||||
|
||||
|
||||
def _refusal(status: int, code: str, message: str) -> Refusal:
|
||||
response = web.json_response(
|
||||
{"success": False, "code": code, "error": message}, status=status
|
||||
@@ -527,7 +507,6 @@ class ImageTagJobsMixin:
|
||||
released when the task ends, also before its runner ever starts.
|
||||
"""
|
||||
from ..autotag import MODELS, get_autotag_service
|
||||
from ..config import AutotagConfig
|
||||
|
||||
backup_on = self._safety_backup_policy() == "on"
|
||||
status = await self._run_in_executor(get_autotag_service().get_models_status)
|
||||
@@ -550,15 +529,8 @@ class ImageTagJobsMixin:
|
||||
response = model_busy_response()
|
||||
return None, Refusal(409, "model-busy", MODEL_BUSY_ERROR, response)
|
||||
run = AutotagRun(
|
||||
model_type,
|
||||
existing,
|
||||
options,
|
||||
keep_in_memory,
|
||||
backup_on,
|
||||
owner,
|
||||
skip=tuple(AutotagConfig.SKIPPED_TAGS),
|
||||
model_type, existing, options, keep_in_memory, backup_on, owner
|
||||
)
|
||||
admitted = False
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
AUTOTAG_BATCH_JOB_KIND,
|
||||
@@ -569,12 +541,11 @@ class ImageTagJobsMixin:
|
||||
groups=AUTOTAG_GROUP_NAMES,
|
||||
runner=functools.partial(self._run_autotag_batch, run),
|
||||
)
|
||||
admitted = True
|
||||
except JobBusy as refused:
|
||||
return None, _busy_refusal(refused)
|
||||
finally:
|
||||
if not admitted:
|
||||
model_gate.release(owner)
|
||||
except BaseException as refused:
|
||||
model_gate.release(owner)
|
||||
if isinstance(refused, JobBusy):
|
||||
return None, _busy_refusal(refused)
|
||||
raise
|
||||
job.task.add_done_callback(lambda _task: model_gate.release(owner))
|
||||
return job, None
|
||||
|
||||
@@ -625,9 +596,7 @@ class ImageTagJobsMixin:
|
||||
return _batch_cancelled(job)
|
||||
job.publish({"type": "phase", "phase": "model"})
|
||||
job.update(status_line="Loading the tagging model")
|
||||
spec = _Spec(
|
||||
"auto", (), model_type=run.model_type, options=run.options, skip=run.skip
|
||||
)
|
||||
spec = _Spec("auto", (), model_type=run.model_type, options=run.options)
|
||||
service = await self._prepare_tagging_model(job, spec, run.owner)
|
||||
if service is None:
|
||||
return "failed"
|
||||
@@ -653,7 +622,6 @@ class ImageTagJobsMixin:
|
||||
f"Tagging started · {job.scope.get('label') or run.model_type} · "
|
||||
f"{_plural(total, 'image')} · {policy}"
|
||||
)
|
||||
job.log_line(_skipped_tags_line(spec.skip))
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
remaining = await self._autotag_prepass(job, targets, run.existing == "skip")
|
||||
@@ -663,9 +631,7 @@ class ImageTagJobsMixin:
|
||||
spec = dataclasses.replace(
|
||||
spec, targets=tuple(item.target for item in remaining)
|
||||
)
|
||||
loop_run = _Run(
|
||||
job, spec, epoch, run.owner, service, skip=SkipMatcher(spec.skip)
|
||||
)
|
||||
loop_run = _Run(job, spec, epoch, run.owner, service)
|
||||
if not await self._tag_each_file(loop_run, done_before=done):
|
||||
job.update(status_line=STORAGE_RESTORED)
|
||||
job.log_line(f"Tagging stopped · {STORAGE_RESTORED}")
|
||||
@@ -771,9 +737,6 @@ class ImageTagJobsMixin:
|
||||
started += f" · {spec.model_type}"
|
||||
job.log_line(started)
|
||||
run = _Run(job, spec, epoch, owner)
|
||||
if spec.mode == "auto":
|
||||
job.log_line(_skipped_tags_line(spec.skip))
|
||||
run.skip = SkipMatcher(spec.skip)
|
||||
try:
|
||||
if spec.mode == "auto":
|
||||
run.service = await self._prepare_tagging_model(job, spec, owner)
|
||||
@@ -929,7 +892,6 @@ class ImageTagJobsMixin:
|
||||
capture,
|
||||
spec.model_type,
|
||||
spec.options,
|
||||
run.skip,
|
||||
on_skipped=functools.partial(store.release_use, capture.token),
|
||||
)
|
||||
except Exception:
|
||||
|
||||
@@ -39,8 +39,10 @@ from .thumbnail_encode import (
|
||||
PREVIEW_HELD,
|
||||
EncoderClosed,
|
||||
PreviewWriteHeld,
|
||||
replace_unless_held,
|
||||
run_encoder,
|
||||
sweep_stray_temps,
|
||||
temp_sibling,
|
||||
thumbnail_frame,
|
||||
writer_settings,
|
||||
write_thumbnail,
|
||||
@@ -64,12 +66,7 @@ try:
|
||||
from ...database.models import normalize_image_path
|
||||
from ...database.run_attribution import recorded_output_lookup
|
||||
from ...utils.card_seed import card_seed, header_params, parsed_positive
|
||||
from ...utils.media_folders import (
|
||||
in_skipped_folder,
|
||||
in_trash_below_roots,
|
||||
in_trash_folder,
|
||||
is_skipped_folder,
|
||||
)
|
||||
from ...utils.media_folders import in_skipped_folder, is_skipped_folder
|
||||
from ...utils.parallel import map_parallel
|
||||
from ...utils.json_strict import dumps_strict
|
||||
from ...utils.inspection_response import (
|
||||
@@ -83,7 +80,7 @@ try:
|
||||
OpenerUnavailable,
|
||||
RevealGate,
|
||||
RevealTargetChanged,
|
||||
is_same_origin_request,
|
||||
is_local_request,
|
||||
reveal_executor,
|
||||
reveal_in_file_manager,
|
||||
)
|
||||
@@ -95,12 +92,7 @@ except ImportError:
|
||||
from database.models import normalize_image_path
|
||||
from database.run_attribution import recorded_output_lookup
|
||||
from utils.card_seed import card_seed, header_params, parsed_positive
|
||||
from utils.media_folders import (
|
||||
in_skipped_folder,
|
||||
in_trash_below_roots,
|
||||
in_trash_folder,
|
||||
is_skipped_folder,
|
||||
)
|
||||
from utils.media_folders import in_skipped_folder, is_skipped_folder
|
||||
from utils.parallel import map_parallel
|
||||
from utils.json_strict import dumps_strict
|
||||
from utils.inspection_response import (
|
||||
@@ -114,7 +106,7 @@ except ImportError:
|
||||
OpenerUnavailable,
|
||||
RevealGate,
|
||||
RevealTargetChanged,
|
||||
is_same_origin_request,
|
||||
is_local_request,
|
||||
reveal_executor,
|
||||
reveal_in_file_manager,
|
||||
)
|
||||
@@ -285,20 +277,10 @@ def _forbidden(message="Access denied"):
|
||||
return web.json_response({"success": False, "error": message}, status=403)
|
||||
|
||||
|
||||
def _file_not_found():
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Image file not found"},
|
||||
status=404,
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
|
||||
def _media_access_error(path, allowed_dirs):
|
||||
"""The response refusing *path*, or None when it may be read.
|
||||
"""Return a 403 response when *path* must not be served, else None.
|
||||
|
||||
Fails closed: an empty *allowed_dirs* denies everything (403). A file in a
|
||||
trash folder below its root is thrown away, not an output: a 404, as if it
|
||||
were not there (#543), also through a link or another spelling.
|
||||
Fails closed: an empty *allowed_dirs* denies everything.
|
||||
"""
|
||||
if Path(path).suffix.lower() not in SERVABLE_EXTENSIONS:
|
||||
return _forbidden("Only media files can be served")
|
||||
@@ -306,8 +288,6 @@ def _media_access_error(path, allowed_dirs):
|
||||
return _forbidden("No allowed image directories configured")
|
||||
if not any(_is_within(path, root) for root in allowed_dirs):
|
||||
return _forbidden()
|
||||
if in_trash_below_roots(path, allowed_dirs):
|
||||
return _file_not_found()
|
||||
return None
|
||||
|
||||
|
||||
@@ -337,6 +317,11 @@ def _thumbnail_batch_size(workers):
|
||||
return max(4, workers * 2)
|
||||
|
||||
|
||||
# Preview files are written by thumbnail_encode in the configured format
|
||||
# (thumbnail_settings, #317); these names stay importable from here.
|
||||
_temp_sibling = temp_sibling
|
||||
_replace_unless_held = replace_unless_held
|
||||
|
||||
# A preview that could not be replaced (#249 S5 M6). Windows refuses to rename
|
||||
# over a file another process has open (WinError 5 or 32), such as a preview
|
||||
# the gallery is serving; the old preview stays and the file counts as an error.
|
||||
@@ -389,7 +374,6 @@ def _video_placeholder_image(thumbnail_size):
|
||||
font=font,
|
||||
)
|
||||
except (OSError, AttributeError):
|
||||
# a placeholder without its label still works
|
||||
pass
|
||||
return img
|
||||
|
||||
@@ -493,7 +477,6 @@ def _preview_names(folders, folder):
|
||||
except OSError:
|
||||
continue
|
||||
except OSError:
|
||||
# an unreadable folder lists as empty; nothing there to cache
|
||||
pass
|
||||
folders[folder] = names
|
||||
return names
|
||||
@@ -812,12 +795,12 @@ def _reveal_not_found():
|
||||
)
|
||||
|
||||
|
||||
def _not_same_origin():
|
||||
def _not_local():
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "not-same-origin",
|
||||
"error": "This page's address does not match the server's.",
|
||||
"code": "not-local",
|
||||
"error": "Show in folder works only on the computer running ComfyUI.",
|
||||
},
|
||||
status=403,
|
||||
)
|
||||
@@ -996,8 +979,6 @@ def _settled_entry(root, relative):
|
||||
return None
|
||||
if in_skipped_folder(folders) or len(folders) > MAX_SCAN_DEPTH:
|
||||
return None
|
||||
if in_trash_below_roots(path, [root]):
|
||||
return None # a linked folder on the way leads into a trash folder
|
||||
folder = os.fspath(root)
|
||||
for part in ("", *folders):
|
||||
folder = os.path.join(folder, part) if part else folder
|
||||
@@ -1533,7 +1514,7 @@ class ImageRoutesMixin:
|
||||
async def clear_image_favorite_route(request):
|
||||
return await self.update_image_favorite(request, favorite=False)
|
||||
|
||||
# POST, same-origin: opens the server computer's file manager (D38).
|
||||
# POST, local-only: opens this computer's file manager (D38).
|
||||
@routes.post("/prompt_manager/images/reveal")
|
||||
async def reveal_image_route(request):
|
||||
return await self.reveal_image(request)
|
||||
@@ -2780,9 +2761,6 @@ class ImageRoutesMixin:
|
||||
if _has_traversal(filepath):
|
||||
return _forbidden()
|
||||
suffix = Path(filepath).suffix.lower()
|
||||
if in_trash_folder(filepath):
|
||||
# Before any file access, so no preview is made for it (#543).
|
||||
return _file_not_found()
|
||||
if suffix not in SERVABLE_EXTENSIONS and not (
|
||||
# A JPEG XL preview is served from thumbnails/ only (#317).
|
||||
suffix in thumbnail_settings.THUMBNAIL_EXTENSIONS
|
||||
@@ -2806,8 +2784,6 @@ class ImageRoutesMixin:
|
||||
# a fall-through to the next root.
|
||||
if not _is_within(image_path, output_path):
|
||||
return _forbidden()
|
||||
if in_trash_below_roots(image_path, output_dirs):
|
||||
return _file_not_found()
|
||||
if image_path.is_file():
|
||||
return self._file_response(image_path)
|
||||
|
||||
@@ -2859,9 +2835,8 @@ class ImageRoutesMixin:
|
||||
if _is_thumbnail_cache_path(filepath):
|
||||
return _forbidden()
|
||||
image_path = root / filepath
|
||||
denied = _media_access_error(image_path, [root])
|
||||
if denied is not None:
|
||||
return denied
|
||||
if not _is_within(image_path, root):
|
||||
return _forbidden()
|
||||
if not image_path.is_file():
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Image file not found"}, status=404
|
||||
@@ -2975,7 +2950,6 @@ class ImageRoutesMixin:
|
||||
result = await self._run_in_executor(
|
||||
self._generate_gallery_thumbnails_sync, output_dirs
|
||||
)
|
||||
await self._release_job_memory()
|
||||
return web.json_response(result)
|
||||
|
||||
except Exception as e:
|
||||
@@ -3708,12 +3682,12 @@ class ImageRoutesMixin:
|
||||
async def reveal_image(self, request):
|
||||
"""POST ``{root_index, root_scope, path}``: show the file in its folder.
|
||||
|
||||
Same-origin and JSON-only, directly or through a proxy: the folder
|
||||
opens on the computer running ComfyUI, wherever the browser is
|
||||
(maintainer decision, #533). One reveal at a time (429 busy).
|
||||
Local-only (a same-origin browser request from a loopback client, no
|
||||
forwarding header) and JSON-only: the browser must run on the
|
||||
computer whose file manager opens. One reveal at a time (429 busy).
|
||||
"""
|
||||
if not is_same_origin_request(request.headers, request.scheme):
|
||||
return _not_same_origin()
|
||||
if not is_local_request(request.remote, request.headers, request.scheme):
|
||||
return _not_local()
|
||||
if request.content_type != "application/json":
|
||||
# A non-JSON POST is a CORS "simple" request with no preflight.
|
||||
return web.json_response(
|
||||
@@ -3791,9 +3765,7 @@ class ImageRoutesMixin:
|
||||
{
|
||||
"success": False,
|
||||
"code": "unavailable",
|
||||
"error": (
|
||||
"No file manager is available on the computer running ComfyUI."
|
||||
),
|
||||
"error": "No file manager is available on this computer.",
|
||||
},
|
||||
status=501,
|
||||
)
|
||||
@@ -3998,8 +3970,7 @@ class ImageRoutesMixin:
|
||||
**observed,
|
||||
)
|
||||
|
||||
# Header and text reads: the interactive image pool (#523).
|
||||
body = await self._run_image(inspect)
|
||||
body = await self._run_in_executor(inspect)
|
||||
if body is None:
|
||||
return web.json_response(
|
||||
{
|
||||
@@ -4212,22 +4183,6 @@ class ImageRoutesMixin:
|
||||
# (which Windows refuses on open files) is not held up by it.
|
||||
return _CardFacts(width, height, *self._card_parse(path, records))
|
||||
|
||||
def _cached_card_facts(self, entry: tuple) -> _CardFacts | None:
|
||||
path, mtime, root, index = entry[:4]
|
||||
return self._workspace_card_cache.get((index, str(root)), {}).get((path, mtime))
|
||||
|
||||
def _card_facts_of(self, entries: list, *, parse: bool) -> list[_CardFacts]:
|
||||
"""Card facts of *entries*, in order; files not cached yet are read on
|
||||
the interactive image pool (#523), cached ones answer here."""
|
||||
found = [self._cached_card_facts(entry) for entry in entries]
|
||||
missing = [i for i, facts in enumerate(found) if facts is None]
|
||||
read = self._map_image(
|
||||
lambda i: self._card_facts(entries[i], parse=parse), missing
|
||||
)
|
||||
for i, facts in zip(missing, read):
|
||||
found[i] = facts
|
||||
return found
|
||||
|
||||
def _card_facts(self, entry: tuple, *, parse: bool) -> _CardFacts:
|
||||
"""One listed image's card facts, read once per ``(path, mtime)``.
|
||||
|
||||
@@ -4236,11 +4191,11 @@ class ImageRoutesMixin:
|
||||
and drops the rest, and a roots change drops them all. Only files of
|
||||
the current scan are stored, so the cache is bounded by the listing.
|
||||
"""
|
||||
cached = self._cached_card_facts(entry)
|
||||
if cached is not None:
|
||||
return cached
|
||||
path, mtime, root, index = entry[:4]
|
||||
root_key, key = (index, str(root)), (path, mtime)
|
||||
cached = self._workspace_card_cache.get(root_key, {}).get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
facts = self._read_card_facts(path, parse=parse)
|
||||
if facts is None:
|
||||
# Not stored: a locked or half-written file may read next time.
|
||||
@@ -4266,17 +4221,15 @@ class ImageRoutesMixin:
|
||||
if linked
|
||||
else {}
|
||||
)
|
||||
# Videos: the card reads no video metadata. Library previews show
|
||||
# no card facts, so they skip the parse (``parse=linked``).
|
||||
stills = [image for image in images if not image["is_video"]]
|
||||
read = self._card_facts_of(
|
||||
[entries[(image["root_index"], image["path"])] for image in stills],
|
||||
parse=linked,
|
||||
)
|
||||
facts_of = {id(image): facts for image, facts in zip(stills, read)}
|
||||
for image in images:
|
||||
entry = entries[(image["root_index"], image["path"])]
|
||||
facts = facts_of.get(id(image), _NO_CARD_FACTS)
|
||||
# Videos: the card reads no video metadata. Library previews show
|
||||
# no card facts, so they skip the parse (``parse=linked``).
|
||||
facts = (
|
||||
_NO_CARD_FACTS
|
||||
if image["is_video"]
|
||||
else self._card_facts(entry, parse=linked)
|
||||
)
|
||||
image["width"], image["height"] = facts.width, facts.height
|
||||
if not linked:
|
||||
continue
|
||||
|
||||
@@ -250,7 +250,7 @@ class JobRoutesMixin:
|
||||
try:
|
||||
await response.write(_summary_event(job))
|
||||
async with contextlib.aclosing(job.events()) as events:
|
||||
async for _ in events:
|
||||
async for _payload in events:
|
||||
await response.write(_summary_event(job))
|
||||
# The last event is published before the job takes its final state.
|
||||
await response.write(_summary_event(job))
|
||||
|
||||
@@ -204,8 +204,6 @@ class Job:
|
||||
self.total = 0
|
||||
self.counters = dict.fromkeys(groups, 0)
|
||||
self.outcomes = {group: [] for group in groups}
|
||||
# Groups whose report lists every row, beyond MAX_OUTCOME_ROWS (#550).
|
||||
self.complete = {}
|
||||
self.log = []
|
||||
self._cancel_requested = False
|
||||
self.last_event = None
|
||||
@@ -299,36 +297,31 @@ class Job:
|
||||
)
|
||||
self.log_line(log_line)
|
||||
|
||||
def record_counted(self, group, count, rows=(), *, complete=False):
|
||||
def record_counted(self, group, count, rows=()):
|
||||
"""Count ``count`` outcomes in ``group`` at once, listing ``rows``.
|
||||
|
||||
For outcomes that need no log line each (a thumbnail job's already
|
||||
fresh previews): the counter is exact, ``rows`` (``record``'s row
|
||||
shape) are listed up to MAX_OUTCOME_ROWS, and nothing is logged.
|
||||
``complete`` also keeps every row for the report (#550: a record of
|
||||
each change that must not be cut short; the caller bounds it).
|
||||
"""
|
||||
if group not in self.counters:
|
||||
raise ValueError(f"Unknown outcome group {group!r} for {self.kind}")
|
||||
self.counters[group] += max(0, int(count))
|
||||
listed = self.outcomes[group]
|
||||
kept = self.complete.setdefault(group, []) if complete else None
|
||||
for row in rows:
|
||||
if kept is None and len(listed) >= MAX_OUTCOME_ROWS:
|
||||
if len(listed) >= MAX_OUTCOME_ROWS:
|
||||
break
|
||||
index = row.get("root_index")
|
||||
if isinstance(index, bool) or not isinstance(index, int) or index < 0:
|
||||
index = None
|
||||
shaped = {
|
||||
"path": _relative_path(row.get("path")),
|
||||
"root": str(row.get("root") or ""),
|
||||
"root_index": index,
|
||||
"detail": str(row.get("detail") or ""),
|
||||
}
|
||||
if kept is not None:
|
||||
kept.append(shaped)
|
||||
if len(listed) < MAX_OUTCOME_ROWS:
|
||||
listed.append(dict(shaped))
|
||||
listed.append(
|
||||
{
|
||||
"path": _relative_path(row.get("path")),
|
||||
"root": str(row.get("root") or ""),
|
||||
"root_index": index,
|
||||
"detail": str(row.get("detail") or ""),
|
||||
}
|
||||
)
|
||||
|
||||
def finish(self, state="finished"):
|
||||
"""End the job in ``state`` and release every subscriber.
|
||||
@@ -413,9 +406,8 @@ class Job:
|
||||
"""The downloadable report: the summary, every outcome group and the log.
|
||||
|
||||
Each group carries its exact count, its listed rows (at most
|
||||
MAX_OUTCOME_ROWS, or every row of a ``complete`` group) and whether
|
||||
the list is cut short. Rows and log lines already hold root-relative
|
||||
paths only.
|
||||
MAX_OUTCOME_ROWS) and whether the list is cut short. Rows and log lines
|
||||
already hold root-relative paths only.
|
||||
"""
|
||||
summary = self.summary()
|
||||
summary.pop("listed", None)
|
||||
@@ -425,21 +417,16 @@ class Job:
|
||||
"generated_at": self._clock(),
|
||||
"job": summary,
|
||||
"outcomes": {
|
||||
group: self._report_group(group, rows)
|
||||
group: {
|
||||
"count": self.counters.get(group, len(rows)),
|
||||
"rows": [dict(row) for row in rows],
|
||||
"truncated": self.counters.get(group, len(rows)) > len(rows),
|
||||
}
|
||||
for group, rows in self.outcomes.items()
|
||||
},
|
||||
"log": list(self.log),
|
||||
}
|
||||
|
||||
def _report_group(self, group, listed):
|
||||
rows = self.complete.get(group, listed)
|
||||
count = self.counters.get(group, len(rows))
|
||||
return {
|
||||
"count": count,
|
||||
"rows": [dict(row) for row in rows],
|
||||
"truncated": count > len(rows),
|
||||
}
|
||||
|
||||
def report_filename(self):
|
||||
"""The download name: prompt-manager-KIND-YYYYMMDD-HHMM.json (local time)."""
|
||||
stamp = time.strftime("%Y%m%d-%H%M", time.localtime(self._clock()))
|
||||
|
||||
@@ -207,7 +207,6 @@ class LibraryLocationRoutesMixin:
|
||||
try:
|
||||
size = os.path.getsize(location.db_path)
|
||||
except OSError:
|
||||
# no library file yet: report size 0
|
||||
pass
|
||||
return {
|
||||
"path": self._library_public_path(location.db_path),
|
||||
@@ -374,7 +373,6 @@ class LibraryLocationRoutesMixin:
|
||||
line += ". Logs move at the next restart."
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=line)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _fail_library_move(job, code, message):
|
||||
|
||||
@@ -205,4 +205,3 @@ class LogCleanupMixin:
|
||||
line += f", {failed} could not be deleted"
|
||||
job.log_line(line)
|
||||
job.update(processed=len(files), progress=100, status_line=line)
|
||||
return None
|
||||
|
||||
@@ -436,7 +436,7 @@ class LoraImportRoutesMixin:
|
||||
job.log_line(f"{CANCELLED} {CACHE_NOTE}")
|
||||
job.update(status_line=CANCELLED)
|
||||
job.finish("cancelled")
|
||||
return None
|
||||
return
|
||||
except Exception:
|
||||
self.logger.exception("LoRA import staging failed")
|
||||
return _fail_job(job, "import-failed", f"{FAILED} {CACHE_NOTE}")
|
||||
@@ -455,10 +455,10 @@ class LoraImportRoutesMixin:
|
||||
job.log_line(f"{CANCELLED} {CACHE_NOTE}")
|
||||
job.update(status_line=CANCELLED)
|
||||
job.finish("cancelled")
|
||||
return None
|
||||
return
|
||||
ok, _backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return None
|
||||
return
|
||||
# I1: the files are listed and fingerprinted again right before the
|
||||
# replace, so a file changed, removed or added while the downloads ran
|
||||
# (LoraManager refreshing its CivitAI info) aborts with nothing deleted.
|
||||
@@ -466,7 +466,6 @@ class LoraImportRoutesMixin:
|
||||
return _fail_job(job, "plan-changed", f"{SOURCE_CHANGED} {CACHE_NOTE}")
|
||||
job.update(status_line="Replacing the LoRA collection", progress=95)
|
||||
await self._replace_in_job(job, db, plan, staged)
|
||||
return None
|
||||
|
||||
def _count_downloads(self, job, staged):
|
||||
downloaded = staged.stats.get("downloaded", 0)
|
||||
@@ -519,4 +518,3 @@ class LoraImportRoutesMixin:
|
||||
staged.stats.get("download_errors", 0),
|
||||
)
|
||||
job.update(progress=100, processed=job.total, status_line=line)
|
||||
return None
|
||||
|
||||
@@ -44,7 +44,6 @@ _FILES_CHANGED = (
|
||||
"The LoRA metadata files changed during the scan; the previous imports "
|
||||
"were kept. Run the scan again."
|
||||
)
|
||||
_LORA_SCAN_FAILED = "The LoRA import failed. Check the server log for details."
|
||||
_IMPORTS_CHANGED = (
|
||||
"The imported LoRA prompts changed during the scan; the previous imports "
|
||||
"were kept. Run the scan again."
|
||||
@@ -408,21 +407,10 @@ class LoraIntegrationMixin:
|
||||
return
|
||||
if isinstance(error, PlanChanged):
|
||||
message = _IMPORTS_CHANGED
|
||||
elif isinstance(
|
||||
error,
|
||||
(
|
||||
LoraFilesChanged,
|
||||
LoraSettingsUnusable,
|
||||
EmptyReplacement,
|
||||
PairDependencyError,
|
||||
),
|
||||
):
|
||||
elif isinstance(error, (LoraFilesChanged, LoraSettingsUnusable)):
|
||||
message = str(error)
|
||||
elif isinstance(error, OSError):
|
||||
# Reduced to its reason and the file's basename.
|
||||
message = self._public_error(error)
|
||||
else:
|
||||
message = _LORA_SCAN_FAILED
|
||||
message = self._public_error(error)
|
||||
if isinstance(
|
||||
error,
|
||||
(
|
||||
@@ -548,7 +536,6 @@ class LoraIntegrationMixin:
|
||||
except asyncio.CancelledError:
|
||||
interrupted = True
|
||||
except Exception:
|
||||
# the await after this loop raises it for the caller
|
||||
pass
|
||||
if interrupted:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
"""Maintenance routes: reviewed, read-only plans (#249 System S6).
|
||||
|
||||
``POST /prompt_manager/maintenance/preview {operation, options:{}}`` reviews
|
||||
one operation (``duplicate_prompts``, ``optimize``, ``unused_components``,
|
||||
``missing_links`` or ``pair_metadata``, #550; ``database/maintenance.py``;
|
||||
``duplicate_files``,
|
||||
one operation (``duplicate_prompts``, ``optimize``, ``unused_components`` or
|
||||
``missing_links``; ``database/maintenance.py``; ``duplicate_files``,
|
||||
``py/api/duplicate_files.py``; ``rotated_logs``, the rotated log copies,
|
||||
``py/api/log_cleanup.py``, which has its own apply job) off the event loop and
|
||||
answers a ``Plan``: ``{success, token, operation, expires_at, scope:{count,
|
||||
@@ -98,11 +97,7 @@ MAINTENANCE_TITLES = {
|
||||
"unused_components": "Unused prompt components",
|
||||
"missing_links": "Missing-file links",
|
||||
"workflow_storage": "Store each workflow once",
|
||||
# #550, beyond the concept: fills old Pair prompts' empty category and tags.
|
||||
"pair_metadata": "Pair category and tags",
|
||||
}
|
||||
# Operations whose job keeps every changed row for its report (#550).
|
||||
COMPLETE_RECORD = frozenset({"pair_metadata"})
|
||||
_OPERATION_ERROR = "operation must be one of: " + ", ".join(PREVIEW_OPERATIONS)
|
||||
MAINTENANCE_GROUPS = ("changed", "protected", "errors")
|
||||
MAX_TOKEN_LENGTH = 128
|
||||
@@ -286,27 +281,9 @@ def short_space_message(short):
|
||||
)
|
||||
|
||||
|
||||
def _pair_metadata_line(result):
|
||||
count = result["changed"]
|
||||
noun = (
|
||||
"prompt from its saved pairs"
|
||||
if count == 1
|
||||
else "prompts from their saved pairs"
|
||||
)
|
||||
categories = result.get("categories", 0)
|
||||
tag_sets = result.get("tag_sets", 0)
|
||||
return (
|
||||
f"Filled {count:,} {noun} "
|
||||
f"({categories:,} {'category' if categories == 1 else 'categories'}, "
|
||||
f"{tag_sets:,} {'tag set' if tag_sets == 1 else 'tag sets'})"
|
||||
)
|
||||
|
||||
|
||||
def _changed_line(operation, count, marks=0, result=None):
|
||||
def _changed_line(operation, count, marks=0):
|
||||
if operation == "optimize":
|
||||
return "Optimized the database"
|
||||
if operation == "pair_metadata":
|
||||
return _pair_metadata_line(result or {"changed": count})
|
||||
if operation == "duplicate_prompts":
|
||||
noun = "duplicate prompt" if count == 1 else "duplicate prompts"
|
||||
return f"Merged {count:,} {noun}"
|
||||
@@ -691,31 +668,11 @@ class MaintenanceRoutesMixin:
|
||||
rows = await self._run_in_executor(
|
||||
self._maintenance_rows, plan.operation, result["rows"]
|
||||
)
|
||||
job.record_counted(
|
||||
"changed",
|
||||
result["changed"],
|
||||
rows,
|
||||
complete=plan.operation in COMPLETE_RECORD,
|
||||
)
|
||||
if "kept" in result:
|
||||
# #550: each kept record with its reason, like the changed ones.
|
||||
kept = [
|
||||
{
|
||||
"path": "",
|
||||
"detail": f"Prompt {item['prompt_id']}: kept, "
|
||||
+ "; ".join(item["reasons"]),
|
||||
}
|
||||
for item in result["kept"]
|
||||
]
|
||||
job.record_counted("protected", len(kept), kept, complete=True)
|
||||
else:
|
||||
job.record_counted("protected", payload["protected"])
|
||||
line = _changed_line(
|
||||
plan.operation, result["changed"], result.get("marks", 0), result
|
||||
)
|
||||
job.record_counted("changed", result["changed"], rows)
|
||||
job.record_counted("protected", payload["protected"])
|
||||
line = _changed_line(plan.operation, result["changed"], result.get("marks", 0))
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
return None
|
||||
|
||||
def _maintenance_rows(self, operation, described):
|
||||
"""Outcome rows for the changed records (root-relative paths only)."""
|
||||
@@ -733,8 +690,6 @@ class MaintenanceRoutesMixin:
|
||||
}
|
||||
)
|
||||
return rows
|
||||
if operation == "pair_metadata":
|
||||
return [{"path": "", "detail": _filled_detail(item)} for item in described]
|
||||
if operation == "duplicate_prompts":
|
||||
return [
|
||||
{
|
||||
@@ -752,16 +707,6 @@ class MaintenanceRoutesMixin:
|
||||
]
|
||||
|
||||
|
||||
def _filled_detail(item):
|
||||
"""``Prompt 12: category “portrait”; tags a, b``: one filled prompt (#550)."""
|
||||
parts = []
|
||||
if "category" in item:
|
||||
parts.append(f"category “{item['category']}”")
|
||||
if item.get("tags"):
|
||||
parts.append("tags " + ", ".join(item["tags"]))
|
||||
return f"Prompt {item['prompt_id']}: " + "; ".join(parts)
|
||||
|
||||
|
||||
def backup_note(backup):
|
||||
"""The status line's backup part: the file name, or that there is none."""
|
||||
return f"Safety backup {backup}" if backup else NO_BACKUP
|
||||
|
||||
@@ -9,15 +9,13 @@ by the caller.
|
||||
import os
|
||||
import re
|
||||
|
||||
from .images import _file_not_found, _forbidden, _has_traversal
|
||||
from .images import _forbidden, _has_traversal
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
from ...utils.media_capture import CAPTURE_EXTENSIONS
|
||||
from ...utils.media_folders import in_trash_folder
|
||||
except ImportError:
|
||||
from utils.media_capture import CAPTURE_EXTENSIONS
|
||||
from utils.media_folders import in_trash_folder
|
||||
|
||||
MAX_TARGET_PATH = 1024
|
||||
TARGET_FIELDS = frozenset({"root_index", "root_scope", "path"})
|
||||
@@ -31,9 +29,7 @@ def media_target(data: object, *, media_message: str = TAGGING_MEDIA_MESSAGE):
|
||||
Only the listing identity is accepted: an int root index, the root's
|
||||
64-hex scope and a bounded root-relative POSIX media path, never an
|
||||
absolute, drive-qualified, backslash or ``..`` path (a 400 for a malformed
|
||||
shape, a 403 for an unsafe or non-media path, carrying *media_message*, a
|
||||
404 for a path in a trash folder, #543). A link or a short-name spelling
|
||||
is refused later by the capture store's alias check.
|
||||
shape, a 403 for an unsafe or non-media path, carrying *media_message*).
|
||||
"""
|
||||
if not isinstance(data, dict) or set(data) != TARGET_FIELDS:
|
||||
return None, bad_request("target must name root_index, root_scope and path")
|
||||
@@ -55,6 +51,4 @@ def media_target(data: object, *, media_message: str = TAGGING_MEDIA_MESSAGE):
|
||||
return None, _forbidden()
|
||||
if os.path.splitext(parts[-1])[1].lower() not in CAPTURE_EXTENSIONS:
|
||||
return None, _forbidden(media_message)
|
||||
if in_trash_folder(relative):
|
||||
return None, _file_not_found()
|
||||
return (index, scope, relative), None
|
||||
|
||||
@@ -102,12 +102,7 @@ class PairRoutesMixin:
|
||||
validate_prompt_text(negative)
|
||||
fields = metadata_fields(data)
|
||||
pair_id = await self._run_in_executor(
|
||||
self.db.save_prompt_pair,
|
||||
positive,
|
||||
negative,
|
||||
# The node's Save pair button: as a run, a new prompt gets the metadata.
|
||||
fill_new_positive=True,
|
||||
**fields,
|
||||
self.db.save_prompt_pair, positive, negative, **fields
|
||||
)
|
||||
pair = await self._run_in_executor(self.db.get_prompt_pair, pair_id)
|
||||
return web.json_response({"success": True, "pair": pair})
|
||||
|
||||
@@ -9,10 +9,7 @@ candidates and the dependency fingerprint. ``POST .../items/{id}/decision``
|
||||
records one explicit decision: Apply links the chosen sampler's pair only
|
||||
when the capture is the item's bytes, the current file still hashes to them,
|
||||
the evidence they yield is unchanged and the ingester's compatible-only rules
|
||||
allow it; Leave unresolved links nothing. ``POST .../items/{id}/link`` (#536)
|
||||
links a Library prompt the reviewer chose, under the same capture and
|
||||
compatibility guards; it is their choice, so what the bytes resolve to is not
|
||||
asked.
|
||||
allow it; Leave unresolved links nothing.
|
||||
|
||||
Targets are formatted from the recorded spelling against the current roots
|
||||
(no file access beyond the roots' real paths); a file under no current root
|
||||
@@ -31,13 +28,12 @@ import urllib.parse
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .annotations import _capture_error, _error
|
||||
from .annotations import _HEX64, _TOKEN, _UUID, _capture_error, _error
|
||||
from .images import media_thumbnail_url
|
||||
from .inspection_service import VIDEO_EXTENSIONS
|
||||
from .job_routes import busy_response
|
||||
from .jobs import JobBusy
|
||||
from .prompts import bad_request
|
||||
from .review_guards import UNRESOLVED, _decision_guards, _link_guards
|
||||
|
||||
try:
|
||||
from ...database.library import display_title
|
||||
@@ -63,6 +59,17 @@ MAX_CURSOR = 4096
|
||||
MAX_BODY_BYTES = 64 * 1024
|
||||
# The serialized detail stays under this; past it the detail is ``limited``.
|
||||
MAX_DETAIL_BYTES = 8 * 1024 * 1024
|
||||
UNRESOLVED = "unresolved"
|
||||
_DECISION_FIELDS = frozenset(
|
||||
{
|
||||
"expected_epoch",
|
||||
"expected_revision",
|
||||
"capture_token",
|
||||
"sha256",
|
||||
"dependency_fingerprint",
|
||||
"choice",
|
||||
}
|
||||
)
|
||||
_CURSOR_SECRET = os.urandom(32)
|
||||
# The real path of a spelling; module-level so a test can fake a junction.
|
||||
_realpath = os.path.realpath
|
||||
@@ -77,7 +84,6 @@ _STATUS = {
|
||||
"storage-changed": 409,
|
||||
"source-changed": 409,
|
||||
"capture-expired": 409,
|
||||
"prompt-missing": 409,
|
||||
}
|
||||
_MESSAGES = {
|
||||
"review-missing": "This review item no longer exists. Reload the queue.",
|
||||
@@ -89,9 +95,6 @@ _MESSAGES = {
|
||||
"sampler path cannot be attached safely. The existing links were "
|
||||
"preserved."
|
||||
),
|
||||
"prompt-missing": (
|
||||
"That prompt or recipe is no longer in the Library. Search again."
|
||||
),
|
||||
"dependencies-changed": (
|
||||
"The image's saved links changed since this item was opened. Reopen it "
|
||||
"to see the current association."
|
||||
@@ -248,7 +251,6 @@ class ReviewRoutesMixin:
|
||||
routes.get(ITEMS_URL)(self.list_review_items)
|
||||
routes.get(ITEMS_URL + "/{item_id}")(self.get_review_item)
|
||||
routes.post(ITEMS_URL + "/{item_id}/decision")(self.decide_review_item)
|
||||
routes.post(ITEMS_URL + "/{item_id}/link")(self.link_review_item)
|
||||
|
||||
# -- shared ----------------------------------------------------------------
|
||||
|
||||
@@ -462,43 +464,6 @@ class ReviewRoutesMixin:
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def link_review_item(self, request: web.Request) -> web.Response:
|
||||
"""POST the Library prompt the reviewer chose for one pending item.
|
||||
|
||||
Under the same lease as a decision. The capture must be the item's
|
||||
current bytes; the choice and the file's links are checked in the
|
||||
decision's transaction.
|
||||
"""
|
||||
try:
|
||||
async with self._jobs.lease(REVIEW_DECISION_LEASE):
|
||||
return await self._link_review_item(request)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def _link_review_item(self, request: web.Request) -> web.Response:
|
||||
data, error = await self._read_json_body(request, max_bytes=MAX_BODY_BYTES)
|
||||
if error is not None:
|
||||
return error
|
||||
guards, error = _link_guards(data)
|
||||
if error is not None:
|
||||
return error
|
||||
try:
|
||||
item = await self._run_in_executor(
|
||||
self.db.review_item, request.match_info["item_id"]
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Review item read failed")
|
||||
return _failed()
|
||||
if item is None:
|
||||
return _coded("review-missing")
|
||||
verified, error = await self._verified_review_capture(
|
||||
item, guards, need_metadata=True
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
manual = {"positive_id": guards["positive_id"], "pair_id": guards["pair_id"]}
|
||||
return await self._record_decision(item, guards, None, verified, manual)
|
||||
|
||||
async def _decide_review_item(self, request: web.Request) -> web.Response:
|
||||
data, error = await self._read_json_body(request, max_bytes=MAX_BODY_BYTES)
|
||||
if error is not None:
|
||||
@@ -584,25 +549,22 @@ class ReviewRoutesMixin:
|
||||
return None, _coded("file-changed")
|
||||
metadata = None
|
||||
if need_metadata:
|
||||
# A manual link has no ``choice``: it rests on the bytes alone.
|
||||
if captured is None or (
|
||||
"choice" in guards
|
||||
and not _candidate_still_offered(captured[1], guards["choice"])
|
||||
if captured is None or not _candidate_still_offered(
|
||||
captured[1], guards["choice"]
|
||||
):
|
||||
return None, _coded("review-conflict", "evidence-changed")
|
||||
metadata = captured[0]
|
||||
image_path = os.path.join(str(root), *relative.split("/"))
|
||||
return (capture, facts, image_path, metadata), None
|
||||
|
||||
def _decide_sync(self, item, guards, candidate, verified, manual=None):
|
||||
def _decide_sync(self, item, guards, candidate, verified):
|
||||
"""Worker thread: resolve the file's spellings, decide, build the view.
|
||||
|
||||
Returns ``(result, view)``; a refusal raises with ``exc.view`` set.
|
||||
"""
|
||||
_capture, facts, image_path, metadata = verified or (None, None, None, None)
|
||||
aliases, known = (), None
|
||||
selected = candidate is not None or manual is not None
|
||||
if selected:
|
||||
if candidate is not None:
|
||||
aliases, known = physical_spellings(self.db, image_path, item["locator"])
|
||||
try:
|
||||
result = self.db.decide_review_item(
|
||||
@@ -611,9 +573,8 @@ class ReviewRoutesMixin:
|
||||
expected_revision=guards["expected_revision"],
|
||||
# The bytes the re-hash proved, or the body's for Leave unresolved.
|
||||
sha256=facts.sha256 if facts is not None else guards["sha256"],
|
||||
outcome="selected" if selected else UNRESOLVED,
|
||||
outcome="selected" if candidate is not None else UNRESOLVED,
|
||||
candidate=candidate,
|
||||
manual=manual,
|
||||
image_path=image_path,
|
||||
metadata=metadata,
|
||||
expected_fingerprint=guards["dependency_fingerprint"],
|
||||
@@ -628,11 +589,11 @@ class ReviewRoutesMixin:
|
||||
roots, real_roots = self._review_roots()
|
||||
return result, self._review_view(result["item"], roots, real_roots)
|
||||
|
||||
async def _record_decision(self, item, guards, candidate, verified, manual=None):
|
||||
async def _record_decision(self, item, guards, candidate, verified):
|
||||
capture = verified[0] if verified else None
|
||||
try:
|
||||
result, view = await self._run_in_executor(
|
||||
self._decide_sync, item, guards, candidate, verified, manual
|
||||
self._decide_sync, item, guards, candidate, verified
|
||||
)
|
||||
except ReviewCommandError as exc:
|
||||
return self._decision_refused(exc)
|
||||
@@ -668,6 +629,36 @@ class ReviewRoutesMixin:
|
||||
return _coded(exc.code, message, **extra)
|
||||
|
||||
|
||||
def _decision_guards(data: dict):
|
||||
"""The validated decision fields, or ``(None, error)``."""
|
||||
if set(data) != _DECISION_FIELDS:
|
||||
return None, bad_request(
|
||||
"Request must contain exactly " + ", ".join(sorted(_DECISION_FIELDS))
|
||||
)
|
||||
epoch, revision = data["expected_epoch"], data["expected_revision"]
|
||||
if not isinstance(epoch, str) or _UUID.fullmatch(epoch) is None:
|
||||
return None, bad_request("expected_epoch must be a canonical UUID")
|
||||
if type(revision) is not int or revision < 0:
|
||||
return None, bad_request("expected_revision must be a non-negative integer")
|
||||
token = data["capture_token"]
|
||||
if token is not None and (
|
||||
not isinstance(token, str) or _TOKEN.fullmatch(token) is None
|
||||
):
|
||||
return None, bad_request("Invalid capture token")
|
||||
for name in ("sha256", "dependency_fingerprint"):
|
||||
value = data[name]
|
||||
if not isinstance(value, str) or _HEX64.fullmatch(value) is None:
|
||||
return None, bad_request(
|
||||
f"{name} must be 64 lowercase hexadecimal characters"
|
||||
)
|
||||
choice = data["choice"]
|
||||
if not isinstance(choice, str) or (
|
||||
choice != UNRESOLVED and _HEX64.fullmatch(choice) is None
|
||||
):
|
||||
return None, bad_request("choice must be a candidate hash or unresolved")
|
||||
return {**data}, None
|
||||
|
||||
|
||||
def _verify_and_read(store, capture, item, root, need_metadata: bool):
|
||||
"""Worker thread: ``(relative, facts, captured)``, or None off the root.
|
||||
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
"""Body validation of Needs review decisions and manual links (#249 E3, #536)."""
|
||||
|
||||
from .annotations import _HEX64, _TOKEN, _UUID
|
||||
from .prompts import bad_request
|
||||
|
||||
UNRESOLVED = "unresolved"
|
||||
_GUARD_FIELDS = frozenset(
|
||||
{
|
||||
"expected_epoch",
|
||||
"expected_revision",
|
||||
"capture_token",
|
||||
"sha256",
|
||||
"dependency_fingerprint",
|
||||
}
|
||||
)
|
||||
_DECISION_FIELDS = _GUARD_FIELDS | {"choice"}
|
||||
_LINK_FIELDS = _GUARD_FIELDS | {"positive_id", "pair_id"}
|
||||
|
||||
|
||||
def _guard_error(data: dict, fields: frozenset):
|
||||
"""The first problem with a decision's shared fields, as a response."""
|
||||
if set(data) != fields:
|
||||
return bad_request("Request must contain exactly " + ", ".join(sorted(fields)))
|
||||
epoch, revision = data["expected_epoch"], data["expected_revision"]
|
||||
if not isinstance(epoch, str) or _UUID.fullmatch(epoch) is None:
|
||||
return bad_request("expected_epoch must be a canonical UUID")
|
||||
if type(revision) is not int or revision < 0:
|
||||
return bad_request("expected_revision must be a non-negative integer")
|
||||
token = data["capture_token"]
|
||||
if token is not None and (
|
||||
not isinstance(token, str) or _TOKEN.fullmatch(token) is None
|
||||
):
|
||||
return bad_request("Invalid capture token")
|
||||
for name in ("sha256", "dependency_fingerprint"):
|
||||
value = data[name]
|
||||
if not isinstance(value, str) or _HEX64.fullmatch(value) is None:
|
||||
return bad_request(f"{name} must be 64 lowercase hexadecimal characters")
|
||||
return None
|
||||
|
||||
|
||||
def _decision_guards(data: dict):
|
||||
"""The validated decision fields, or ``(None, error)``."""
|
||||
error = _guard_error(data, _DECISION_FIELDS)
|
||||
if error is not None:
|
||||
return None, error
|
||||
choice = data["choice"]
|
||||
if not isinstance(choice, str) or (
|
||||
choice != UNRESOLVED and _HEX64.fullmatch(choice) is None
|
||||
):
|
||||
return None, bad_request("choice must be a candidate hash or unresolved")
|
||||
return {**data}, None
|
||||
|
||||
|
||||
def _link_guards(data: dict):
|
||||
"""The validated manual link fields, or ``(None, error)``."""
|
||||
error = _guard_error(data, _LINK_FIELDS)
|
||||
if error is not None:
|
||||
return None, error
|
||||
if data["capture_token"] is None:
|
||||
return None, bad_request("A link needs a capture of the file")
|
||||
positive, pair = data["positive_id"], data["pair_id"]
|
||||
if type(positive) is not int or positive < 1:
|
||||
return None, bad_request("positive_id must be a positive integer")
|
||||
if pair is not None and (type(pair) is not int or pair < 1):
|
||||
return None, bad_request("pair_id must be a positive integer or null")
|
||||
return {**data}, None
|
||||
@@ -54,25 +54,9 @@ _SETTINGS_FILE_KEYS = {
|
||||
"AUTOTAG_EXISTING": ("autotag", "defaults", "existing"),
|
||||
"AUTOTAG_GENERAL_THRESHOLD": ("autotag", "defaults", "general_threshold"),
|
||||
"AUTOTAG_CHARACTER_THRESHOLD": ("autotag", "defaults", "character_threshold"),
|
||||
"AUTOTAG_TEMPLATE": ("autotag", "defaults", "template"),
|
||||
"AUTOTAG_SUGGESTION_LIMIT": ("autotag", "defaults", "suggestion_limit"),
|
||||
"AUTOTAG_SKIPPED_TAGS": ("autotag", "defaults", "skipped_tags"),
|
||||
# #431: the saved instructions and each engine's default, always written
|
||||
# together (server-side pairing).
|
||||
"AUTOTAG_INSTRUCTIONS": ("autotag", "defaults", "instructions"),
|
||||
"AUTOTAG_DEFAULT_INSTRUCTIONS": ("autotag", "defaults", "default_instructions"),
|
||||
}
|
||||
# The old single template: written only as the GGUF default's mirror for a
|
||||
# downgrade (#431 B1.1), deleted when that default is Built-in.
|
||||
_TEMPLATE_MIRROR_KEY = ("autotag", "defaults", "template")
|
||||
TEMPLATE_REPLACED_ERROR = (
|
||||
"LLM tag template was replaced by Saved instructions. Reload the page."
|
||||
)
|
||||
# Whole-list settings (#431): a save without ``expected_revision`` would merge
|
||||
# a stale copy over a list changed elsewhere, so it is refused.
|
||||
_REVISION_REQUIRED = frozenset(
|
||||
{"autotag_skipped_tags", "autotag_instructions", "autotag_default_instructions"}
|
||||
)
|
||||
REVISION_REQUIRED_ERROR = "Reload the page and try again."
|
||||
# The thumbnail settings a running Convert outdated job fixes (#317).
|
||||
_THUMBNAIL_SETTINGS = ("THUMBNAIL_FORMAT", "THUMBNAIL_QUALITY", "THUMBNAIL_EDGE")
|
||||
# Attributes that live on GalleryConfig; the others on PromptManagerConfig.
|
||||
@@ -153,20 +137,13 @@ def _settings_mutation(updates, new_roots):
|
||||
"""The config commit for a settings save: only the submitted keys change.
|
||||
|
||||
Every other section and key (integrations and the CivitAI key, database,
|
||||
unknown sections) is kept as it is on disk. A save of the instructions
|
||||
also writes the old ``template`` key as the GGUF default's mirror.
|
||||
unknown sections) is kept as it is on disk.
|
||||
"""
|
||||
|
||||
def mutate(config):
|
||||
for name, value in updates.items():
|
||||
*sections, key = _SETTINGS_FILE_KEYS[name]
|
||||
_set_nested(config, sections, key, _jsonable(value))
|
||||
if "AUTOTAG_INSTRUCTIONS" in updates:
|
||||
*sections, key = _TEMPLATE_MIRROR_KEY
|
||||
mirror = _mirror_text(
|
||||
updates["AUTOTAG_INSTRUCTIONS"], updates["AUTOTAG_DEFAULT_INSTRUCTIONS"]
|
||||
)
|
||||
_set_nested(config, sections, key, _DELETE if mirror is None else mirror)
|
||||
_set_nested(config, sections, key, value)
|
||||
if new_roots is not None:
|
||||
gallery = _section(config, "gallery")
|
||||
monitoring = _section(gallery, "monitoring")
|
||||
@@ -178,37 +155,10 @@ def _settings_mutation(updates, new_roots):
|
||||
return mutate
|
||||
|
||||
|
||||
# ``_set_nested``'s value that removes the key instead of setting it.
|
||||
_DELETE = object()
|
||||
|
||||
|
||||
def _jsonable(value):
|
||||
"""A runtime value as config.json holds it (tuples become lists)."""
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_jsonable(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {key: _jsonable(item) for key, item in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
def _mirror_text(entries, defaults):
|
||||
"""The GGUF default's text, or None when it is Built-in (#431 B1.1)."""
|
||||
from ..tagging_defaults import BUILTIN_INSTRUCTIONS_ID
|
||||
|
||||
for item in entries:
|
||||
if item["id"] == defaults["gguf"] != BUILTIN_INSTRUCTIONS_ID:
|
||||
return item["text"]
|
||||
return None
|
||||
|
||||
|
||||
def _set_nested(config, sections, key, value):
|
||||
"""``config[s1][s2]...[key] = value``, each section a fresh dict;
|
||||
``_DELETE`` removes the key."""
|
||||
"""``config[s1][s2]...[key] = value``, each section a fresh dict."""
|
||||
if not sections:
|
||||
if value is _DELETE:
|
||||
config.pop(key, None)
|
||||
else:
|
||||
config[key] = value
|
||||
config[key] = value
|
||||
return
|
||||
merged = _section(config, sections[0])
|
||||
_set_nested(merged, sections[1:], key, value)
|
||||
@@ -226,7 +176,7 @@ def _validate_autotag_settings(data):
|
||||
MAX_SUGGESTION_LIMIT,
|
||||
MODEL_TYPES,
|
||||
validate_saved_threshold,
|
||||
validate_skipped_tags,
|
||||
validate_template,
|
||||
)
|
||||
|
||||
updates = {}
|
||||
@@ -255,7 +205,12 @@ def _validate_autotag_settings(data):
|
||||
raise ValueError(f"{error}.") from None
|
||||
updates[key.upper()] = value
|
||||
if "autotag_template" in data:
|
||||
raise ValueError(TEMPLATE_REPLACED_ERROR)
|
||||
try:
|
||||
updates["AUTOTAG_TEMPLATE"] = validate_template(
|
||||
data["autotag_template"], "LLM tag template"
|
||||
)
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{error}.") from None
|
||||
if "autotag_suggestion_limit" in data:
|
||||
value = data["autotag_suggestion_limit"]
|
||||
if type(value) is not int or not 1 <= value <= MAX_SUGGESTION_LIMIT:
|
||||
@@ -264,41 +219,9 @@ def _validate_autotag_settings(data):
|
||||
f"{MAX_SUGGESTION_LIMIT}."
|
||||
)
|
||||
updates["AUTOTAG_SUGGESTION_LIMIT"] = value
|
||||
if "autotag_skipped_tags" in data:
|
||||
updates["AUTOTAG_SKIPPED_TAGS"] = tuple(
|
||||
validate_skipped_tags(data["autotag_skipped_tags"])
|
||||
)
|
||||
updates.update(_validate_instruction_settings(data))
|
||||
return updates
|
||||
|
||||
|
||||
def _validate_instruction_settings(data):
|
||||
"""Server-side pairing (#431): a save that carries either instruction key
|
||||
gets the other from the running configuration (which holds the migrated
|
||||
list), the defaults are checked against the merged list, and both are
|
||||
written. Empty when the save carries neither."""
|
||||
from ..config import AutotagConfig
|
||||
from ..tagging_defaults import (
|
||||
validate_default_instructions,
|
||||
validate_instructions,
|
||||
)
|
||||
|
||||
if not {"autotag_instructions", "autotag_default_instructions"} & set(data):
|
||||
return {}
|
||||
if "autotag_instructions" in data:
|
||||
entries = validate_instructions(data["autotag_instructions"])
|
||||
else:
|
||||
entries = [dict(item) for item in AutotagConfig.INSTRUCTIONS]
|
||||
defaults = data.get(
|
||||
"autotag_default_instructions", dict(AutotagConfig.DEFAULT_INSTRUCTIONS)
|
||||
)
|
||||
defaults = validate_default_instructions(defaults, entries)
|
||||
return {
|
||||
"AUTOTAG_INSTRUCTIONS": tuple(entries),
|
||||
"AUTOTAG_DEFAULT_INSTRUCTIONS": defaults,
|
||||
}
|
||||
|
||||
|
||||
async def _read_json(request):
|
||||
"""Parse a JSON object body through the package's capped reader."""
|
||||
from . import _read_json_body
|
||||
@@ -326,7 +249,6 @@ class SettingsRoutesMixin:
|
||||
"""Get current settings."""
|
||||
try:
|
||||
from ..config import AutotagConfig, PromptManagerConfig, GalleryConfig
|
||||
from ..tagging_defaults import DEFAULT_TAG_TEMPLATE
|
||||
|
||||
root_paths = [
|
||||
self._public_path(d) for d in GalleryConfig.MONITORING_DIRECTORIES
|
||||
@@ -359,21 +281,8 @@ class SettingsRoutesMixin:
|
||||
"autotag_character_threshold": (
|
||||
AutotagConfig.CHARACTER_THRESHOLD
|
||||
),
|
||||
# Read-only for one release (#431): a page cached
|
||||
# from before the upgrade still renders.
|
||||
"autotag_template": AutotagConfig.default_instructions_text(
|
||||
"gguf"
|
||||
),
|
||||
"autotag_instructions": [
|
||||
dict(item) for item in AutotagConfig.INSTRUCTIONS
|
||||
],
|
||||
"autotag_default_instructions": dict(
|
||||
AutotagConfig.DEFAULT_INSTRUCTIONS
|
||||
),
|
||||
# Built-in's text, which System shows read-only.
|
||||
"autotag_builtin_instructions": DEFAULT_TAG_TEMPLATE,
|
||||
"autotag_template": AutotagConfig.TEMPLATE,
|
||||
"autotag_suggestion_limit": AutotagConfig.SUGGESTION_LIMIT,
|
||||
"autotag_skipped_tags": list(AutotagConfig.SKIPPED_TAGS),
|
||||
"cpu_count": PromptManagerConfig.max_worker_threads(),
|
||||
"gallery_root_paths": root_paths,
|
||||
"gallery_root_path": root_paths[0] if root_paths else "",
|
||||
@@ -474,10 +383,6 @@ class SettingsRoutesMixin:
|
||||
{"success": False, "error": "expected_revision must be a string"},
|
||||
status=400,
|
||||
)
|
||||
if expected_revision is None and _REVISION_REQUIRED & set(data):
|
||||
return web.json_response(
|
||||
{"success": False, "error": REVISION_REQUIRED_ERROR}, status=400
|
||||
)
|
||||
try:
|
||||
updates = self._validate_settings(data)
|
||||
except ValueError as ve:
|
||||
@@ -606,12 +511,6 @@ class SettingsRoutesMixin:
|
||||
if name in _AUTOTAG_SETTINGS:
|
||||
setattr(AutotagConfig, name[len(_AUTOTAG_PREFIX) :], value)
|
||||
continue
|
||||
if "AUTOTAG_INSTRUCTIONS" in updates:
|
||||
# What config.json now holds as the old key (the mirror).
|
||||
AutotagConfig._LEGACY_TEMPLATE = AutotagConfig.mirror_template()
|
||||
for name, value in updates.items():
|
||||
if name in _AUTOTAG_SETTINGS:
|
||||
continue
|
||||
owner = GalleryConfig if name in _GALLERY_SETTINGS else PromptManagerConfig
|
||||
setattr(owner, name, value)
|
||||
restart_fields = []
|
||||
|
||||
@@ -41,13 +41,6 @@ def _coded(status: int, code: str, message: str) -> web.Response:
|
||||
)
|
||||
|
||||
|
||||
def _missing_if_not_found(response: web.Response) -> web.Response:
|
||||
"""A file in a trash folder answers as a missing file does (#543)."""
|
||||
if response.status == 404:
|
||||
return _coded(404, "file-missing", "Image file not found")
|
||||
return response
|
||||
|
||||
|
||||
def _no_store(response: web.Response) -> web.Response:
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return response
|
||||
@@ -78,7 +71,7 @@ class SourceRescopeMixin:
|
||||
path = root / relative
|
||||
denied = _media_access_error(path, [root])
|
||||
if denied is not None:
|
||||
return _missing_if_not_found(denied)
|
||||
return denied
|
||||
try:
|
||||
mtime = path.stat().st_mtime if path.is_file() else None
|
||||
except OSError:
|
||||
@@ -96,7 +89,7 @@ class SourceRescopeMixin:
|
||||
return bad_request("Send exactly one target")
|
||||
target, error = media_target(data["target"])
|
||||
if error is not None:
|
||||
return _missing_if_not_found(error)
|
||||
return error
|
||||
old_index, scope, relative = target
|
||||
if old_index > MAX_ROOT_INDEX:
|
||||
return bad_request("Invalid gallery root")
|
||||
|
||||
@@ -8,8 +8,7 @@ previews behind those URLs are made here, in three ways:
|
||||
- **On first view.** ``GET /prompt_manager/images/serve/thumbnails/<rel>_thumb
|
||||
<ext>`` for a preview that does not exist yet is admitted on the event loop
|
||||
(``_PreviewGate``): at most ``ON_DEMAND_LIMIT`` previews are written at once,
|
||||
on this API's interactive image pool (``IMAGE_POOL_SIZE`` threads, shared
|
||||
with card facts and inspector reads; never the default executor), one flight runs
|
||||
on this API's own small pool (never the default executor), one flight runs
|
||||
per file, and the requests waiting are capped (``MAX_WAITING``); past the
|
||||
cap a request gets its fallback at once. Inside the admission the preview is
|
||||
mapped back to its media file under the same root (the request text is
|
||||
@@ -25,8 +24,7 @@ previews behind those URLs are made here, in three ways:
|
||||
- **On capture.** The image monitor hands every new output it settles to
|
||||
``queue_capture_thumbnail``, which queues its preview for one background
|
||||
worker thread (``AutoThumbnailer``). Each wake takes at most Worker threads
|
||||
(``WORKER_THREADS``) files and writes them on the long-lived job pool
|
||||
(``utils.parallel``, shared with scans and thumbnail jobs); enqueuing
|
||||
(``WORKER_THREADS``) files and writes them on that many threads; enqueuing
|
||||
is O(1), so the monitor and ComfyUI's queue never wait. A file already
|
||||
queued is not queued again, the queue is bounded (``MAX_QUEUED``) and a
|
||||
preview left out is made on its first view instead.
|
||||
@@ -51,7 +49,7 @@ write starts (after its slot) and again just before its rename.
|
||||
|
||||
Shutdown: ``close_auto_thumbnails`` stops the worker at a file boundary,
|
||||
kills a running ffmpeg, waits a bounded time (and logs when the wait ran
|
||||
out) and shuts the image pool down; after the app's cleanup
|
||||
out) and shuts the view pool down; after the app's cleanup
|
||||
(``final=True``) nothing starts a worker again. Importing this module again
|
||||
(a package reload) stops the workers the previous import left.
|
||||
"""
|
||||
@@ -62,14 +60,13 @@ import functools
|
||||
import io
|
||||
import os
|
||||
import threading
|
||||
from concurrent.futures import CancelledError, ThreadPoolExecutor
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import thumbnail_settings
|
||||
from .images import (
|
||||
in_trash_below_roots,
|
||||
MEDIA_EXTENSIONS,
|
||||
THUMBNAIL_CLEAR_TITLE,
|
||||
VIDEO_EXTENSIONS,
|
||||
@@ -106,11 +103,8 @@ PLACEHOLDER_SIZE = (thumbnail_settings.DEFAULT_EDGE, thumbnail_settings.DEFAULT_
|
||||
PLACEHOLDER_QUALITY = 85
|
||||
# Source extensions in the order a shared preview name looks for them (R2).
|
||||
_SOURCE_ORDER = thumbnail_settings.IMAGE_SOURCE_ORDER + tuple(sorted(VIDEO_EXTENSIONS))
|
||||
# Threads of the interactive image pool (#523): view previews, card facts and
|
||||
# inspector reads. Fixed, so image buffers stay in a few threads' arenas.
|
||||
IMAGE_POOL_SIZE = 3
|
||||
# Previews views may write at once (the whole image pool).
|
||||
ON_DEMAND_LIMIT = IMAGE_POOL_SIZE
|
||||
# Previews views may write at once (the size of the view pool).
|
||||
ON_DEMAND_LIMIT = 3
|
||||
# View requests that may wait on the gate; one more gets its fallback at once.
|
||||
MAX_WAITING = 64
|
||||
# How long a pool thread waits for the capture worker's flight of the same file.
|
||||
@@ -129,7 +123,7 @@ AUTO_REFUSED_KINDS = frozenset(
|
||||
{RESTORE_JOB_KIND, DUPLICATE_CLEANUP_JOB_KIND, SOURCE_CHANGE_JOB_KIND}
|
||||
)
|
||||
WORKER_NAME = "PromptManagerThumbnailWorker"
|
||||
POOL_PREFIX = "pm-image"
|
||||
POOL_PREFIX = "pm-preview"
|
||||
# Identifies this import's workers: a reload stops the ones it did not start.
|
||||
_MODULE_TOKEN = object()
|
||||
|
||||
@@ -230,8 +224,6 @@ def _quick_source(root, parts):
|
||||
for name in (base + extension, base + extension.upper()):
|
||||
path = folder / name
|
||||
if _regular_file(path) and _is_within(path, root):
|
||||
if in_trash_below_roots(path, [root]):
|
||||
return None # also the fallback past the gate's cap (#543)
|
||||
return path, extension in VIDEO_EXTENSIONS
|
||||
return None
|
||||
|
||||
@@ -256,8 +248,6 @@ def _thumbnail_source(root, parts):
|
||||
found = _scan_folder(root.joinpath(*folder_parts), base, set(wanted))
|
||||
if found is None or not _is_within(found, root):
|
||||
return None
|
||||
if in_trash_below_roots(found, [root]):
|
||||
return None
|
||||
rel = Path(*folder_parts, found.name)
|
||||
dst = root / media_thumbnail_rel(rel)
|
||||
if dst != _current_preview(root, parts):
|
||||
@@ -299,7 +289,7 @@ class AutoThumbnailer:
|
||||
``generate(src, dst, is_video, *, stop, guard, track)`` writes one
|
||||
preview and returns ``_generate_one``'s result dict; ``may_write()``
|
||||
says whether automatic previews may be written now. Views call
|
||||
``ensure`` on the image pool, the monitor calls ``enqueue`` from its
|
||||
``ensure`` on the view pool, the monitor calls ``enqueue`` from its
|
||||
thread.
|
||||
"""
|
||||
|
||||
@@ -496,15 +486,12 @@ class AutoThumbnailer:
|
||||
self._worker_running = False
|
||||
self._changed.notify_all()
|
||||
return
|
||||
workers = max(1, int(self._workers() or 1))
|
||||
take = min(len(self._queue), workers)
|
||||
take = min(len(self._queue), max(1, int(self._workers() or 1)))
|
||||
batch = [self._queue.popleft() for _ in range(take)]
|
||||
self._batch = len(batch)
|
||||
self._changed.notify_all()
|
||||
try:
|
||||
# The job pool's size is the setting; a smaller batch uses
|
||||
# fewer of its threads (#523: one long-lived pool).
|
||||
map_parallel(self._write_queued, batch, workers)
|
||||
map_parallel(self._write_queued, batch, len(batch))
|
||||
finally:
|
||||
with self._changed:
|
||||
self._batch = 0
|
||||
@@ -541,7 +528,6 @@ class AutoThumbnailer:
|
||||
try:
|
||||
process.kill()
|
||||
except OSError:
|
||||
# the process has already exited
|
||||
pass
|
||||
if worker is None:
|
||||
return True
|
||||
@@ -575,7 +561,7 @@ class _PreviewGate:
|
||||
"""Admission of view requests on the event loop (#249 I2).
|
||||
|
||||
Waiters are coroutines: at most ``limit`` previews are written at once
|
||||
on the image pool, one flight runs per key, and past ``MAX_WAITING``
|
||||
on the view pool, one flight runs per key, and past ``MAX_WAITING``
|
||||
waiting requests ``run`` answers ``OVER`` at once.
|
||||
"""
|
||||
|
||||
@@ -630,17 +616,13 @@ class _PreviewGate:
|
||||
flight.set_result(result)
|
||||
|
||||
|
||||
def _on_image_thread():
|
||||
return threading.current_thread().name.startswith(POOL_PREFIX + "_")
|
||||
|
||||
|
||||
class AutoThumbnailMixin:
|
||||
"""Automatic previews on the API: views, the monitor's queue, scans."""
|
||||
|
||||
_auto_thumbnailer = None
|
||||
_auto_thumbnails_shut = False
|
||||
_preview_gate_obj = None
|
||||
_image_pool_obj = None
|
||||
_preview_pool_obj = None
|
||||
_auto_thumbnailer_lock = threading.Lock()
|
||||
|
||||
def _auto_thumbnails(self):
|
||||
@@ -663,7 +645,7 @@ class AutoThumbnailMixin:
|
||||
self._auto_thumbnails_shut = True
|
||||
|
||||
def close_auto_thumbnails(self, final=False, timeout=None):
|
||||
"""Stop the capture worker and the image pool, and forget them.
|
||||
"""Stop the capture worker and the view pool, and forget them.
|
||||
|
||||
Bounded by ``timeout`` (see AutoThumbnailer.close). ``final`` (the
|
||||
package teardown) also keeps a late monitor event from starting a
|
||||
@@ -674,7 +656,7 @@ class AutoThumbnailMixin:
|
||||
if final:
|
||||
self._auto_thumbnails_shut = True
|
||||
thumbnailer, self._auto_thumbnailer = self._auto_thumbnailer, None
|
||||
pool, self._image_pool_obj = self._image_pool_obj, None
|
||||
pool, self._preview_pool_obj = self._preview_pool_obj, None
|
||||
self._preview_gate_obj = None
|
||||
closed = True
|
||||
if thumbnailer is not None:
|
||||
@@ -718,67 +700,16 @@ class AutoThumbnailMixin:
|
||||
gate = self._preview_gate_obj = _PreviewGate(ON_DEMAND_LIMIT, self.logger)
|
||||
return gate
|
||||
|
||||
def _image_pool(self):
|
||||
"""The interactive image pool (#523): IMAGE_POOL_SIZE threads for view
|
||||
previews (behind their gate), card facts and inspector reads, so
|
||||
image buffers never spread over the default executor's threads.
|
||||
|
||||
None after the package teardown (``close_auto_thumbnails(final=True)``):
|
||||
a late request then runs on the default executor (``run_in_executor``
|
||||
with None) and starts no thread the teardown no longer stops.
|
||||
"""
|
||||
def _preview_pool(self):
|
||||
with AutoThumbnailMixin._auto_thumbnailer_lock:
|
||||
if self._image_pool_obj is None and not self._auto_thumbnails_shut:
|
||||
self._image_pool_obj = ThreadPoolExecutor(
|
||||
IMAGE_POOL_SIZE, thread_name_prefix=POOL_PREFIX
|
||||
if self._preview_pool_obj is None:
|
||||
self._preview_pool_obj = ThreadPoolExecutor(
|
||||
ON_DEMAND_LIMIT, thread_name_prefix=POOL_PREFIX
|
||||
)
|
||||
return self._image_pool_obj
|
||||
return self._preview_pool_obj
|
||||
|
||||
async def _run_image(self, func, *args, **kwargs):
|
||||
"""``_run_in_executor`` for interactive image work: the image pool.
|
||||
|
||||
After the teardown, or with the pool shut down in between (a test
|
||||
reset), the call runs on the default executor as before.
|
||||
"""
|
||||
call = functools.partial(func, *args, **kwargs)
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
future = loop.run_in_executor(self._image_pool(), call)
|
||||
except RuntimeError: # submit refused: the pool was shut down
|
||||
future = loop.run_in_executor(None, call)
|
||||
return await future
|
||||
|
||||
def _map_image(self, func, items):
|
||||
"""``func`` over *items* on the image pool, in order (blocking).
|
||||
|
||||
Called from an image pool thread, after the teardown, or with the
|
||||
pool shut down in between (its queued items are cancelled), the
|
||||
items it did not run run on the calling thread.
|
||||
"""
|
||||
items = list(items)
|
||||
pool = None if not items or _on_image_thread() else self._image_pool()
|
||||
if pool is None:
|
||||
return [func(item) for item in items]
|
||||
futures = []
|
||||
try:
|
||||
for item in items:
|
||||
futures.append(pool.submit(func, item))
|
||||
except RuntimeError:
|
||||
pass # shut down in between: the rest run here
|
||||
results = []
|
||||
for index, item in enumerate(items):
|
||||
future = futures[index] if index < len(futures) else None
|
||||
if future is None or future.cancelled():
|
||||
results.append(func(item))
|
||||
continue
|
||||
try:
|
||||
results.append(future.result())
|
||||
except CancelledError: # cancelled while this caller waited
|
||||
results.append(func(item))
|
||||
return results
|
||||
|
||||
def _image_pool_threads(self):
|
||||
pool = self._image_pool_obj
|
||||
def _preview_pool_threads(self):
|
||||
pool = self._preview_pool_obj
|
||||
return [] if pool is None else list(getattr(pool, "_threads", ()))
|
||||
|
||||
def _video_placeholder_bytes(self):
|
||||
@@ -793,7 +724,7 @@ class AutoThumbnailMixin:
|
||||
return await self._preview_gate().run(
|
||||
flight_key(Path(dst).parent, (Path(dst).name,)),
|
||||
functools.partial(self._ensure_view, src, dst, is_video),
|
||||
self._image_pool(),
|
||||
self._preview_pool(),
|
||||
)
|
||||
|
||||
def _ensure_view(self, src, dst, is_video):
|
||||
@@ -837,13 +768,13 @@ class AutoThumbnailMixin:
|
||||
The flight answers only the outcome (a history flight may be the one
|
||||
joined); the file it previews is derived here: the preview path is
|
||||
the request's own, and a fallback looks its source up again on the
|
||||
image pool (bounded by it, never the default executor).
|
||||
view pool (bounded by it, never the default executor).
|
||||
"""
|
||||
if _has_traversal(filepath) or not _is_preview_request(filepath):
|
||||
return None
|
||||
parts = tuple(filepath.split("/"))
|
||||
gate = self._preview_gate()
|
||||
pool = self._image_pool()
|
||||
pool = self._preview_pool()
|
||||
for root in output_dirs:
|
||||
root = Path(root)
|
||||
outcome = await gate.run(
|
||||
@@ -904,7 +835,7 @@ class AutoThumbnailMixin:
|
||||
if not folded.startswith(base) or not _is_within(path, root):
|
||||
continue
|
||||
rel = Path(os.path.relpath(path, root))
|
||||
if "thumbnails" in rel.parts[:-1] or in_trash_below_roots(path, [root]):
|
||||
if "thumbnails" in rel.parts[:-1]:
|
||||
return None
|
||||
dst = root / media_thumbnail_rel(rel)
|
||||
if not _is_within(root / "thumbnails", root) or not _is_within(
|
||||
|
||||
@@ -153,7 +153,6 @@ class ThumbnailConvertMixin:
|
||||
job.log_line("Conversion failed · internal error")
|
||||
state = "failed"
|
||||
job.finish(state)
|
||||
await self._release_job_memory()
|
||||
|
||||
async def _convert_job_body(self, job, options):
|
||||
output_dirs = await self._run_in_executor(self._get_all_output_dirs)
|
||||
|
||||
@@ -158,7 +158,6 @@ def _unlink_quietly(path):
|
||||
try:
|
||||
Path(path).unlink()
|
||||
except FileNotFoundError:
|
||||
# already gone, which is what was wanted
|
||||
pass
|
||||
except OSError as exc:
|
||||
logger.debug(f"Could not remove temp file {Path(path).name}: {exc}")
|
||||
@@ -222,7 +221,6 @@ def shutdown_encoders():
|
||||
try:
|
||||
process.kill()
|
||||
except OSError:
|
||||
# the process has already exited
|
||||
pass
|
||||
|
||||
|
||||
@@ -361,12 +359,10 @@ def thumbnail_frame(img, edge):
|
||||
try:
|
||||
img.seek(0)
|
||||
except (EOFError, AttributeError, ValueError):
|
||||
# not seekable or a single frame: use the current one
|
||||
pass
|
||||
try:
|
||||
img.draft("RGB", (edge, edge)) # JPEG: decode at a reduced scale
|
||||
except (AttributeError, ValueError, OSError):
|
||||
# draft is only a speedup; the full decode below still works
|
||||
pass
|
||||
frame = img.convert("RGB") if img.mode != "RGB" else img.copy()
|
||||
frame.thumbnail((edge, edge), Image.Resampling.LANCZOS)
|
||||
|
||||
@@ -226,7 +226,6 @@ class ThumbnailJobsMixin:
|
||||
)
|
||||
state = "failed"
|
||||
job.finish(state)
|
||||
await self._release_job_memory()
|
||||
|
||||
async def _thumbnail_job_body(self, job, options):
|
||||
"""Plan, then generate in worker batches; returns the final state.
|
||||
|
||||
@@ -275,4 +275,3 @@ class WorkflowStorageRoutesMixin:
|
||||
job.record_counted("changed", 1)
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=line)
|
||||
return None
|
||||
|
||||
@@ -30,9 +30,9 @@ except ImportError:
|
||||
# The tag template and thresholds live with the saved tagging defaults
|
||||
# (#328), so config.py never imports PIL.
|
||||
try:
|
||||
from .tagging_defaults import DEFAULT_TAG_TEMPLATE, read_wd14_label_rows
|
||||
from .tagging_defaults import DEFAULT_TAG_TEMPLATE
|
||||
except ImportError:
|
||||
from py.tagging_defaults import DEFAULT_TAG_TEMPLATE, read_wd14_label_rows
|
||||
from py.tagging_defaults import DEFAULT_TAG_TEMPLATE
|
||||
|
||||
try:
|
||||
from ..utils import exit_signal
|
||||
@@ -559,6 +559,7 @@ class AutoTagService:
|
||||
# A missing onnxruntime is refused by load_model before this runs
|
||||
# (#424); a ModuleNotFoundError here becomes the same refusal.
|
||||
import onnxruntime as ort
|
||||
import csv
|
||||
|
||||
config = MODELS[model_type]
|
||||
model_dir = self.models_dir / config["subdir"]
|
||||
@@ -575,8 +576,12 @@ class AutoTagService:
|
||||
|
||||
session = ort.InferenceSession(str(model_path), providers=providers)
|
||||
|
||||
# Read tag list from CSV (the reader the Skipped tags preview shares)
|
||||
tags_list = read_wd14_label_rows(tags_path)
|
||||
# Read tag list from CSV
|
||||
tags_list = []
|
||||
with open(tags_path, "r", encoding="utf-8") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
tags_list.append(row)
|
||||
|
||||
self.logger.info(
|
||||
f"WD14 model loaded ({len(tags_list)} tags, "
|
||||
@@ -987,7 +992,6 @@ class AutoTagService:
|
||||
prompt: Optional[str] = None,
|
||||
general_threshold: Optional[float] = None,
|
||||
character_threshold: Optional[float] = None,
|
||||
skip: Any = None,
|
||||
) -> List[Tuple[str, Optional[float]]]:
|
||||
"""The top ``limit`` suggestions as ``(name, confidence)`` pairs.
|
||||
|
||||
@@ -996,9 +1000,7 @@ class AutoTagService:
|
||||
``None``. Names are distinct case-insensitively (first one wins).
|
||||
A name Save would reject (over 50 characters, a control character
|
||||
such as a newline, a comma) is skipped before the dedupe and the
|
||||
``limit`` count. A name ``skip`` matches (a ``SkipMatcher``, the
|
||||
Skipped tags compiled for this request or job, #431) is dropped next,
|
||||
so it uses up no slot. ``prompt`` is used for this call only;
|
||||
``limit`` count. ``prompt`` is used for this call only;
|
||||
``custom_prompt`` is never changed.
|
||||
"""
|
||||
if type(limit) is not int or not 1 <= limit <= MAX_SUGGESTION_LIMIT:
|
||||
@@ -1015,8 +1017,6 @@ class AutoTagService:
|
||||
name = validate_vocabulary_tag_name(raw)
|
||||
except ValueError:
|
||||
continue
|
||||
if skip is not None and skip.matches(name):
|
||||
continue
|
||||
key = name.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
|
||||
@@ -30,7 +30,6 @@ routes = server_instance.routes
|
||||
# Extension info
|
||||
extension_uri = None # Will be set in __init__.py
|
||||
|
||||
import hashlib # noqa: E402
|
||||
import os # noqa: E402
|
||||
from typing import Any, Dict, List, Optional, Tuple # noqa: E402
|
||||
|
||||
@@ -588,15 +587,8 @@ try:
|
||||
MODEL_TYPES,
|
||||
WD14_MODEL_TYPES,
|
||||
validate_saved_threshold,
|
||||
validate_skipped_tags,
|
||||
validate_template,
|
||||
validate_threshold,
|
||||
BUILTIN_INSTRUCTIONS_ID,
|
||||
INSTRUCTION_ENGINES,
|
||||
MAX_SAVED_INSTRUCTIONS,
|
||||
builtin_instructions,
|
||||
validate_default_instructions,
|
||||
validate_instructions,
|
||||
)
|
||||
except ImportError:
|
||||
from py.tagging_defaults import ( # noqa: F401 - re-exported
|
||||
@@ -608,15 +600,8 @@ except ImportError:
|
||||
MODEL_TYPES,
|
||||
WD14_MODEL_TYPES,
|
||||
validate_saved_threshold,
|
||||
validate_skipped_tags,
|
||||
validate_template,
|
||||
validate_threshold,
|
||||
BUILTIN_INSTRUCTIONS_ID,
|
||||
INSTRUCTION_ENGINES,
|
||||
MAX_SAVED_INSTRUCTIONS,
|
||||
builtin_instructions,
|
||||
validate_default_instructions,
|
||||
validate_instructions,
|
||||
)
|
||||
|
||||
|
||||
@@ -632,31 +617,6 @@ def _valid_existing(value: Any) -> str:
|
||||
return value
|
||||
|
||||
|
||||
def _valid_skipped_tags(value: Any) -> Tuple[str, ...]:
|
||||
return tuple(validate_skipped_tags(value))
|
||||
|
||||
|
||||
def _reason(error: ValueError) -> str:
|
||||
"""A settings sentence as a load warning's reason ("entry 2 is empty")."""
|
||||
text = str(error).rstrip(".")
|
||||
for prefix in ("Skipped tags", "Saved instructions"):
|
||||
if text.startswith(prefix):
|
||||
return text.removeprefix(prefix).lstrip(": ")
|
||||
return text
|
||||
|
||||
|
||||
def _text_id(text: str) -> str:
|
||||
"""A migrated entry's id: deterministic, so every load gives the same."""
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:12]
|
||||
|
||||
|
||||
def _fallback_warning(key: str, error: ValueError, default: Any) -> str:
|
||||
"""The one load warning of a saved tagging default that fails its check."""
|
||||
if key == "skipped_tags":
|
||||
return f"autotag.defaults.skipped_tags {_reason(error)}; using no skipped tags"
|
||||
return f"autotag.defaults.{key} {error}; using {default!r}"
|
||||
|
||||
|
||||
def _valid_suggestion_limit(value: Any) -> int:
|
||||
if type(value) is not int or not 1 <= value <= MAX_SUGGESTION_LIMIT:
|
||||
raise ValueError(f"must be an integer from 1 to {MAX_SUGGESTION_LIMIT}")
|
||||
@@ -677,18 +637,8 @@ class AutotagConfig:
|
||||
EXISTING = "skip"
|
||||
GENERAL_THRESHOLD = 0.35
|
||||
CHARACTER_THRESHOLD = 0.85
|
||||
TEMPLATE = DEFAULT_TAG_TEMPLATE
|
||||
SUGGESTION_LIMIT = DEFAULT_SUGGESTION_LIMIT
|
||||
SKIPPED_TAGS: Tuple[str, ...] = ()
|
||||
# #431: the saved instructions (Built-in is never stored) and the default
|
||||
# of each language-model engine, by id.
|
||||
INSTRUCTIONS: Tuple[Dict[str, str], ...] = ()
|
||||
DEFAULT_INSTRUCTIONS: Dict[str, str] = {
|
||||
"gguf": BUILTIN_INSTRUCTIONS_ID,
|
||||
"hf": BUILTIN_INSTRUCTIONS_ID,
|
||||
}
|
||||
# The old single template as read at load (None when absent or invalid);
|
||||
# only the migration reads it. Saves keep it as the GGUF default's mirror.
|
||||
_LEGACY_TEMPLATE: Optional[str] = None
|
||||
|
||||
# config.json key -> (attribute, validator, default)
|
||||
_KEYS = {
|
||||
@@ -704,13 +654,12 @@ class AutotagConfig:
|
||||
lambda v: validate_saved_threshold("value", v),
|
||||
0.85,
|
||||
),
|
||||
"template": ("TEMPLATE", validate_template, DEFAULT_TAG_TEMPLATE),
|
||||
"suggestion_limit": (
|
||||
"SUGGESTION_LIMIT",
|
||||
_valid_suggestion_limit,
|
||||
DEFAULT_SUGGESTION_LIMIT,
|
||||
),
|
||||
# #431: words whose suggested tags are never offered.
|
||||
"skipped_tags": ("SKIPPED_TAGS", _valid_skipped_tags, ()),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -723,142 +672,17 @@ class AutotagConfig:
|
||||
type(value) is int and 1 <= value <= MAX_SUGGESTION_LIMIT
|
||||
) # bool, float and str are rejected
|
||||
|
||||
@classmethod
|
||||
def all_instructions(cls) -> List[Dict[str, str]]:
|
||||
"""Built-in first, then the saved entries (copies)."""
|
||||
return [builtin_instructions()] + [dict(item) for item in cls.INSTRUCTIONS]
|
||||
|
||||
@classmethod
|
||||
def default_instructions_text(cls, engine: str) -> str:
|
||||
"""The text of ``engine``'s (``gguf`` or ``hf``) default instructions."""
|
||||
if engine not in INSTRUCTION_ENGINES:
|
||||
raise ValueError(f"{engine} takes no instructions")
|
||||
wanted = cls.DEFAULT_INSTRUCTIONS.get(engine, BUILTIN_INSTRUCTIONS_ID)
|
||||
for item in cls.INSTRUCTIONS:
|
||||
if item["id"] == wanted:
|
||||
return item["text"]
|
||||
return DEFAULT_TAG_TEMPLATE
|
||||
|
||||
@classmethod
|
||||
def mirror_template(cls) -> Optional[str]:
|
||||
"""What an older build reads as its one template (#431 B1.1): the
|
||||
GGUF default's text, or None (no key) when that default is Built-in."""
|
||||
if cls.DEFAULT_INSTRUCTIONS.get("gguf") == BUILTIN_INSTRUCTIONS_ID:
|
||||
return None
|
||||
return cls.default_instructions_text("gguf")
|
||||
|
||||
@classmethod
|
||||
def get_config(cls) -> Dict[str, Any]:
|
||||
"""The saved defaults; ``template`` only as the GGUF default's mirror."""
|
||||
"""The saved defaults; the template only when it is not the built-in
|
||||
one, so a later built-in template still reaches who never set one."""
|
||||
defaults = {
|
||||
key: getattr(cls, attr) for key, (attr, _v, _d) in cls._KEYS.items()
|
||||
}
|
||||
defaults["skipped_tags"] = list(defaults["skipped_tags"])
|
||||
defaults["instructions"] = [dict(item) for item in cls.INSTRUCTIONS]
|
||||
defaults["default_instructions"] = dict(cls.DEFAULT_INSTRUCTIONS)
|
||||
mirror = cls.mirror_template()
|
||||
if mirror is not None:
|
||||
defaults["template"] = mirror
|
||||
if defaults["template"] == DEFAULT_TAG_TEMPLATE:
|
||||
del defaults["template"]
|
||||
return {"defaults": defaults}
|
||||
|
||||
@classmethod
|
||||
def _load_instructions(cls, defaults: Dict[str, Any]) -> None:
|
||||
"""Each instruction key alone, then the cross-key default check, then
|
||||
the in-memory migration of the old template (#431). Nothing is
|
||||
written here, so the revision never changes at load."""
|
||||
cls._LEGACY_TEMPLATE = None
|
||||
if "template" in defaults:
|
||||
try:
|
||||
cls._LEGACY_TEMPLATE = validate_template(defaults["template"])
|
||||
except ValueError as error:
|
||||
config_logger.warning(
|
||||
f"autotag.defaults.template {error}; using the built-in one"
|
||||
)
|
||||
entries: List[Dict[str, str]] = []
|
||||
damaged = False
|
||||
if "instructions" in defaults:
|
||||
try:
|
||||
entries = validate_instructions(defaults["instructions"])
|
||||
except ValueError as error:
|
||||
damaged = True
|
||||
config_logger.warning(
|
||||
f"autotag.defaults.instructions {_reason(error)}; "
|
||||
"using Built-in only"
|
||||
)
|
||||
chosen = {engine: BUILTIN_INSTRUCTIONS_ID for engine in INSTRUCTION_ENGINES}
|
||||
stored = defaults.get("default_instructions")
|
||||
if damaged:
|
||||
# Both defaults fall back with the list (its one warning says so);
|
||||
# the old template is migrated as on a first upgrade.
|
||||
pass
|
||||
elif "default_instructions" in defaults and (
|
||||
not isinstance(stored, dict) or set(stored) != set(INSTRUCTION_ENGINES)
|
||||
):
|
||||
config_logger.warning(
|
||||
"autotag.defaults.default_instructions must name gguf and hf; "
|
||||
"using Built-in"
|
||||
)
|
||||
elif isinstance(stored, dict):
|
||||
ids = {item["id"] for item in entries}
|
||||
for engine in INSTRUCTION_ENGINES:
|
||||
value = stored[engine]
|
||||
if isinstance(value, str) and (
|
||||
value == BUILTIN_INSTRUCTIONS_ID or value in ids
|
||||
):
|
||||
chosen[engine] = value
|
||||
else:
|
||||
config_logger.warning(
|
||||
f"autotag.defaults.default_instructions.{engine} names no "
|
||||
"saved instructions; using Built-in"
|
||||
)
|
||||
cls._migrate_template(
|
||||
entries,
|
||||
chosen,
|
||||
first=damaged or "instructions" not in defaults,
|
||||
keep_defaults=not damaged and "default_instructions" in defaults,
|
||||
)
|
||||
cls.INSTRUCTIONS = tuple(entries)
|
||||
cls.DEFAULT_INSTRUCTIONS = chosen
|
||||
|
||||
@classmethod
|
||||
def _migrate_template(cls, entries, chosen, *, first, keep_defaults) -> None:
|
||||
"""Bring a template saved by an older build in as a named entry."""
|
||||
legacy = cls._LEGACY_TEMPLATE
|
||||
if legacy is None or legacy == DEFAULT_TAG_TEMPLATE:
|
||||
return
|
||||
if first:
|
||||
# The first upgrade: it becomes "Custom template", the default for
|
||||
# both engines unless defaults are stored already (20A).
|
||||
entries.append(
|
||||
{"id": _text_id(legacy), "name": "Custom template", "text": legacy}
|
||||
)
|
||||
if not keep_defaults:
|
||||
for engine in INSTRUCTION_ENGINES:
|
||||
chosen[engine] = entries[-1]["id"]
|
||||
return
|
||||
if any(item["text"] == legacy for item in entries):
|
||||
return # the mirror of a saved default
|
||||
# Edited on an older build after an upgrade (B1.2): import it.
|
||||
if len(entries) >= MAX_SAVED_INSTRUCTIONS:
|
||||
config_logger.warning(
|
||||
"The tagging template edited on an older version was not imported: "
|
||||
f"the saved instructions are full ({MAX_SAVED_INSTRUCTIONS})"
|
||||
)
|
||||
return
|
||||
names = {item["name"].casefold() for item in entries}
|
||||
number = 2
|
||||
while f"custom template ({number})" in names:
|
||||
number += 1
|
||||
name = f"Custom template ({number})"
|
||||
ids = {item["id"] for item in entries}
|
||||
new_id, n = _text_id(legacy), 2
|
||||
while new_id in ids:
|
||||
new_id, n = _text_id(f"{legacy}\n{n}"), n + 1
|
||||
entries.append({"id": new_id, "name": name, "text": legacy})
|
||||
config_logger.info(
|
||||
"Imported the tagging template edited on an older version as " f"'{name}'"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def update_config(cls, new_config: Any) -> None:
|
||||
if not isinstance(new_config, dict):
|
||||
@@ -874,9 +698,12 @@ class AutotagConfig:
|
||||
try:
|
||||
setattr(cls, attr, validate(defaults[key]))
|
||||
except ValueError as error:
|
||||
config_logger.warning(_fallback_warning(key, error, default))
|
||||
config_logger.warning(
|
||||
f"autotag.defaults.{key} {error}; using {default!r}"
|
||||
if key != "template"
|
||||
else f"autotag.defaults.template {error}; using the built-in one"
|
||||
)
|
||||
setattr(cls, attr, default)
|
||||
cls._load_instructions(defaults)
|
||||
|
||||
|
||||
class PromptManagerConfig:
|
||||
|
||||
@@ -109,7 +109,6 @@ def _write_atomically(path: str, payload: bytes) -> None:
|
||||
try:
|
||||
os.unlink(temp_path)
|
||||
except OSError:
|
||||
# nothing to clean up; the original error is raised below
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
@@ -642,7 +642,6 @@ def layout_summary(layout: LoraManagerLayout, home: Path) -> Dict:
|
||||
relative = Path(settings_file).relative_to(home)
|
||||
settings_file = os.path.join("~", str(relative))
|
||||
except ValueError:
|
||||
# outside the home folder: show the full path
|
||||
pass
|
||||
return {
|
||||
"source": layout.source,
|
||||
|
||||
@@ -585,7 +585,6 @@ def _read_capped(resp, cap: int) -> Optional[bytes]:
|
||||
if int(declared) > cap:
|
||||
return None
|
||||
except ValueError:
|
||||
# a bad header says nothing; the read below enforces the cap
|
||||
pass
|
||||
|
||||
chunks = []
|
||||
@@ -708,7 +707,6 @@ def _save_atomically(image, local_path: Path) -> None:
|
||||
try:
|
||||
os.remove(temp)
|
||||
except OSError:
|
||||
# nothing to clean up; the original error is raised below
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
@@ -4,13 +4,8 @@ Server-free on purpose: ``py.api`` must import without ComfyUI's ``server``
|
||||
module, which ``py/config.py`` needs; config re-exports all of this.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import unicodedata
|
||||
from typing import Any, Dict, Iterable, List, Set, Tuple
|
||||
from typing import Any
|
||||
|
||||
# The tagging engines (``py/autotag.py`` MODELS) and the options they take.
|
||||
WD14_MODEL_TYPES = frozenset({"wd14-swinv2", "wd14-vit"})
|
||||
@@ -93,230 +88,3 @@ def validate_template(value: Any, name: str = "instructions") -> str:
|
||||
if len(encoded) > MAX_INSTRUCTION_BYTES:
|
||||
raise ValueError(f"{name} {'are' if name == 'instructions' else 'is'} too long")
|
||||
return value
|
||||
|
||||
|
||||
# Skipped tags (#431): words whose suggested tags are never offered.
|
||||
MAX_SKIPPED_TAGS = 200
|
||||
MAX_SKIPPED_TAG_CHARS = 50
|
||||
_WORD_SEPARATORS = re.compile(r"[ _-]+")
|
||||
_FORBIDDEN_IN_ENTRY = re.compile(r"[,\x00-\x1f\x7f-\x9f\u2028\u2029]")
|
||||
|
||||
|
||||
def tag_words(text: str) -> Tuple[str, ...]:
|
||||
"""The words of a tag or a skipped entry: NFC, case-folded, split on runs
|
||||
of space, ``_`` and ``-``. Every other character belongs to a word."""
|
||||
folded = unicodedata.normalize("NFC", text).casefold()
|
||||
return tuple(word for word in _WORD_SEPARATORS.split(folded) if word)
|
||||
|
||||
|
||||
def validate_skipped_tags(value: Any) -> List[str]:
|
||||
"""The Skipped tags list, each entry trimmed, or ValueError with the
|
||||
sentence the System page shows (entries are numbered from 1)."""
|
||||
if not isinstance(value, list):
|
||||
raise ValueError("Skipped tags must be a list of words.")
|
||||
if len(value) > MAX_SKIPPED_TAGS:
|
||||
raise ValueError(f"Skipped tags: at most {MAX_SKIPPED_TAGS} entries.")
|
||||
entries: List[str] = []
|
||||
seen: Dict[Tuple[str, ...], int] = {}
|
||||
for number, raw in enumerate(value, 1):
|
||||
if not isinstance(raw, str):
|
||||
raise ValueError(f"Skipped tags: entry {number} must be text.")
|
||||
entry = raw.strip()
|
||||
if not entry:
|
||||
raise ValueError(f"Skipped tags: entry {number} is empty.")
|
||||
if len(entry) > MAX_SKIPPED_TAG_CHARS:
|
||||
raise ValueError(
|
||||
f"Skipped tags: entry {number} must be at most "
|
||||
f"{MAX_SKIPPED_TAG_CHARS} characters."
|
||||
)
|
||||
if _FORBIDDEN_IN_ENTRY.search(entry):
|
||||
raise ValueError(
|
||||
f"Skipped tags: entry {number} must not contain a comma "
|
||||
"or a line break."
|
||||
)
|
||||
words = tag_words(entry)
|
||||
if not words:
|
||||
raise ValueError(f"Skipped tags: entry {number} is empty.")
|
||||
if words in seen:
|
||||
raise ValueError(
|
||||
f"Skipped tags: entry {number} repeats entry {seen[words]}."
|
||||
)
|
||||
seen[words] = number
|
||||
entries.append(entry)
|
||||
return entries
|
||||
|
||||
|
||||
class SkipMatcher:
|
||||
"""Whether a suggested tag contains a skipped entry's words, consecutive
|
||||
and whole (#431, 17B). Compiled once per request or job; the cost of a
|
||||
check depends on the tag's words, not on the number of entries."""
|
||||
|
||||
def __init__(self, entries: Iterable[str] = ()):
|
||||
self._entries = frozenset(filter(None, (tag_words(e) for e in entries)))
|
||||
self._longest = max((len(words) for words in self._entries), default=0)
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
return bool(self._entries)
|
||||
|
||||
def matches(self, name: str) -> bool:
|
||||
if not self._entries:
|
||||
return False
|
||||
words = tag_words(name)
|
||||
for start in range(len(words)):
|
||||
stop = min(len(words), start + self._longest)
|
||||
for end in range(start + 1, stop + 1):
|
||||
if words[start:end] in self._entries:
|
||||
return True
|
||||
return False
|
||||
|
||||
def counting(self) -> "CountingSkip":
|
||||
"""A per-request view that also remembers which names it hid."""
|
||||
return CountingSkip(self)
|
||||
|
||||
|
||||
class CountingSkip:
|
||||
"""A :class:`SkipMatcher` that records the distinct names it hid, so the
|
||||
Tag image dialog can say how many skipped tags were not suggested."""
|
||||
|
||||
def __init__(self, matcher: SkipMatcher):
|
||||
self._matcher = matcher
|
||||
self.hidden: Set[str] = set()
|
||||
|
||||
def matches(self, name: str) -> bool:
|
||||
if self._matcher.matches(name):
|
||||
self.hidden.add(name.casefold())
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# A WD14 label list's tags the engine can suggest: general (0) and character (4).
|
||||
WD14_SUGGESTED_CATEGORIES = frozenset({"0", "4"})
|
||||
_label_cache: Dict[str, Tuple[Tuple[int, int], Tuple[str, ...]]] = {}
|
||||
_label_cache_lock = threading.Lock()
|
||||
|
||||
|
||||
def read_wd14_label_rows(path) -> List[Dict[str, str]]:
|
||||
"""Every row of a WD14 ``selected_tags.csv``, read as UTF-8 and closed
|
||||
before returning (the handle never outlives the call, for Windows)."""
|
||||
with open(path, "r", encoding="utf-8", newline="") as handle:
|
||||
return list(csv.DictReader(handle))
|
||||
|
||||
|
||||
def wd14_label_names(path) -> Tuple[str, ...]:
|
||||
"""The names a WD14 label list can suggest (categories 0 and 4), cached
|
||||
by ``(mtime_ns, size)``. OSError (missing) and ValueError (unreadable)
|
||||
propagate; the caller skips that list."""
|
||||
key = os.fspath(path)
|
||||
stat = os.stat(key)
|
||||
stamp = (stat.st_mtime_ns, stat.st_size)
|
||||
with _label_cache_lock:
|
||||
cached = _label_cache.get(key)
|
||||
if cached is not None and cached[0] == stamp:
|
||||
return cached[1]
|
||||
try:
|
||||
rows = read_wd14_label_rows(key)
|
||||
except csv.Error as exc: # a damaged list (a NUL byte, an oversized field)
|
||||
raise ValueError(f"unreadable label list: {exc}") from None
|
||||
names = tuple(
|
||||
name
|
||||
for row in rows
|
||||
if row.get("category", "0") in WD14_SUGGESTED_CATEGORIES
|
||||
and (name := (row.get("name") or "").strip())
|
||||
)
|
||||
with _label_cache_lock:
|
||||
_label_cache[key] = (stamp, names)
|
||||
return names
|
||||
|
||||
|
||||
# Saved instructions (#431): named language-model instructions with a default
|
||||
# per engine. Built-in is never stored; readers put it first.
|
||||
BUILTIN_INSTRUCTIONS_ID = "builtin"
|
||||
BUILTIN_INSTRUCTIONS_NAME = "Built-in"
|
||||
MAX_SAVED_INSTRUCTIONS = 20
|
||||
MAX_INSTRUCTION_NAME_CHARS = 60
|
||||
INSTRUCTION_ENGINES = ("gguf", "hf")
|
||||
ENGINE_LABELS = {"gguf": "GGUF JoyCaption", "hf": "HuggingFace JoyCaption"}
|
||||
_INSTRUCTION_ID = re.compile(r"[0-9a-f]{12}")
|
||||
_CONTROL = re.compile(r"[\x00-\x1f\x7f-\x9f
]")
|
||||
|
||||
|
||||
def builtin_instructions() -> Dict[str, str]:
|
||||
"""The read-only Built-in entry, always the shipped template."""
|
||||
return {
|
||||
"id": BUILTIN_INSTRUCTIONS_ID,
|
||||
"name": BUILTIN_INSTRUCTIONS_NAME,
|
||||
"text": DEFAULT_TAG_TEMPLATE,
|
||||
}
|
||||
|
||||
|
||||
def _instruction_entry(raw: Any, number: int) -> Dict[str, str]:
|
||||
prefix = f"Saved instructions: entry {number}"
|
||||
if (
|
||||
not isinstance(raw, dict)
|
||||
or set(raw) != {"id", "name", "text"}
|
||||
or not isinstance(raw["id"], str)
|
||||
or _INSTRUCTION_ID.fullmatch(raw["id"]) is None
|
||||
):
|
||||
raise ValueError(f"{prefix} has an invalid id.")
|
||||
name = raw["name"].strip() if isinstance(raw["name"], str) else ""
|
||||
if not name:
|
||||
raise ValueError(f"{prefix} needs a name.")
|
||||
if len(name) > MAX_INSTRUCTION_NAME_CHARS:
|
||||
raise ValueError(
|
||||
f"{prefix} name must be at most {MAX_INSTRUCTION_NAME_CHARS} characters."
|
||||
)
|
||||
if _CONTROL.search(name):
|
||||
raise ValueError(f"{prefix} name must not contain a line break.")
|
||||
if name.casefold() == BUILTIN_INSTRUCTIONS_NAME.casefold():
|
||||
raise ValueError("Saved instructions: the name “Built-in” is reserved.")
|
||||
try:
|
||||
text = validate_template(raw["text"], f"{prefix} text")
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{error}.") from None
|
||||
return {"id": raw["id"], "name": name, "text": text}
|
||||
|
||||
|
||||
def validate_instructions(value: Any) -> List[Dict[str, str]]:
|
||||
"""The saved entries ``[{id, name, text}]`` in order (names trimmed), or
|
||||
ValueError with the sentence System shows (Built-in is not counted)."""
|
||||
if not isinstance(value, list):
|
||||
raise ValueError("Saved instructions must be a list.")
|
||||
if len(value) > MAX_SAVED_INSTRUCTIONS:
|
||||
raise ValueError(
|
||||
f"Saved instructions: at most {MAX_SAVED_INSTRUCTIONS} entries besides "
|
||||
"Built-in."
|
||||
)
|
||||
entries: List[Dict[str, str]] = []
|
||||
ids: Dict[str, int] = {}
|
||||
names: Dict[str, int] = {}
|
||||
for number, raw in enumerate(value, 1):
|
||||
item = _instruction_entry(raw, number)
|
||||
if item["id"] in ids:
|
||||
raise ValueError(
|
||||
f"Saved instructions: entry {number} repeats the id of entry "
|
||||
f"{ids[item['id']]}."
|
||||
)
|
||||
key = item["name"].casefold()
|
||||
if key in names:
|
||||
raise ValueError(
|
||||
f"Saved instructions: entry {number} name “{item['name']}” is "
|
||||
"already used."
|
||||
)
|
||||
ids[item["id"]] = number
|
||||
names[key] = number
|
||||
entries.append(item)
|
||||
return entries
|
||||
|
||||
|
||||
def validate_default_instructions(value: Any, entries) -> Dict[str, str]:
|
||||
"""``{gguf: id, hf: id}``, each Built-in or an id of ``entries``."""
|
||||
if not isinstance(value, dict) or set(value) != set(INSTRUCTION_ENGINES):
|
||||
raise ValueError("Default instructions must name GGUF and HuggingFace.")
|
||||
ids = {BUILTIN_INSTRUCTIONS_ID} | {item["id"] for item in entries}
|
||||
for engine in INSTRUCTION_ENGINES:
|
||||
if not isinstance(value[engine], str) or value[engine] not in ids:
|
||||
raise ValueError(
|
||||
f"Default for {ENGINE_LABELS[engine]} must be one of the saved "
|
||||
"instructions."
|
||||
)
|
||||
return {engine: value[engine] for engine in INSTRUCTION_ENGINES}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "promptmanager"
|
||||
description = "A powerful ComfyUI custom node that extends the standard text encoder with persistent prompt storage, advanced search capabilities, and an automatic image gallery system using SQLite."
|
||||
version = "3.4.1"
|
||||
version = "3.3.1"
|
||||
license = {file = "LICENSE"}
|
||||
requires-python = ">=3.12"
|
||||
dependencies = ["watchdog>=2.1.0", "Pillow>=8.0.0"]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Runtime dependencies for PromptManager.
|
||||
# aiohttp, numpy and torch come from ComfyUI itself and are not listed here.
|
||||
watchdog>=2.1.0 # Output directory monitoring
|
||||
watchdog>=2.1.0 # Output directory monitoring
|
||||
Pillow>=8.0.0 # Image metadata extraction and thumbnails
|
||||
|
||||
@@ -6,6 +6,7 @@ import { THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate
|
||||
// app.js 1428–1432) and its backup box with the concept's System `.notice`; the differences are D55. The job view
|
||||
// has no concept counterpart: it is compared with the approved mockup `activity-scan.html` (a local, ignored file),
|
||||
// whose structure and CSS values are restated below (D56). Each comparison is shown to notice a mutation.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
|
||||
@@ -362,14 +362,10 @@ test('Show in folder asks the server to open the listed file and announces each
|
||||
const body=sent.postDataJSON();
|
||||
expect(Object.keys(body).sort()).toEqual(['path','root_index','root_scope']);
|
||||
expect(body).toMatchObject({root_index:0,path:'inspector/complete.png'});expect(body.root_scope).toMatch(/^[0-9a-f]{64}$/);
|
||||
await expect(toast).toHaveText('Opened the folder on the computer running ComfyUI.');
|
||||
// #533: the answer is really on screen and reachable above the modal viewer, not covered by it.
|
||||
await expect(toast).toBeVisible();
|
||||
expect(await toast.evaluate(node=>{const box=node.getBoundingClientRect();const top=document.elementFromPoint(box.x+box.width/2,box.y+box.height/2);return top===node;})).toBe(true);
|
||||
await expect(page.locator('#viewer-dialog').getByRole('status').filter({hasText:'Opened the folder on the computer running ComfyUI.'})).toHaveCount(1);
|
||||
await expect(toast).toHaveText('Opened the folder on this computer.');
|
||||
const opened=async()=>(await (await request.get('/__test__/reveal')).json()).opened;
|
||||
expect(await opened()).toEqual(['output/inspector/complete.png']);
|
||||
for (const [outcome,message] of [['unavailable','No file manager is available on the computer running ComfyUI.'],['fail','The folder could not be opened. Try again.']]) {
|
||||
for (const [outcome,message] of [['unavailable','No file manager is available on this computer.'],['fail','The folder could not be opened. Try again.']]) {
|
||||
await expect(await request.post('/__test__/reveal',{data:{outcome}})).toBeOK();
|
||||
await show.click();await expect(toast).toHaveText(message);
|
||||
}
|
||||
@@ -377,56 +373,11 @@ test('Show in folder asks the server to open the listed file and announces each
|
||||
await page.route('**/prompt_manager/images/reveal',route=>route.fulfill({status:429,contentType:'application/json',body:JSON.stringify({success:false,code:'busy',error:'The folder is already opening.'})}));
|
||||
await show.click();await expect(toast).toHaveText('The folder is already opening.');
|
||||
await page.unroute('**/prompt_manager/images/reveal');
|
||||
// A proxied click opens the folder: the proxy's headers, not the browser's address, name the page (#533).
|
||||
// A proxied request counts as remote, so the real route answers 403 not-local.
|
||||
await expect(await request.post('/__test__/reveal',{data:{outcome:'opened'}})).toBeOK();
|
||||
await page.route('**/prompt_manager/images/reveal',route=>route.continue({headers:{...route.request().headers(),'x-forwarded-for':'203.0.113.7'}}));
|
||||
await show.click();await expect(toast).toHaveText('Opened the folder on the computer running ComfyUI.');
|
||||
expect(await opened()).toHaveLength(2);
|
||||
await page.unroute('**/prompt_manager/images/reveal');
|
||||
// A plain cross-site POST is refused, and the toast says so.
|
||||
// Chromium keeps its own Origin on a routed fetch, so the real route is asked directly and the toast gets its answer.
|
||||
const refused=await request.post('/prompt_manager/images/reveal',{data:body,headers:{origin:'https://evil.example'}});
|
||||
expect(refused.status()).toBe(403);expect((await refused.json()).code).toBe('not-same-origin');
|
||||
await page.route('**/prompt_manager/images/reveal',route=>route.fulfill({status:403,contentType:'application/json',body:JSON.stringify({success:false,code:'not-same-origin',error:"This page's address does not match the server's."})}));
|
||||
await show.click();await expect(toast).toHaveText("The folder was not opened: this page's address does not match the server's.");
|
||||
await expect(toast).toBeVisible();
|
||||
expect(await opened()).toHaveLength(2);
|
||||
// #533: closing the viewer takes the toast back to the page, still showing the message.
|
||||
await page.keyboard.press('Escape');await expect(page.locator('#viewer-dialog')).toHaveCount(0);
|
||||
await expect(toast).toBeVisible();await expect(page.locator('body > #workspace-toast')).toHaveCount(1);
|
||||
});
|
||||
// #533 D176: real modals in the real top layer, with the page's own toast module.
|
||||
test('a toast stays on screen across nested, switched and closing modals',async({page,request})=>{
|
||||
await fullFixture(page,request);
|
||||
await page.evaluate(async()=>{
|
||||
const url=performance.getEntriesByType('resource').map(entry=>entry.name).find(name=>name.includes('/toast.mjs'));
|
||||
const {showWorkspaceToast}=await import(url);
|
||||
window.say=showWorkspaceToast;
|
||||
window.modal=(id,first=false)=>{const dialog=document.createElement('dialog');dialog.id=id;dialog.textContent=id;document.body[first?'prepend':'append'](dialog);dialog.showModal();return dialog;};
|
||||
window.onTop=()=>{const node=document.getElementById('workspace-toast');const box=node.getBoundingClientRect();return box.width>0&&document.elementFromPoint(box.x+box.width/2,box.y+box.height/2)===node;};
|
||||
});
|
||||
const parent=()=>page.evaluate(()=>{const host=document.getElementById('workspace-toast')?.parentElement;return host?host.id||host.localName:null;});
|
||||
const onTop=()=>page.evaluate(()=>window.onTop());
|
||||
await page.evaluate(()=>{window.modal('a');say('one');});
|
||||
expect(await parent()).toBe('a');expect(await onTop()).toBe(true);
|
||||
// Nested: the top one holds it, even earlier in the page than the one under it.
|
||||
await page.evaluate(()=>{window.modal('b',true);say('two');});
|
||||
expect(await parent()).toBe('b');expect(await onTop()).toBe(true);
|
||||
// The top one closes while the toast shows: the message stays, in the modal now on top.
|
||||
await page.evaluate(()=>document.getElementById('b').close());
|
||||
await expect.poll(parent).toBe('a');expect(await onTop()).toBe(true);
|
||||
await expect(page.locator('#workspace-toast')).toHaveText('two');
|
||||
// Switched: the next modal takes the next toast.
|
||||
await page.evaluate(()=>{document.getElementById('a').close();window.modal('c');say('three');});
|
||||
expect(await parent()).toBe('c');expect(await onTop()).toBe(true);
|
||||
// Closed and removed at once, as the workspace's dialogs do: home, still showing.
|
||||
await page.evaluate(()=>{const c=document.getElementById('c');c.close();c.remove();});
|
||||
await expect.poll(parent).toBe('body');expect(await onTop()).toBe(true);
|
||||
await expect(page.locator('#workspace-toast')).toHaveText('three');
|
||||
// Removed while open, with no close event: the next message still shows.
|
||||
await page.evaluate(()=>{window.modal('d');say('four');document.getElementById('d').remove();say('five');});
|
||||
expect(await parent()).toBe('body');expect(await onTop()).toBe(true);
|
||||
await expect(page.locator('#workspace-toast')).toHaveText('five');
|
||||
await show.click();await expect(toast).toHaveText('Show in folder works only in a browser on the computer running ComfyUI.');
|
||||
expect(await opened()).toEqual(['output/inspector/complete.png']);
|
||||
});
|
||||
test('full inspector shows six facts and scoped provenance',async({page,request},testInfo)=>{
|
||||
await page.setViewportSize({width:1600,height:1100});await fullFixture(page,request);const panel=inspector(page);
|
||||
@@ -464,7 +415,7 @@ test('full inspector preserves long role text and exact sends',async({page,reque
|
||||
await fullFixture(page,request);const panel=inspector(page);const positive=panel.locator('.prompt-section.positive .prompt-text');await expect(positive).toContainText('猫 😀');const text=await positive.textContent();expect(text.length).toBeGreaterThan(1000);
|
||||
await panel.locator('.detail-footer button.send').click();const sender=page.getByRole('dialog',{name:'Send to ComfyUI',exact:true});
|
||||
await expect(sender.getByRole('combobox',{name:'ComfyUI window',exact:true}).locator('option')).toContainText(['Exact host']);
|
||||
await sender.getByRole('combobox',{name:'ComfyUI window',exact:true}).selectOption({label:'Exact host'});await sender.getByRole('combobox',{name:'Destination node',exact:true}).selectOption({label:'#pair Exact pair'});
|
||||
await sender.getByRole('combobox',{name:'ComfyUI window',exact:true}).selectOption({label:'Exact host'});await sender.getByRole('combobox',{name:'Destination node',exact:true}).selectOption({label:'Exact pair'});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();await sender.getByRole('button',{name:'Apply to ComfyUI',exact:true}).click();await expect(sender).toContainText('Sent to ComfyUI.');
|
||||
expect(await host.evaluate(()=>window.sent)).toEqual([{positive:text,negative:'blurred edges, watermark'}]);
|
||||
} finally {await host.evaluate(()=>window.bridge.dispose());await host.close();}
|
||||
@@ -676,10 +627,6 @@ test('viewer matches the concept shell and toolbar order',async({page,request})=
|
||||
expect(modePaths.map(path=>path.split(' > ').slice(2).join(' > '))).toEqual(['button#viewer-mode.icon-btn','button#viewer-mode.icon-btn > svg.icon','button#viewer-mode.icon-btn > svg.icon > path']);
|
||||
expect(production.paths[production.paths.indexOf(modePaths[2])+1]).toBe('dialog#viewer-dialog > div.dialog-header > button#viewer-close');
|
||||
production.paths=production.paths.filter(path=>!modePaths.includes(path));
|
||||
// #532 D170: production's shortcuts button sits right before the view switch; the concept has none.
|
||||
const keysPaths=production.paths.filter(path=>path.includes('button#viewer-keys'));
|
||||
expect(keysPaths.map(path=>path.split(' > ').slice(2).join(' > '))).toEqual(['button#viewer-keys.icon-btn','button#viewer-keys.icon-btn > svg.icon','button#viewer-keys.icon-btn > svg.icon > path']);
|
||||
production.paths=production.paths.filter(path=>!keysPaths.includes(path));
|
||||
// #480 D159: the Images viewer's Delete… (`danger`) follows Download; the concept has none.
|
||||
expect(production.toolbar[8].text).toBe('Delete…');
|
||||
production.toolbar.splice(8,1);
|
||||
|
||||
@@ -107,7 +107,7 @@ export async function exerciseCurrentPairSend(page, roles) {
|
||||
const send = viewer.getByRole('button', { name: /^Send prompt pair to ComfyUI: / });await send.click();
|
||||
const sender = page.getByRole('dialog', { name: 'Send to ComfyUI', exact: true });await expect(sender).toBeVisible();
|
||||
await sender.getByRole('combobox', { name: 'ComfyUI window', exact: true }).selectOption({ label: 'Canonical ComfyUI' });
|
||||
await sender.getByRole('combobox', { name: 'Destination node', exact: true }).selectOption({ label: '#canonical-pair Canonical destination' });
|
||||
await sender.getByRole('combobox', { name: 'Destination node', exact: true }).selectOption({ label: 'Canonical destination' });
|
||||
await sender.getByRole('button', { name: 'Preview changes', exact: true }).click();
|
||||
await expect(sender).toContainText(roles.positive);await expect(sender).toContainText(roles.negative);
|
||||
await sender.getByRole('button', { name: 'Apply to ComfyUI', exact: true }).click();await expect(sender).toContainText('Sent to ComfyUI.');
|
||||
|
||||
@@ -194,7 +194,7 @@ function productionSent(host) {
|
||||
return async page=>{
|
||||
const sender=page.getByRole('dialog',{name:'Send to ComfyUI',exact:true});
|
||||
await sender.getByRole('combobox',{name:'ComfyUI window',exact:true}).selectOption({label:'Concept ComfyUI'});
|
||||
await sender.getByRole('combobox',{name:'Destination node',exact:true}).selectOption({label:'#pair-1 Concept destination'});
|
||||
await sender.getByRole('combobox',{name:'Destination node',exact:true}).selectOption({label:'Concept destination'});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();
|
||||
await expect.poll(()=>host.evaluate(()=>window.previewed.length)).toBe(1);
|
||||
const roles=await host.evaluate(()=>window.previewed.pop());
|
||||
|
||||
@@ -474,10 +474,7 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`images selecti
|
||||
expect.soft(actual.bulkbar.bar.items.map(item=>item.text),'production bar').toEqual(['1 selected','Add tags','Auto-tag selected',
|
||||
'Set category','Add to collection','Export metadata','Delete…','Clear']);
|
||||
expect.soft(compare(empty,emptyActual,manifest,['bulkbar','cards','panel']),'nothing checked').toEqual([]);
|
||||
// D186: the panel's sticky offset clears the shown bar; at one column the bar is one sideways row (its items are checked above).
|
||||
const stuckPanel={...oracle,panel:{...oracle.panel,...(width<=700?{top:actual.panel.top}:{}),box:{...oracle.panel.box,top:actual.panel.box.top}}};
|
||||
expect(parseFloat(actual.panel.box.top)).toBeGreaterThanOrEqual(20);
|
||||
expect.soft(compare(stuckPanel,actual,manifest,width<=700?['panel']:['bulkbar','cards','panel']),'one card checked').toEqual([]);
|
||||
expect.soft(compare(oracle,actual,manifest,['bulkbar','cards','panel']),'one card checked').toEqual([]);
|
||||
// The bar comparison notices a change to the production bar.
|
||||
await mutate(page,{css:'.bulkbar { padding: 4px }'});
|
||||
expect(compare(oracle,await inventoryOf(page,'#workspace-main'),manifest,['bulkbar']).map(item=>item.path)).toContain('bulkbar.bar.paddingTop');
|
||||
|
||||
@@ -259,7 +259,7 @@ async function productionSendOf(page,context) {
|
||||
const sender=page.getByRole('dialog',{name:'Send to ComfyUI',exact:true});await expect(sender).toBeVisible();
|
||||
const windows=sender.getByRole('combobox',{name:'ComfyUI window',exact:true});
|
||||
await expect(windows.locator('option')).toContainText(['Test ComfyUI']);await windows.selectOption({label:'Test ComfyUI'});
|
||||
await sender.getByRole('combobox',{name:'Destination node',exact:true}).selectOption({label:'#pair-1 My prompt pair'});
|
||||
await sender.getByRole('combobox',{name:'Destination node',exact:true}).selectOption({label:'My prompt pair'});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();
|
||||
await sender.getByRole('button',{name:'Apply to ComfyUI',exact:true}).click();
|
||||
await expect(sender).toContainText('Sent to ComfyUI.');
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
// A job's end toast follows the job, which has no time limit: under load it outlasts the 5 s expect default (#533, as
|
||||
// #520). Do `action`, whose POST to `url` answers with the job it started, then wait for that job's event stream to end.
|
||||
export const APPLY_URL = '/prompt_manager/maintenance/apply';
|
||||
export const RETURN_URL = '/prompt_manager/images/removed/return';
|
||||
|
||||
export async function startAndAwaitJob(page, action, url = APPLY_URL) {
|
||||
const applied = page.waitForResponse(response => response.url().endsWith(url));
|
||||
await action();
|
||||
const { job } = await (await applied).json();
|
||||
const events = await page.request.get(`/prompt_manager/jobs/${job.id}/events`);
|
||||
if (!events.ok()) throw new Error(`Job ${job.id} events answered ${events.status()}`);
|
||||
return job;
|
||||
}
|
||||
@@ -233,7 +233,7 @@ function productionSent(host) {
|
||||
return async page=>{
|
||||
const sender=page.getByRole('dialog',{name:'Send to ComfyUI',exact:true});
|
||||
await sender.getByRole('combobox',{name:'ComfyUI window',exact:true}).selectOption({label:'Concept ComfyUI'});
|
||||
await sender.getByRole('combobox',{name:'Destination node',exact:true}).selectOption({label:'#pair-1 Concept destination'});
|
||||
await sender.getByRole('combobox',{name:'Destination node',exact:true}).selectOption({label:'Concept destination'});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();
|
||||
await expect.poll(()=>host.evaluate(()=>window.previewed.length)).toBe(1);
|
||||
const roles=await host.evaluate(()=>window.previewed.pop());
|
||||
@@ -374,9 +374,7 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`the Library se
|
||||
await captureCanonicalState(page,testInfo,`production-library-selection-${width}-${theme}`);
|
||||
await testInfo.attach('library-selection',{body:JSON.stringify({reference,actual},null,2),contentType:'application/json'});
|
||||
expect(reference.bar.buttons.map(button=>button.text),'the concept bar has every Library action').toEqual(['Add tags','Set category','Clear']);
|
||||
// D186: at one column the bar is one row that scrolls sideways, with the same buttons.
|
||||
const mapped=width<=700?{...reference,bar:{...reference.bar,flexWrap:'nowrap',buttons:reference.bar.buttons.map((button,index)=>({...button,left:actual.bar.buttons[index].left,top:actual.bar.buttons[index].top}))}}:reference;
|
||||
expect(actual).toEqual(mapped);
|
||||
expect(actual).toEqual(reference);
|
||||
// The comparison notices a moved checkbox (spec §7 mutation check).
|
||||
await mutate(page,{css:'.prompt-card .image-overlays { inset: 20px 9px auto }'});
|
||||
expect((await measureSelection(page,{card:CARDS,main:'#workspace-main'})).overlay.top).not.toBe(reference.overlay.top);
|
||||
|
||||
@@ -33,7 +33,7 @@ async function chooseDestination(page) {
|
||||
const host=sender.getByRole('combobox',{name:'ComfyUI window',exact:true});
|
||||
await expect(host.locator('option')).toContainText(['Test ComfyUI']);
|
||||
await host.selectOption({label:'Test ComfyUI'});
|
||||
await sender.getByRole('combobox',{name:'Destination node',exact:true}).selectOption({label:'#pair-1 My prompt pair'});
|
||||
await sender.getByRole('combobox',{name:'Destination node',exact:true}).selectOption({label:'My prompt pair'});
|
||||
return sender;
|
||||
}
|
||||
|
||||
@@ -165,8 +165,7 @@ test('host apply rejection retains the current text and requires a new preview',
|
||||
await page.goto(workspace);await select(page,'0:shared/same.png');await details(page).getByRole('button',{name:/^Send positive prompt to ComfyUI: /}).click();
|
||||
const sender=await chooseDestination(page);await sender.getByRole('button',{name:'Preview changes',exact:true}).click();
|
||||
await sender.getByRole('button',{name:'Apply to ComfyUI',exact:true}).click();await expect(sender).toContainText(/Workflow changed/);
|
||||
// D175: a refusal wrote nothing, so Apply may try again.
|
||||
await expect(sender.getByRole('button',{name:'Apply to ComfyUI',exact:true})).toBeEnabled();
|
||||
await expect(sender.getByRole('button',{name:'Apply to ComfyUI',exact:true})).toBeDisabled();
|
||||
expect(await host.evaluate(()=>window.hostValues.positive)).toBe('Existing positive');
|
||||
} finally {await host.evaluate(()=>window.fixtureHost.dispose());await host.close();}
|
||||
});
|
||||
@@ -194,7 +193,7 @@ test('empty negative send clears only the negative destination and preserves its
|
||||
test('no open ComfyUI window is actionable and rediscovery finds a newly opened host',async({page,context})=>{
|
||||
await page.goto(workspace);await select(page,'0:shared/same.png');
|
||||
await details(page).getByRole('button',{name:/^Send positive prompt to ComfyUI: /}).click();
|
||||
const sender=dialog(page);await expect(sender).toContainText('No ComfyUI window answered in this browser');
|
||||
const sender=dialog(page);await expect(sender).toContainText('No compatible ComfyUI windows found.');
|
||||
const host=await fakeHost(context,page);
|
||||
try {
|
||||
await sender.getByRole('button',{name:'Find ComfyUI windows',exact:true}).click();
|
||||
@@ -320,44 +319,3 @@ test('copy toast and fallback dialog',async({page})=>{
|
||||
await expect(fallback).toContainText('Automatic clipboard access requires a secure browser context.');
|
||||
await page.keyboard.press('Escape');await expect(fallback).toBeHidden();await expect(copy).toBeFocused();
|
||||
});
|
||||
|
||||
// #542 D191: "All N eligible nodes" previews every node, sends one after another and names each refusal.
|
||||
test('All eligible nodes previews each node, sends in node id order and names a refusal',async({page,context},testInfo)=>{
|
||||
const host=await context.newPage();await host.goto('/prompt_manager/workspace#/overview');
|
||||
await host.evaluate(async()=>{
|
||||
const {createBridgeHost}=await import('/prompt_manager/js/shared/toolkit/prompt-bridge.js');
|
||||
const targets=['12','3','7'].map(id=>({id,label:id==='7'?'Edited in ComfyUI':'Prompt Manager (+/-)',type:'PromptManagerPair',roles:['positive','negative']}));
|
||||
window.hostValues=Object.fromEntries(targets.map(target=>[target.id,{positive:`Existing ${target.id}`,negative:'Existing negative'}]));window.applied=[];
|
||||
window.fixtureHost=createBridgeHost({origin:location.origin,namespace:'/',label:'Test ComfyUI',adapter:{
|
||||
listTargets:()=>targets,
|
||||
preview:(id,roles)=>{const target=targets.find(item=>item.id===id);return{preview:{target,fields:Object.entries(roles).map(([role,after])=>({role,before:window.hostValues[id][role],after}))},snapshot:{id,roles}};},
|
||||
apply:({id,roles})=>{if(id==='7')throw new Error('Workflow or prompt changed. Preview again.');Object.assign(window.hostValues[id],roles);window.applied.push(id);return{targetId:id,values:roles};},
|
||||
}});
|
||||
});
|
||||
try {
|
||||
await page.goto(workspace);await select(page,'0:shared/same.png');await details(page).getByRole('button',{name:/^Send prompt pair to ComfyUI: /}).click();
|
||||
const sender=dialog(page);const nodes=sender.getByRole('combobox',{name:'Destination node',exact:true});
|
||||
await sender.getByRole('combobox',{name:'ComfyUI window',exact:true}).selectOption({label:'Test ComfyUI'});
|
||||
await expect(nodes.locator('option')).toHaveText(['Choose a destination node','#12 Prompt Manager (+/-)','#3 Prompt Manager (+/-)','#7 Edited in ComfyUI','All 3 eligible nodes']);
|
||||
await expect(nodes).toHaveValue('');
|
||||
await nodes.selectOption({label:'All 3 eligible nodes'});
|
||||
await expect(sender.getByRole('status')).toHaveText('Sending to all 3 eligible nodes in Test ComfyUI. Apply to send it, or preview the changes first.');
|
||||
await page.screenshot({path:testInfo.outputPath('send-all-option.png')});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();
|
||||
const reviews=sender.locator('section.prompt-sender-node');
|
||||
await expect(reviews.locator('> h4')).toHaveText(['#3 Prompt Manager (+/-)','#7 Edited in ComfyUI','#12 Prompt Manager (+/-)']);
|
||||
await expect(sender.getByRole('region',{name:'#12 Prompt Manager (+/-)',exact:true})).toContainText('Existing 12');
|
||||
expect(await host.evaluate(()=>window.applied)).toEqual([]);
|
||||
await sender.getByRole('button',{name:'Apply to ComfyUI',exact:true}).click();
|
||||
await expect(sender.getByRole('alert')).toHaveText('Sent to 2 of 3 nodes in Test ComfyUI. #7 Edited in ComfyUI: Workflow or prompt changed. Preview again.');
|
||||
expect(await host.evaluate(()=>window.applied)).toEqual(['3','12']);
|
||||
expect(await host.evaluate(()=>window.hostValues['7'].positive)).toBe('Existing 7');
|
||||
await page.screenshot({path:testInfo.outputPath('send-all-result.png')});
|
||||
await page.setViewportSize({width:390,height:844});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();await expect(reviews).toHaveCount(3);
|
||||
await reviews.last().scrollIntoViewIfNeeded();
|
||||
expect(await page.evaluate(()=>document.querySelector('dialog.prompt-sender').scrollWidth<=document.querySelector('dialog.prompt-sender').clientWidth+1)).toBe(true);
|
||||
await page.screenshot({path:testInfo.outputPath('send-all-mobile.png')});
|
||||
const apply=sender.getByRole('button',{name:'Apply to ComfyUI',exact:true});await apply.focus();await expect(apply).toBeFocused();
|
||||
} finally {await host.evaluate(()=>window.fixtureHost.dispose());await host.close();}
|
||||
});
|
||||
|
||||
@@ -26,10 +26,6 @@ async function openReview(page, theme) {
|
||||
await openProduction(page, 'review', theme);
|
||||
await expect(panel(page).getByRole('combobox', { name: 'Sampler context', exact: true })).toBeEnabled();
|
||||
await settle(page);
|
||||
// D179, D181: the two buttons the concept lacks are pinned here, then left out of the comparison.
|
||||
await expect(panel(page).getByRole('button', { name: 'Leave unresolved', exact: true })).toBeVisible();
|
||||
await expect(panel(page).getByRole('button', { name: 'Link to a prompt…', exact: true })).toBeVisible();
|
||||
await page.addStyleTag({ content: '.review-panel .review-leave-unresolved, .review-panel .review-link-open { display: none !important; }' });
|
||||
}
|
||||
async function settle(page) {
|
||||
await page.mouse.move(0, 0);
|
||||
@@ -94,7 +90,7 @@ for (const [width, height] of SIZES) for (const theme of THEMES) {
|
||||
expect(surface.children[1].text).toBe(CONCEPT_FILE);
|
||||
const actions = surface.children.at(-1);
|
||||
expect(actions.children.map(button => [button.text, button.primary])).toEqual([['Leave for later', false], ['Apply association', true]]);
|
||||
expect(node(surface, 'select').options.map(option => option[0])).not.toContain('Leave unresolved'); // D179
|
||||
expect(node(surface, 'select').options.at(-1)).toEqual(['Leave unresolved', false]);
|
||||
expect(node(main, 'div.image-grid').children.map(card => node(card, 'h3').text))
|
||||
.toEqual(['alpine_00001.png', 'dunes_00001.png', 'forest_00001.png']);
|
||||
expect(actual.noPageScroll).toBe(true);
|
||||
@@ -173,7 +169,7 @@ test('the comparison notices a dropped notice border and reordered actions', asy
|
||||
const reordered = await productionInventory(page);
|
||||
expect(diffInventory(reference, reordered).map(item => item.path)).toEqual(expect.arrayContaining([
|
||||
'main.children.2.children.1.children.6.children.0.text', 'main.children.2.children.1.children.6.children.0.primary']));
|
||||
await panel(page).locator('.dialog-actions').evaluate(row => row.append(row.firstElementChild));
|
||||
await panel(page).locator('.dialog-actions').evaluate(row => row.prepend(row.lastElementChild));
|
||||
expect(diffInventory(reference, await productionInventory(page))).toEqual([]);
|
||||
});
|
||||
|
||||
|
||||
@@ -85,9 +85,6 @@ class TestModel:
|
||||
suggestions: list | None = None
|
||||
# The engine's model table (``py/autotag.py`` MODELS), set at startup.
|
||||
models_config: dict = {}
|
||||
# Where the WD14 label lists live (the Skipped tags preview, #431), set at
|
||||
# startup; the tagging fixture writes or removes a small list there.
|
||||
models_dir = None
|
||||
|
||||
def get_models_status(self) -> dict:
|
||||
from py.autotag import package_fields
|
||||
@@ -118,26 +115,21 @@ class TestModel:
|
||||
return ["generated", Path(image_path).parent.parent.name]
|
||||
|
||||
def generate_tag_suggestions(
|
||||
self, image_path: str, *, limit: int, skip=None, **kwargs: object
|
||||
self, image_path: str, *, limit: int, **kwargs: object
|
||||
) -> list[tuple[str, float | None]]:
|
||||
"""Fixed scored suggestions (WD14) or unscored ones (language models),
|
||||
without the ones ``skip`` hides before the limit, as the engine does
|
||||
(#431)."""
|
||||
"""Fixed scored suggestions (WD14) or unscored ones (language models)."""
|
||||
scored = self.loaded is not None and self.loaded.startswith("wd14")
|
||||
if self.suggestions is not None:
|
||||
pairs = [
|
||||
return [
|
||||
(name, confidence if scored else None)
|
||||
for name, confidence in self.suggestions
|
||||
for name, confidence in self.suggestions[:limit]
|
||||
]
|
||||
else:
|
||||
names = ["sandstone", "desert", "arch", "sky", "rock", "dune", "canyon"]
|
||||
names += [f"tag_{index:02d}" for index in range(100)]
|
||||
pairs = [
|
||||
(name, round(0.98 - index * 0.008, 3) if scored else None)
|
||||
for index, name in enumerate(names)
|
||||
]
|
||||
kept = [pair for pair in pairs if skip is None or not skip.matches(pair[0])]
|
||||
return kept[:limit]
|
||||
names = ["sandstone", "desert", "arch", "sky", "rock", "dune", "canyon"]
|
||||
names += [f"tag_{index:02d}" for index in range(100)]
|
||||
return [
|
||||
(name, round(0.98 - index * 0.008, 3) if scored else None)
|
||||
for index, name in enumerate(names[:limit])
|
||||
]
|
||||
|
||||
def download_model(self, model_type: str, progress_callback=None) -> bool:
|
||||
raise AssertionError("browser tests never download a model")
|
||||
@@ -241,7 +233,6 @@ def create_app(root: Path) -> web.Application:
|
||||
|
||||
TestModel.models_config = autotag.MODELS
|
||||
model = TestModel()
|
||||
model.models_dir = root / "models"
|
||||
autotag.get_autotag_service = lambda: model
|
||||
api = PromptManagerAPI()
|
||||
api.add_routes(routes)
|
||||
|
||||
@@ -109,7 +109,6 @@ class ResetWatch:
|
||||
stream.write(text)
|
||||
stream.flush()
|
||||
except (OSError, ValueError):
|
||||
# the stream closed; a watcher must not fail the server
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -162,15 +162,13 @@ async function serverJson(request, url) {
|
||||
}
|
||||
|
||||
const number = value => value.toLocaleString('en-US');
|
||||
// The sidebar's Images count (state.mjs), which System's Images metric shows (#555).
|
||||
const IMAGES_COUNT_URL = '/prompt_manager/images/output?mode=workspace&limit=1';
|
||||
|
||||
/** The concept's shell and Overview with the recorded deviations applied, given the server's real overview. */
|
||||
function mapOverviewConcept(reference, overview, images) {
|
||||
function mapOverviewConcept(reference, overview) {
|
||||
// D165: the discreet legacy-pages line follows the settings layout.
|
||||
reference = { ...reference, tree: [...reference.tree, 'p.legacy-pages.small'] };
|
||||
const relabel = { 'Database integrity': 'Not checked', 'Positive prompts': number(overview.library.positive_prompts),
|
||||
'Negative variants': number(overview.library.negative_variants), Images: number(images) };
|
||||
'Negative variants': number(overview.library.negative_variants), Images: number(overview.images.indexed) };
|
||||
const rows = {
|
||||
// D93: the real pending count; Review metadata opens Needs review (the concept's go-review).
|
||||
'3 ambiguous image links': row => ({ ...row,
|
||||
@@ -181,9 +179,8 @@ function mapOverviewConcept(reference, overview, images) {
|
||||
'One display configuration': row => ({ ...row, description: 'Grid and density settings are shared across views.' }),
|
||||
// D93: the real sources, not samples.
|
||||
'2 sample output sources': row => ({ ...row, title: `${number(overview.sources.effective)} output sources` }),
|
||||
// D96: the dialog shows runs by category and database record counts, not prompt, tag or date usage (D153: no
|
||||
// ratings; #555: the counts are labelled as records, not as the Library or Images page counts).
|
||||
Statistics: row => ({ ...row, description: 'Runs by category, with database record counts.' }),
|
||||
// D96: the dialog shows runs by category and library totals, not prompt, tag or date usage (D153: no ratings).
|
||||
Statistics: row => ({ ...row, description: 'Runs by category, with library totals.' }),
|
||||
};
|
||||
const surfaces = reference.columns.surfaces.map(surface => ({ ...surface,
|
||||
rows: surface.rows.map(row => (rows[row.title] ?? (value => value))(row)) }));
|
||||
@@ -221,8 +218,6 @@ async function conceptOverview(page, theme) {
|
||||
async function productionOverview(page, theme) {
|
||||
await openProduction(page, 'system/overview', theme);
|
||||
await expect(page.locator('#workspace-main .summary-metrics')).toBeVisible();
|
||||
// #555: the Images metric is the shell's Images count: compare once it was read.
|
||||
await expect(page.locator('#workspace-main .summary-metrics > .metric:nth-child(3) strong')).toHaveText(/^[\d,]+$/);
|
||||
// The thumbnail row is its own read (I2): compare once it answered.
|
||||
await expect(page.locator('#workspace-main .operation h3').filter({ hasText: 'thumbnails' })).toHaveText('No missing thumbnails');
|
||||
// So is the backup row (S9).
|
||||
@@ -236,7 +231,6 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`system shell and overview match the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const overview = await serverJson(request, '/prompt_manager/system/overview');
|
||||
const images = (await serverJson(request, IMAGES_COUNT_URL)).total;
|
||||
// The fixture's thumbnails all exist, so the concept's "No missing thumbnails" is also the real title.
|
||||
expect((await serverJson(request, '/prompt_manager/images/thumbnails/status')).pending).toBe(0);
|
||||
const concept = await conceptOverview(page, theme);
|
||||
@@ -245,7 +239,7 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-system-${theme}-${width}.png`), fullPage: true });
|
||||
const production = await productionOverview(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-system-${theme}-${width}.png`), fullPage: true });
|
||||
expect(diffInventory(mapOverviewConcept(concept, overview, images), production)).toEqual([]);
|
||||
expect(diffInventory(mapOverviewConcept(concept, overview), production)).toEqual([]);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
}
|
||||
@@ -271,9 +265,8 @@ const UNMAPPED_PATHS = [
|
||||
test('system comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const overview = await serverJson(request, '/prompt_manager/system/overview');
|
||||
const images = (await serverJson(request, IMAGES_COUNT_URL)).total;
|
||||
const concept = await conceptOverview(page, 'dark');
|
||||
const expected = mapOverviewConcept(concept, overview, images);
|
||||
const expected = mapOverviewConcept(concept, overview);
|
||||
const production = await productionOverview(page, 'dark');
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Unmapped, the concept differs exactly where a deviation is recorded, and nowhere else.
|
||||
@@ -297,7 +290,7 @@ test('system comparison detects mutations', async ({ page, request }) => {
|
||||
});
|
||||
|
||||
/**
|
||||
* D96: the concept's table, its rows the real ones; then "Database records" (#555) in the same table markup. D153 (#447): the
|
||||
* D96: the concept's table, its rows the real ones; then "Library totals" in the same table markup. D153 (#447): the
|
||||
* concept's "Avg. rating" column and a "Rated images" total are not drawn.
|
||||
*/
|
||||
function mapStatisticsConcept(reference, statistics) {
|
||||
@@ -309,10 +302,9 @@ function mapStatisticsConcept(reference, statistics) {
|
||||
tables: [
|
||||
{ ...runs, head: runs.head.filter(([, , text]) => text !== 'Avg. rating'),
|
||||
rows: statistics.by_category.map(row => [row.category ?? 'Uncategorized', number(row.runs)]) },
|
||||
{ caption: 'Database records', head: [['th', 'col', 'Total'], ['th', 'col', 'Count']],
|
||||
rows: [['Prompt records', number(statistics.totals.total_prompts)],
|
||||
['Tags on prompts', number(statistics.totals.total_tags)],
|
||||
['Image records', number(statistics.totals.total_images)]] },
|
||||
{ caption: 'Library totals', head: [['th', 'col', 'Total'], ['th', 'col', 'Count']],
|
||||
rows: [['Prompts', number(statistics.totals.total_prompts)], ['Tags', number(statistics.totals.total_tags)],
|
||||
['Images', number(statistics.totals.total_images)]] },
|
||||
],
|
||||
};
|
||||
}
|
||||
@@ -428,7 +420,7 @@ for (const theme of THEMES) test(`statistics and health dialogs match the concep
|
||||
expect(diffInventory(mapHealthConcept(concept.health), await page.evaluate(measureDialog)).map(item => item.path))
|
||||
.toEqual(['operations.rows.0.title']);
|
||||
await page.evaluate(() => { document.querySelector('#modal .operation h3').textContent = 'Integrity check'; });
|
||||
// And a removed "Database records" table and a changed caption in the statistics dialog.
|
||||
// And a removed "Library totals" table and a changed caption in the statistics dialog.
|
||||
await page.keyboard.press('Escape');
|
||||
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Explore', exact: true }).click();
|
||||
|
||||
@@ -88,7 +88,7 @@ function measureIntegrations(mainSelector) {
|
||||
};
|
||||
}
|
||||
|
||||
const SEND_REASON = 'Sending asks when there is more than one ComfyUI window or node, so “Ask each time” is the only option.';
|
||||
const SEND_REASON = 'Sending always asks for the ComfyUI window and node, so “Ask each time” is the only option.';
|
||||
const RENDERER_REASON = 'Prompt Manager nodes always follow ComfyUI’s own renderer setting.';
|
||||
const RENDERER_NONE = 'No ComfyUI window is open in this browser, so its renderer is not known.';
|
||||
const RETURN_REASON = 'Not available: sending applies the text in ComfyUI and keeps this page open.';
|
||||
|
||||
@@ -64,31 +64,11 @@ function measureMaintenance(mainSelector) {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* D192 (#550, beyond the concept): Database adds "Pair category and tags" after "Unused prompt components", shaped like
|
||||
* the concept's plain Review plan row (Optimize database's).
|
||||
*/
|
||||
function mapPairRow(reference) {
|
||||
return {
|
||||
...reference,
|
||||
surfaces: reference.surfaces.map(surface => {
|
||||
const plain = surface.rows.find(row => row.title === 'Optimize database');
|
||||
const at = surface.rows.findIndex(row => row.title === 'Unused prompt components');
|
||||
if (!plain || at < 0) return surface;
|
||||
return { ...surface, tree: [...surface.tree, 'div.operation'], rows: [...surface.rows.slice(0, at + 1),
|
||||
{ ...plain, title: 'Pair category and tags', description: 'Fill an empty category or tags on prompts older ' +
|
||||
'Pair node runs saved, from their saved pairs. Only empty fields of never-edited prompts.' },
|
||||
...surface.rows.slice(at + 1)] };
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/** D102: the Metadata cache row says why its review is unavailable and its button is aria-disabled (D93's dimmed style). */
|
||||
function mapMaintenanceConcept(reference) {
|
||||
const mapped = mapPairRow(reference);
|
||||
return {
|
||||
...mapped,
|
||||
surfaces: mapped.surfaces.map(surface => ({ ...surface, rows: surface.rows.map(row => row.title !== 'Metadata cache' ? row : {
|
||||
...reference,
|
||||
surfaces: reference.surfaces.map(surface => ({ ...surface, rows: surface.rows.map(row => row.title !== 'Metadata cache' ? row : {
|
||||
...row, description: `${row.description} Nothing is cached in this browser.`,
|
||||
actions: row.actions.map(action => ({ ...action, opacity: '0.55', cursor: 'not-allowed' })) }) })),
|
||||
};
|
||||
@@ -229,8 +209,8 @@ test('maintenance comparison detects mutations', async ({ page, request }) => {
|
||||
const expected = mapMaintenanceConcept(concept);
|
||||
const production = await productionMaintenance(page, 'dark');
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Mapped only for the D192 row, the section differs only on the D102 row.
|
||||
expect(diffInventory(mapPairRow(concept), production).map(item => item.path).sort()).toEqual([
|
||||
// Unmapped, the section differs only on the D102 row.
|
||||
expect(diffInventory(concept, production).map(item => item.path).sort()).toEqual([
|
||||
'surfaces.1.rows.3.actions.0.cursor', 'surfaces.1.rows.3.actions.0.opacity', 'surfaces.1.rows.3.description']);
|
||||
const measure = () => page.evaluate(measureMaintenance, '#workspace-main');
|
||||
const paths = async () => diffInventory(expected, await measure()).map(item => item.path);
|
||||
|
||||
@@ -33,10 +33,6 @@ test.afterEach(async ({ page, request, browserName }) => {
|
||||
const PREVIEW_URL = '/prompt_manager/autotag/batch/preview';
|
||||
const MODEL_OPTIONS = ['WD14 SwinV2 · ~400 MB', 'WD14 ViT · ~400 MB', 'GGUF JoyCaption · ~4.5 GB', 'HuggingFace JoyCaption · ~16 GB'];
|
||||
const SUGGESTIONS = 'From 1 to 100: the most suggestions Generate shows for one image.';
|
||||
const SKIPPED_DESCRIPTION = 'Suggested tags that contain one of these words are hidden. Words are separated by spaces, _ or -; '
|
||||
+ 'other characters such as ( ) belong to the word.';
|
||||
const DEFAULT_DESCRIPTION = 'The Tag image dialog and batches start from this.';
|
||||
const INSTRUCTIONS_DESCRIPTION = 'Instructions for the language-model engines. Built-in can’t be changed; duplicate it to make your own.';
|
||||
const BATCH = 'Tag each output image from its own pixels with the model above. Prompts are not changed. Review the scope first.';
|
||||
|
||||
/** Runs in the page: Auto-tagging's surface under `mainSelector`: rows, field, actions, operation, geometry and tokens. */
|
||||
@@ -66,7 +62,7 @@ function measureTagging(mainSelector) {
|
||||
text: [...row.firstElementChild.children].map(node => [name(node), text(node), node.localName === 'label' ? node.htmlFor : null]),
|
||||
control: control(row), flexWrap: getComputedStyle(row).flexWrap,
|
||||
controlBelow: row.lastElementChild.getBoundingClientRect().top >= row.firstElementChild.getBoundingClientRect().bottom - 0.5 })),
|
||||
field: field && { tree: [...field.children].map(name), label: [text(field.querySelector('label')), field.querySelector('label').htmlFor],
|
||||
field: { tree: [...field.children].map(name), label: [text(field.querySelector('label')), field.querySelector('label').htmlFor],
|
||||
textarea: { id: field.querySelector('textarea').id, value: field.querySelector('textarea').value,
|
||||
...css(field.querySelector('textarea'), 'fontSize', 'minHeight', 'backgroundColor', 'color', 'borderTopColor') },
|
||||
...css(field, 'display', 'rowGap', 'fontSize', 'marginTop', 'marginBottom') },
|
||||
@@ -74,31 +70,6 @@ function measureTagging(mainSelector) {
|
||||
...css(actions, 'display', 'flexWrap', 'columnGap', 'justifyContent'),
|
||||
primary: css(actions.querySelector('.ca-primary'), 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft'),
|
||||
plain: css(actions.querySelector('button:not(.ca-primary)'), 'backgroundColor', 'color', 'fontSize', 'borderTopColor') },
|
||||
// D202 (#431): the Skipped tags block (the concept has none).
|
||||
skipped: (block => block && { tree: [...block.children].map(name), key: block.dataset.key,
|
||||
text: [...block.firstElementChild.children].map(node => [node.localName, text(node), node.localName === 'label' ? node.htmlFor : null]),
|
||||
add: [...block.querySelector('.skip-add').children].map(node => [name(node), node.id, node.getAttribute('placeholder'), text(node)]),
|
||||
preview: [text(block.querySelector('.skip-preview')), block.querySelector('.skip-preview').getAttribute('role')],
|
||||
...css(block, 'paddingTop', 'paddingBottom', 'borderTopWidth', 'borderTopColor'),
|
||||
label: css(block.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
description: css(block.querySelector('.setting-block-text p'), 'fontSize', 'color', 'marginTop'),
|
||||
input: css(block.querySelector('input'), 'fontSize', 'backgroundColor', 'color', 'borderTopColor') })(
|
||||
surface.querySelector(':scope > .setting-block[data-setting="tag-skipped"]')),
|
||||
// D203 (#431): Saved instructions (the concept has a template field instead).
|
||||
instructions: (block => block && { tree: [...block.children].map(name), key: block.dataset.key,
|
||||
text: [...block.firstElementChild.children].map(node => [node.localName, text(node)]),
|
||||
list: [...block.querySelectorAll('[role="option"]')].map(option => [text(option.firstElementChild),
|
||||
[...option.querySelectorAll('.badges > .chip')].map(chip => [text(chip), chip.className]), option.getAttribute('aria-selected')]),
|
||||
listRole: block.querySelector('.instruction-list').getAttribute('role'),
|
||||
actions: [...block.querySelectorAll('.instruction-actions > button')].map(item => [text(item), item.className, item.disabled]),
|
||||
editor: [...block.querySelectorAll('.instruction-editor .field')].map(item => [text(item.querySelector('label > span')),
|
||||
item.querySelector('input, textarea').value, item.querySelector('input, textarea').readOnly]),
|
||||
note: text(block.querySelector('#instruction-note')),
|
||||
layout: getComputedStyle(block.querySelector('.instructions-layout')).gridTemplateColumns.split(' ').length,
|
||||
...css(block, 'paddingTop', 'paddingBottom', 'borderTopWidth', 'borderTopColor'),
|
||||
label: css(block.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
textarea: css(block.querySelector('textarea'), 'fontSize', 'backgroundColor', 'borderTopColor') })(
|
||||
surface.querySelector(':scope > .setting-block[data-setting="tag-instructions"]')),
|
||||
operations: [...surface.querySelectorAll(':scope > .operation')].map(row => ({ tree: [...row.children].map(name),
|
||||
title: text(row.querySelector('h3')), description: text(row.querySelector('p')),
|
||||
actions: [...row.querySelectorAll('.row-actions > button')].map(item => [text(item), item.className]) })),
|
||||
@@ -132,21 +103,13 @@ function measureTagging(mainSelector) {
|
||||
* The concept's Auto-tagging with the recorded deviations applied, for the fixture (all engines on disk, none loaded).
|
||||
* `phone` is a viewport at the concept's 700px breakpoint or below.
|
||||
*/
|
||||
function mapTaggingConcept(reference, { builtin, saved, phone }) {
|
||||
function mapTaggingConcept(reference, { template, phone }) {
|
||||
const [model, existing, general, character] = reference.rows;
|
||||
const [preview] = reference.actions.buttons;
|
||||
// D203: Built-in first, then the saved entries, in each default select.
|
||||
const names = ['Built-in', ...saved.map(entry => entry.name)];
|
||||
const defaultRow = (engine, long) => ({ ...existing,
|
||||
text: [['label', `Default for ${long}`, `instruction-default-${engine}`], ['p', DEFAULT_DESCRIPTION, null]],
|
||||
control: { ...existing.control, id: `instruction-default-${engine}`, options: names.map(label => [label, false]),
|
||||
selected: names[1] } });
|
||||
return {
|
||||
...reference,
|
||||
// D133 adds a row after the thresholds, D202 the Skipped tags block after it; D203 replaces the template field
|
||||
// with Saved instructions, the two default rows and their note; D134 adds the operation after the actions.
|
||||
tree: [...reference.tree.slice(0, 6), 'div.setting-row', 'div.setting-block', 'div.setting-block', 'div.setting-row',
|
||||
'div.setting-row', 'p.instruction-reset.small', reference.tree[7], 'div.operation'],
|
||||
// D133 adds a row after the thresholds; D134 adds the operation after the actions.
|
||||
tree: [...reference.tree.slice(0, 6), 'div.setting-row', ...reference.tree.slice(6), 'div.operation'],
|
||||
rows: [
|
||||
// D132 (OQ5, OQ6): the four real engines with their download size, and the chosen one's live status.
|
||||
{ ...model, text: [model.text[0], ['p', 'Downloaded. Not loaded.', null]],
|
||||
@@ -157,49 +120,16 @@ function mapTaggingConcept(reference, { builtin, saved, phone }) {
|
||||
// D133: the persisted suggestion count, on the concept's number row.
|
||||
{ ...general, text: [['label', 'Suggestions per image', 'tag-suggestions'], ['p', SUGGESTIONS, null]],
|
||||
control: { ...general.control, id: 'tag-suggestions', value: '10' } },
|
||||
// D203: the default of each language-model engine (the fixture's saved entry is both), on the concept's
|
||||
// select row.
|
||||
defaultRow('gguf', 'GGUF JoyCaption'), defaultRow('hf', 'HuggingFace JoyCaption'),
|
||||
],
|
||||
// D203: Saved instructions in the concept's setting-row tokens (as D202), Built-in selected and read-only.
|
||||
instructions: { tree: ['div.setting-block-text', 'div.instructions-layout'], key: 'autotag_instructions',
|
||||
text: [['label', 'Saved instructions'], ['p', `${INSTRUCTIONS_DESCRIPTION} ${saved.length} of 20 saved · Built\u2011in is not counted.`]],
|
||||
list: [['Built-in', [['Read-only', 'chip']], 'true'],
|
||||
...saved.map((entry, index) => [entry.name, index === 0
|
||||
? [['Default for GGUF', 'chip blue'], ['Default for HuggingFace', 'chip blue']] : [], 'false'])],
|
||||
listRole: 'listbox',
|
||||
actions: [['+ New', '', false], ['Duplicate', '', false], ['Delete', 'danger', true]],
|
||||
editor: [['Name', 'Built-in', true], ['Text', builtin, true]],
|
||||
note: 'Built-in instructions can’t be changed. Duplicate them to make your own.',
|
||||
layout: phone ? 1 : 2,
|
||||
paddingTop: reference.row.paddingTop, paddingBottom: reference.row.paddingBottom,
|
||||
borderTopWidth: reference.row.borderTopWidth, borderTopColor: reference.row.borderTopColor,
|
||||
label: reference.label,
|
||||
textarea: { fontSize: reference.field.textarea.fontSize, backgroundColor: reference.field.textarea.backgroundColor,
|
||||
borderTopColor: reference.field.textarea.borderTopColor } },
|
||||
// D202 (#431): the Skipped tags block in the concept's setting-row tokens: its label and description as a row's,
|
||||
// the row's padding and rule above, the concept's field input; an empty list says so.
|
||||
skipped: { tree: ['div.setting-block-text', 'div.skip-add', 'p.skip-problem.small', 'div.skip-chips.tags', 'p.skip-preview.small'],
|
||||
key: 'autotag_skipped_tags',
|
||||
text: [['label', 'Skipped tags', 'tag-skipped-input'], ['p', SKIPPED_DESCRIPTION, null]],
|
||||
add: [['input', 'tag-skipped-input', 'Add a word or words', ''], ['button', 'tag-skipped-add', null, 'Add']],
|
||||
preview: ['No tags are skipped.', 'status'],
|
||||
paddingTop: reference.row.paddingTop, paddingBottom: reference.row.paddingBottom,
|
||||
borderTopWidth: reference.row.borderTopWidth, borderTopColor: reference.row.borderTopColor,
|
||||
label: reference.label,
|
||||
description: { fontSize: reference.description.fontSize, color: reference.description.color,
|
||||
marginTop: reference.description.marginTop },
|
||||
input: { fontSize: '12px', backgroundColor: reference.field.textarea.backgroundColor, color: reference.field.textarea.color,
|
||||
borderTopColor: reference.field.textarea.borderTopColor } },
|
||||
// D203: no template field.
|
||||
field: null,
|
||||
// The saved template (the server's default until one is saved) replaces the concept's sample text.
|
||||
field: { ...reference.field, textarea: { ...reference.field.textarea, value: template } },
|
||||
// D132: Download (for an engine already on disk, "Download again") and Unload replace the two simulated jobs.
|
||||
actions: { ...reference.actions, buttons: [preview, ['Download again', ''], ['Unload', '']] },
|
||||
// D134: the prompt-tag batch, reviewed first.
|
||||
operations: [{ tree: ['div', 'div.row-actions'], title: 'Tag output images', description: BATCH, actions: [['Review', '']] }],
|
||||
// D132 (#398): every select shows its whole chosen option at every width (at 390 the concept's 120px cap cuts
|
||||
// them; production lets each grow past it).
|
||||
selectsFit: [...reference.selectsFit, true, true].map(() => true),
|
||||
selectsFit: reference.selectsFit.map(() => true),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -223,9 +153,7 @@ async function productionTagging(page, theme) {
|
||||
async function savedTemplate(request, page) {
|
||||
const response = await request.get('/prompt_manager/settings');
|
||||
await expect(response).toBeOK();
|
||||
const { settings } = await response.json();
|
||||
return { builtin: settings.autotag_builtin_instructions, saved: settings.autotag_instructions,
|
||||
phone: page.viewportSize().width <= 700 };
|
||||
return { template: (await response.json()).settings.autotag_template, phone: page.viewportSize().width <= 700 };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
@@ -249,15 +177,14 @@ for (const theme of THEMES) for (const width of [390, 320]) {
|
||||
await page.locator('#workspace-main #tag-existing').selectOption('replace_after_review');
|
||||
await page.locator('#workspace-main section.surface').screenshot({ path: testInfo.outputPath(`production-tagging-selects-${theme}-${width}.png`) });
|
||||
const selects = page.locator('#workspace-main section.surface > .setting-row > select');
|
||||
// D203 adds the two per-engine default selects.
|
||||
await expect(selects).toHaveCount(4);
|
||||
for (let index = 0; index < 4; index += 1) {
|
||||
await expect(selects).toHaveCount(2);
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
const select = selects.nth(index);
|
||||
const values = await select.evaluate(node => [...node.options].map(option => option.value));
|
||||
for (const value of values) {
|
||||
await select.selectOption(value);
|
||||
const measured = await page.evaluate(measureTagging, '#workspace-main');
|
||||
expect(measured.selectsFit, `${await select.getAttribute('id')}=${value}`).toEqual([true, true, true, true]);
|
||||
expect(measured.selectsFit, `${await select.getAttribute('id')}=${value}`).toEqual([true, true]);
|
||||
expect(measured.pageScroll, value).toBe(false);
|
||||
}
|
||||
}
|
||||
@@ -271,16 +198,12 @@ test('auto-tagging comparison detects mutations', async ({ page, request }) => {
|
||||
const expected = mapTaggingConcept(concept, saved);
|
||||
const production = await productionTagging(page, 'dark');
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Unmapped, the section differs only where D132, D133, D134, D202 and D203 are recorded (and the saved template's text).
|
||||
// (The concept has no Skipped tags block, so every path under it is one D202 difference.)
|
||||
expect(diffInventory(concept, production).map(item => (/^(skipped|instructions|field)\b/.test(item.path) ? item.path.split('.')[0]
|
||||
: item.path.split('.').slice(0, 3).join('.')))
|
||||
// Unmapped, the section differs only where D132, D133 and D134 are recorded (and the saved template's text).
|
||||
expect(diffInventory(concept, production).map(item => item.path.split('.').slice(0, 3).join('.'))
|
||||
.filter((path, index, all) => all.indexOf(path) === index).sort()).toEqual([
|
||||
'actions.buttons.1', 'actions.buttons.2', 'field', 'instructions', 'operations.0.actions', 'operations.0.description',
|
||||
'operations.0.title', 'operations.0.tree', 'operations.length', 'rows.0.control', 'rows.0.text',
|
||||
...[4, 5, 6].flatMap(row => ['control', 'controlBelow', 'flexWrap', 'text', 'tree'].map(key => `rows.${row}.${key}`)),
|
||||
'rows.length', 'selectsFit.2', 'selectsFit.3', 'selectsFit.length', 'skipped',
|
||||
...Array.from({ length: 8 }, (_, index) => `tree.${index + 6}`), 'tree.length'].sort());
|
||||
'actions.buttons.1', 'actions.buttons.2', 'field.textarea.value', 'operations.0.actions', 'operations.0.description',
|
||||
'operations.0.title', 'operations.0.tree', 'operations.length', 'rows.0.control', 'rows.0.text', 'rows.4.control', 'rows.4.controlBelow',
|
||||
'rows.4.flexWrap', 'rows.4.text', 'rows.4.tree', 'rows.length', 'tree.6', 'tree.7', 'tree.8', 'tree.9', 'tree.length'].sort());
|
||||
const measure = () => page.evaluate(measureTagging, '#workspace-main');
|
||||
const paths = async () => diffInventory(expected, await measure()).map(item => item.path);
|
||||
// A removed row.
|
||||
@@ -331,8 +254,8 @@ test('auto-tagging at 390 wraps only the engine row', async ({ page, request })
|
||||
const concept = await conceptTagging(page, 'dark');
|
||||
const production = await productionTagging(page, 'dark');
|
||||
expect(diffInventory(mapTaggingConcept(concept, await savedTemplate(request, page)), production)).toEqual([]);
|
||||
expect(production.rows.map(row => row.flexWrap)).toEqual(['wrap', 'nowrap', 'nowrap', 'nowrap', 'nowrap', 'nowrap', 'nowrap']);
|
||||
expect(production.selectsFit).toEqual([true, true, true, true]);
|
||||
expect(production.rows.map(row => row.flexWrap)).toEqual(['wrap', 'nowrap', 'nowrap', 'nowrap', 'nowrap']);
|
||||
expect(production.selectsFit).toEqual([true, true]);
|
||||
});
|
||||
|
||||
/** Runs in the page: the open review dialog: its title, body and the concept's tokens. */
|
||||
|
||||