Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3295ff8b20 | ||
|
|
9640ca86f8 | ||
|
|
8a81114dfc | ||
|
|
7a2a04b752 | ||
|
|
cb94b12c1b | ||
|
|
3cb90b0c1e | ||
|
|
9a43b67008 | ||
|
|
c3c75c3369 | ||
|
|
93bf24f47d | ||
|
|
e29c23de8d | ||
|
|
b82e68a9d0 | ||
|
|
fb02066cda | ||
|
|
ab17818792 | ||
|
|
ef97d01dbd | ||
|
|
b7a43755ae | ||
|
|
2c7e84dedd | ||
|
|
cc65d15d14 | ||
|
|
d88cf9e7c6 | ||
|
|
df70b5dfd6 | ||
|
|
7e1847955b | ||
|
|
ecb6b09813 | ||
|
|
cdb791d4a9 | ||
|
|
f924d7195d | ||
|
|
3789d85e58 | ||
|
|
fb6452934f | ||
|
|
395dc74cbe | ||
|
|
2f061904a2 | ||
|
|
f80e67c812 | ||
|
|
ab6fee3a82 | ||
|
|
f53a43152d | ||
|
|
fe63f01214 | ||
|
|
aab0fedc10 | ||
|
|
085e45e475 | ||
|
|
89572e03c6 |
@@ -1,9 +1,9 @@
|
||||
name: Windows and macOS
|
||||
name: Windows
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Non-blocking portability runs (#420): never on pull requests or in the merge queue, and no context
|
||||
# Non-blocking Windows runs (#420; macOS dropped from CI in #571): never on pull requests or in the merge queue, and no context
|
||||
# here is a required check. A red run is fixed at its cause before the workspace cutover (#334).
|
||||
on:
|
||||
push:
|
||||
@@ -25,10 +25,7 @@ env:
|
||||
# run counts (tests/vendor/ci-workflow.test.mjs recomputes it, #493).
|
||||
# Measured 2026-10-29 on main runs 40935337099 (dd247e3), 40938407252 (dd247e3), 40941240624 (dd247e3):
|
||||
# slowest leg: 7m24s (py3.12 windows) / 7m31s (py3.13 windows) / 7m32s (py3.14 windows) -> 15 min
|
||||
# (windows 4m56s-7m32s, macos 4m39s-6m14s)
|
||||
# native-trash (#499) has no green main runs yet: its probe, run 40949370943 (2026-10-29), took
|
||||
# about 1 minute per macOS leg, so ceil(1.5 x 2) + 3 gives it 6 minutes (step 4). Recalibrate
|
||||
# after three green runs.
|
||||
# (windows 4m56s-7m32s)
|
||||
# native-trash-linux (#543) has no green main runs yet: it installs the dev requirements and runs
|
||||
# one test module, about 1 minute by the same estimate, so ceil(1.5 x 2) + 3 gives it 6 minutes.
|
||||
# Recalibrate after three green runs.
|
||||
@@ -40,7 +37,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [windows-latest, macos-latest]
|
||||
os: [windows-latest]
|
||||
python-version: ["3.12", "3.13", "3.14"]
|
||||
|
||||
steps:
|
||||
@@ -60,7 +57,7 @@ jobs:
|
||||
|
||||
# The runner's TEMP is on C:, the OS disk, where a flush took about 6 ms
|
||||
# and a test database's create-to-delete about 111 ms; on the D: work
|
||||
# disk the same took 0.5 ms and 23 ms (macOS: 19 ms). Tests write their
|
||||
# disk the same took 0.5 ms and 23 ms. Tests write their
|
||||
# databases under TEMP, so it moves to a folder on D: (measured in #487,
|
||||
# run 40915866444). The runner's TEMP was an 8.3 short name
|
||||
# (C:\Users\RUNNER~1\...), so the new one is spelled by its short name too
|
||||
@@ -83,31 +80,6 @@ jobs:
|
||||
PM_TEST_SLOWEST: "40"
|
||||
run: python -m tests.run_suite -v
|
||||
|
||||
# The real Trash on both macOS CPUs (#499 design §10.2). It runs apart from the suite, so a red
|
||||
# suite never skips the native proof.
|
||||
native-trash:
|
||||
name: native trash on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 6
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [macos-latest, macos-15-intel]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Native trash proof
|
||||
timeout-minutes: 4
|
||||
env:
|
||||
PM_NATIVE_TRASH: macos
|
||||
run: python -m unittest tests.test_trash_macos_native tests.test_trash_macos -v
|
||||
|
||||
# The same tests as the fake mount table, on a real ext4 mount used as the output root (#543). A
|
||||
# runner can mount with sudo; the tests run as the normal user, so owner and mode checks are real.
|
||||
native-trash-linux:
|
||||
|
||||
@@ -28,7 +28,7 @@ env:
|
||||
# py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
# 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs.
|
||||
# Windows and macOS moved to portability.yml (#420), which has its own measured limit (#493);
|
||||
# Windows moved to portability.yml (#420; macOS left CI in #571), which has its own measured limit (#493);
|
||||
# this Linux-only matrix keeps 27 minutes until it is re-measured.
|
||||
# real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min
|
||||
# coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min
|
||||
@@ -39,7 +39,7 @@ env:
|
||||
# Pull requests run a fast subset (#420): Python 3.12 on ubuntu-latest, coverage, lint and frontend
|
||||
# modules. The merge queue (merge_group), pushes to main and manual runs add Python 3.13 and 3.14 on
|
||||
# Linux and the real ComfyUI job. "all tests" is the required gate that holds the queue to that set.
|
||||
# Windows and macOS run in portability.yml (push to main, nightly, on demand) and never gate a merge.
|
||||
# Windows runs in portability.yml (push to main, nightly, on demand) and never gate a merge.
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
|
||||
@@ -41,9 +41,13 @@ concurrency:
|
||||
# 7.3m-12.2m), because --shard splits by test count and the slow concept specs sort into shard 1. The page
|
||||
# workflows now run in three shards per browser with the limits unchanged; the per-shard times are
|
||||
# re-measured after three green main runs.
|
||||
# #563: coverage shards reached their 18-minute step limit three times on 2026-10-31 with no failing
|
||||
# test (PR #560 shard 2/3: 814 passed in 17.9m; queue run 41206495451 shard 1/3: 827 passed; PR #564
|
||||
# shard 2/3 twice), against recent green shards of 11m-17m. Coverage now runs in four shards with the
|
||||
# limits unchanged; the per-shard times are re-measured after three green main runs.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/4)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -59,7 +63,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3]
|
||||
shard: [1, 2, 3, 4]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -88,13 +92,13 @@ jobs:
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
# Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests
|
||||
# run on the lightest shard instead.
|
||||
- if: matrix.shard == 3
|
||||
- if: matrix.shard == 4
|
||||
run: npm run --prefix tools/vendor check
|
||||
- if: matrix.shard == 3
|
||||
- if: matrix.shard == 4
|
||||
run: npm run --prefix tools/vendor test
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/4
|
||||
timeout-minutes: 18
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/4
|
||||
- name: Upload raw coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
@@ -136,13 +140,13 @@ jobs:
|
||||
with:
|
||||
pattern: coverage-raw-*
|
||||
path: ${{ runner.temp }}/coverage-raw
|
||||
# coverage:merge only checks that raw files exist; merging two of three shards would report
|
||||
# coverage:merge only checks that raw files exist; merging three of four shards would report
|
||||
# partial coverage, so a missing or empty shard artifact fails here.
|
||||
- name: Verify every shard's raw coverage arrived
|
||||
shell: bash
|
||||
env:
|
||||
STAGED_RAW: ${{ runner.temp }}/coverage-raw
|
||||
EXPECTED_SHARDS: 3
|
||||
EXPECTED_SHARDS: 4
|
||||
run: |
|
||||
found=0
|
||||
for shard in $(seq 1 "$EXPECTED_SHARDS"); do
|
||||
|
||||
@@ -74,14 +74,14 @@ V8 files it writes under `coverage/js/raw`:
|
||||
|
||||
```sh
|
||||
PM_TEST_WORKERS=8 npm run --prefix tools/vendor test:coverage # whole run, one report
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 1/3 # node units, viewer suite, workflows 1/3
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 2/3 # workflows 2/3 (likewise 3/3)
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 1/4 # node units, viewer suite, workflows 1/4
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 2/4 # workflows 2/4 (likewise 3/4, 4/4)
|
||||
npm run --prefix tools/vendor coverage:merge # report and baseline check of every raw file
|
||||
npm run --prefix tools/vendor test:workflows -- --project webkit --shard 2/2
|
||||
```
|
||||
|
||||
`coverage:merge` refuses an empty raw directory; it does not count shards, so run
|
||||
every shard first (CI's merge job checks all three arrived). Measured on a 32-CPU
|
||||
every shard first (CI's merge job checks all four arrived). Measured on a 32-CPU
|
||||
machine in the pinned image (2026-10-21): `test:coverage` takes about 5 minutes
|
||||
with 8 workers and 24 minutes with 1; all four projects, 8 workers, each test 3
|
||||
times (6132 tests), 24 minutes. A CI-sized run (`docker run --cpus=4`,
|
||||
@@ -189,14 +189,14 @@ check, so run it before pushing a new or deleted frontend file instead of
|
||||
waiting for the `frontend coverage` jobs.
|
||||
|
||||
The `viewerjs` check runs on every PR, including backend-only changes. It is an
|
||||
aggregate job. It requires the three `frontend coverage (shard N/3)` jobs and
|
||||
aggregate job. It requires the four `frontend coverage (shard N/4)` jobs and
|
||||
`frontend coverage (merge)` to succeed on every event. The four
|
||||
`page workflows (firefox|webkit, shard N/2)` jobs run only in the merge queue,
|
||||
on `main` and on `workflow_dispatch` (see "Pull requests and the merge queue").
|
||||
`viewerjs` requires them to be skipped on a pull request and to succeed
|
||||
everywhere else. Any other failed, cancelled or skipped job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
|
||||
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3`
|
||||
(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins
|
||||
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/4`
|
||||
(shard 1 also runs the node units and the viewer suite, shard 4 the vendor pins
|
||||
and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3
|
||||
days). The merge job downloads them, fails unless all three arrived with raw
|
||||
files and without clashing names, then runs `coverage:merge` (report plus
|
||||
@@ -261,7 +261,7 @@ the process exits non-zero.
|
||||
job fails anyway and the dump is what matters.
|
||||
- `PM_TEST_SLOWEST=N` (off by default) lists the N slowest tests after the
|
||||
summary, each timed from its `start:` line to the next one like the hang
|
||||
limit. The Windows and macOS jobs (`portability.yml`) set it to 40, so every
|
||||
limit. The Windows jobs (`portability.yml`) set it to 40, so every
|
||||
run shows where its time went (#487). The list prints only when the run
|
||||
ends: a job GitHub cancels or a watchdog exit prints none, so read the
|
||||
timestamps between `start:` lines instead.
|
||||
@@ -327,7 +327,7 @@ Python 3.13/3.14, real ComfyUI and Firefox/WebKit failures first show up in the
|
||||
merge queue, on the merge-group commit rather than the
|
||||
pushed one. Find that run with `gh run list --event merge_group --workflow test.yml`
|
||||
(or `viewerjs-check.yml`) and match its head branch
|
||||
(`gh-readonly-queue/main/pr-<N>-...`). Windows and macOS failures show up only
|
||||
(`gh-readonly-queue/main/pr-<N>-...`). Windows failures show up only
|
||||
after the merge, in `gh run list --workflow portability.yml`.
|
||||
|
||||
Start fixing as soon as a failed log shows the cause, but push the fix only after
|
||||
@@ -345,11 +345,16 @@ and the time left. The steps, in order:
|
||||
monitor's callbacks (Needs review recorder, preview queue, Images listing
|
||||
listener).
|
||||
2. `ImageMonitor.stop_monitoring` works against one 5 s deadline. Closing the
|
||||
image handler joins the image worker; watchdog's `observer.stop()` (not
|
||||
bounded here) and the observer join get what is left, which can be 0 s.
|
||||
3. `close_auto_thumbnails(final=True)` stops the preview worker (30 s) and
|
||||
kills ffmpeg.
|
||||
4. `PromptTracker.shutdown` stops the prompt cleanup thread (2 s).
|
||||
image handler joins the image worker; watchdog's `observer.stop()` and the
|
||||
observer join get what is left, which can be 0 s. `stop()` joins watchdog's
|
||||
emitters without a limit, so it runs on a daemon thread and is waited for
|
||||
only until the deadline; a later stop waits for that same call (#561).
|
||||
3. `close_auto_thumbnails(final=True)` stops the preview worker (30 s),
|
||||
kills ffmpeg and shuts the interactive image pool down.
|
||||
4. `shutdown_job_pool` (`utils/parallel.py`) cancels the long-lived job pool's
|
||||
queued items and joins its threads (5 s; part `workers`, only when a job
|
||||
used the pool, #523).
|
||||
5. `PromptTracker.shutdown` stops the prompt cleanup thread (2 s).
|
||||
|
||||
A failing step, or a second Ctrl+C, is logged with its traceback and does not
|
||||
skip the later steps. A clean run logs one INFO line to PromptManager's log:
|
||||
@@ -361,6 +366,15 @@ stored, with the parts not reached False, and the `atexit` callback is
|
||||
removed: the later steps are skipped, no summary line is logged, and a later
|
||||
call returns that result and runs nothing.
|
||||
|
||||
The observer is started from a short-lived daemon thread, so every thread it
|
||||
creates is a daemon (a thread inherits that flag from the thread that creates
|
||||
it). watchdog 7 on Windows starts a plain `threading.Thread` per watch for its
|
||||
`ReadDirectoryChangesW` reader; created from ComfyUI's main thread it would not
|
||||
be a daemon, and the interpreter waits for every such thread before it runs any
|
||||
`atexit` callback, so the teardown would never start (#561).
|
||||
`tests/test_package_exit.py` runs a child whose observer does both (a plain
|
||||
reader thread, a `stop()` that never returns) and requires its exit.
|
||||
|
||||
ComfyUI never runs aiohttp's `on_cleanup` hooks, because it never calls
|
||||
`runner.cleanup()`. So `atexit` is the live host's path. It runs on Ctrl+C
|
||||
(`main.py` catches `KeyboardInterrupt` and returns) and on a normal exit.
|
||||
@@ -445,12 +459,44 @@ A hot SQLite read that returns many rows aggregates them into one step
|
||||
(`json_group_array`) instead of stepping once per row, so it does not wait for
|
||||
the GIL once per row while ComfyUI generates (#354, `get_image_link_matches`).
|
||||
|
||||
A metadata read never decodes pixels (#523). Pillow's `PngImageFile.text` loads
|
||||
every pixel to reach text chunks after IDAT; the scan (`_extract_comfyui_metadata`)
|
||||
and the image monitor (`ComfyUIMetadataExtractor.extract_metadata`) read PNG text
|
||||
through `utils/png_text.py`, which runs Pillow's own chunk handlers after the
|
||||
first IDAT and seeks over pixel data. The legacy `.text` read is the floor:
|
||||
`tests/test_png_text.py` compares both readers on every image under
|
||||
`tests/fixtures`, a curated corpus (text after IDAT, zTXt, iTXt, split IDAT, bad
|
||||
checksums, truncations, oversized text, APNG) and a seeded fuzz, and asserts that
|
||||
neither reader decodes pixels by patching `ImageFile.load`. The only reads that
|
||||
differ are files whose pixel data is undecodable or cut short: legacy found
|
||||
nothing, the walk keeps the text before the pixels. Animated PNGs still use
|
||||
`.text`.
|
||||
|
||||
Image work runs on PromptManager's own threads (#523). glibc keeps each thread's
|
||||
malloc arena at its high-water mark, so image buffers spread over the default
|
||||
executor's threads stay resident. Interactive image work (view previews behind their
|
||||
gate, workspace card facts, both inspector routes) runs on the API's fixed
|
||||
`pm-image` pool (`IMAGE_POOL_SIZE` 3, `_run_image`, `_map_image`; card facts already
|
||||
cached answer without a trip; while the gate's 3 preview writes hold every thread, a
|
||||
card or inspector read waits for one of them, by design of the merged pool; after the
|
||||
package teardown the pool is not recreated and late calls use the default executor). Jobs (scan metadata and file facts, thumbnail and
|
||||
convert jobs, the legacy gallery generation, capture previews) run through
|
||||
`map_parallel` on one long-lived `pm-worker` pool of Worker threads, reused between
|
||||
jobs; a new Worker threads value replaces it (the teardown also joins a replaced pool
|
||||
still finishing), and a call made from a job thread runs inline. After a scan, thumbnail or convert job, or the legacy gallery generation,
|
||||
`release_free_memory` (`utils/native_memory.py`) calls glibc's `malloc_trim(0)` once;
|
||||
it does nothing on Windows, macOS and musl and never raises. No allocator setting
|
||||
changes. `tests/test_image_pools.py` asserts the thread of each read and write and the
|
||||
trim; `tests/test_utils_parallel.py` the pool's reuse, resize, nesting and shutdown.
|
||||
`tools/perf/memory_drive.py` (manual, Linux, never in CI) measures RSS, heap, threads
|
||||
and arenas under the #523 load against the disposable browser test server.
|
||||
|
||||
The `perf budget` job in `.github/workflows/test.yml` is the reference
|
||||
measurement: `ubuntu-latest`, one Python, no coverage, only the wall-clock tests.
|
||||
It runs on every push to `main` and on `workflow_dispatch`, never on pull
|
||||
requests, because shared-runner timing must not block a merge. Dispatch it on a
|
||||
branch to measure before merging.
|
||||
Shared Windows and macOS runners do not run them; the structural tests cover
|
||||
Shared Windows runners do not run them; the structural tests cover
|
||||
portability. Run it locally with
|
||||
`PROMPTMANAGER_PERF_BUDGET=1 python -m unittest tests.test_api_workspace_images.TestWorkspaceImages.test_filter_budget`
|
||||
and quote the printed p95 in the PR body.
|
||||
@@ -646,10 +692,10 @@ Before changing paths, SQLite, streams, restore or cleanup:
|
||||
ordinary busy timeout; cross-thread handoffs wait on events with generous
|
||||
ceilings, never a short wall clock (#242).
|
||||
- Run native Windows CI early for these changes. Pull requests and the merge
|
||||
queue run only Linux, so start Windows and macOS on the branch with
|
||||
queue run only Linux, so start Windows on the branch with
|
||||
`gh workflow run portability.yml --ref <branch>`.
|
||||
Include a Windows impact and evidence note in the PR. All six Windows/macOS
|
||||
combinations in that run, every Linux leg and every applicable
|
||||
Include a Windows impact and evidence note in the PR. All three Windows
|
||||
legs in that run, every Linux leg and every applicable
|
||||
exact-head check must pass before merge. Portable `ntpath`/`PureWindowsPath`
|
||||
controls do not establish native filesystem, locking or cleanup success.
|
||||
|
||||
@@ -664,11 +710,11 @@ what runs:
|
||||
| `py3.13 on ubuntu-latest`, `py3.14 on ubuntu-latest` | not created | run |
|
||||
| `real ComfyUI compatibility` | skipped | runs |
|
||||
| `coverage gate (85%)`, `black and flake8`, `frontend modules`, `bandit (high severity)` | run | run |
|
||||
| `frontend coverage (shard N/3)`, `frontend coverage (merge)` (vendor pins and tooling tests on shard 3) | run | run |
|
||||
| `frontend coverage (shard N/4)`, `frontend coverage (merge)` (vendor pins and tooling tests on shard 4) | run | run |
|
||||
| `page workflows (firefox\|webkit, shard N/2)` | skipped | run |
|
||||
| `all tests`, `viewerjs` (gates) | run | run |
|
||||
| `perf budget` | skipped | push and dispatch only (not in the queue) |
|
||||
| `py3.12-3.14 on windows-latest/macos-latest` (`portability.yml`) | not run | push to `main`, nightly, `workflow_dispatch` only; never required |
|
||||
| `py3.12-3.14 on windows-latest` (`portability.yml`) | not run | push to `main`, nightly, `workflow_dispatch` only; never required |
|
||||
|
||||
GitHub has one required set for pull requests and the queue. A job skipped by
|
||||
its `if:` reports Success. A dropped matrix leg never reports, so it would stay
|
||||
@@ -681,8 +727,9 @@ guarantee:
|
||||
So a merge group cannot turn green unless all three Linux Python legs, real
|
||||
ComfyUI and the four browser shards passed.
|
||||
|
||||
Windows and macOS (Python 3.12-3.14) are out of the blocking path for now
|
||||
(maintainer, 2026-10-28). They run in `portability.yml` on every push to `main`,
|
||||
Windows (Python 3.12-3.14) is out of the blocking path for now
|
||||
(maintainer, 2026-10-28). macOS is not tested in CI (maintainer, 2026-10-31, #571);
|
||||
its code stays and support is best-effort. They run in `portability.yml` on every push to `main`,
|
||||
nightly and on demand, and no context there is required. A red portability run
|
||||
is still a failure: fix it at its cause, not by rerunning, and have every one
|
||||
green before the workspace cutover (#334). `tests/vendor/ci-workflow.test.mjs` pins the
|
||||
@@ -1407,6 +1454,8 @@ is shorter than the row it reserves.
|
||||
Each Overview card is the count of the page it names: the shell's Library, Images and Tags sidebar counts, one source,
|
||||
so a card and its badge cannot differ. `/prompt_manager/stats` is for the legacy page and counts raw rows. A visit re-reads
|
||||
the three unless a read is in flight. An unread or failed card shows a dash; Retry re-reads only the failed counts.
|
||||
System › Overview's Images metric reads the same shell count (#555), so it, the Overview card and the Images badge show
|
||||
one number; its Statistics dialog labels the legacy row counts as "Database records".
|
||||
Tests: `tests/js/workspace-state.test.js`, `tests/browser/workspace.spec.mjs`, `tests/comfyui/workspace.spec.mjs`.
|
||||
|
||||
### Images page controls and recorded deviations (#249 PR C)
|
||||
@@ -1837,7 +1886,8 @@ Inspect plan keeps D60–D66 (plan D60 → D83, D61 → D84, D65 → D88, D68
|
||||
engines in the mock's order with legacy labels and sizes (`WD14 SwinV2 · ~400
|
||||
MB`, `WD14 ViT · ~400 MB`, `GGUF JoyCaption (Recommended) · ~4.5 GB` in the
|
||||
mock's Ollama slot, `HuggingFace JoyCaption · ~16 GB`); the instructions start
|
||||
from the server's `default_prompt`.
|
||||
from the chosen engine's default saved instructions (#431, D204; before, the
|
||||
server's `default_prompt`).
|
||||
- **D91** (plan D68, ruling T3-bulk): a **Suggestions** number field (1–100) for
|
||||
every model, prefilled from `autotag.defaults.suggestion_limit` (10); a model
|
||||
change keeps it.
|
||||
@@ -2549,8 +2599,8 @@ files to the system trash. **No code path unlinks or overwrites a media file.**
|
||||
("Moving files to the Recycle Bin isn't available on Windows: files larger than the bin's
|
||||
capacity would be deleted permanently."). Library-only removal works everywhere. Tests use `tests/fake_trash.py`; the
|
||||
browser test server installs it, so no test touches a real trash. The native proof,
|
||||
`tests/test_trash_macos_native.py`, runs only with `PM_NATIVE_TRASH=macos` in
|
||||
portability.yml's `native-trash` job (`macos-latest` and `macos-15-intel`): the startup
|
||||
`tests/test_trash_macos_native.py`, runs only with `PM_NATIVE_TRASH=macos` on a Mac, by hand
|
||||
(CI dropped its `native-trash` macOS job in #571; it last passed on `macos-latest` and `macos-15-intel`): the startup
|
||||
volume, a name clash, fresh APFS and FAT32 disk images, an image used as an output root,
|
||||
a read-only image and an unwritable folder (design and probe:
|
||||
`docs/egm/499-macos-trash/`, local).
|
||||
@@ -2929,7 +2979,8 @@ change: ComfyUI needs a restart.
|
||||
A Pair save, by a run or the node's Save pair button, writes the category and tags to the positive prompt only when that
|
||||
save creates the prompt row (maintainer ruling). An existing prompt never changes on a run: empty, cleared or set by a person, including one first seen as
|
||||
another pair's negative. The negative stays empty, the saved pair keeps its own metadata, and `save_prompt_pair` writes
|
||||
no prompt metadata unless called with `fill_new_positive`. No backfill; optional repair is #550.
|
||||
no prompt metadata unless called with `fill_new_positive`. No backfill on a run; the opt-in repair for prompts older
|
||||
Pair runs saved is System › Maintenance › "Pair category and tags" (#550, "Pair category and tags repair" below).
|
||||
Tests: `tests/test_prompt_pair_node.py`, `tests/test_api_pairs.py`, `tests/comfyui/pair-node.spec.mjs`. A Python change: ComfyUI needs a restart.
|
||||
|
||||
### Delete… on the archived shelf (#492)
|
||||
@@ -3844,8 +3895,8 @@ workspace never asks; previews are a cache the server keeps:
|
||||
- `GET /images/serve/thumbnails/<rel>_thumb<ext>` for a missing preview is refused as
|
||||
text first (traversal, a backslash, a drive or stream colon, empty or dot segments, so
|
||||
`D:/x`, `\\server\share` and a leading `/` never reach a joined path), then admitted
|
||||
on the event loop (`_PreviewGate`): 3 previews are written at once on the API's own
|
||||
`pm-preview` pool (never the default executor), one flight runs per file (the history
|
||||
on the event loop (`_PreviewGate`): 3 previews are written at once on the API's
|
||||
interactive `pm-image` pool (never the default executor; #523), one flight runs per file (the history
|
||||
route shares the flight key and the admission), and at most 64 requests wait; one
|
||||
more gets its fallback at once without listing a folder. Inside the admission the
|
||||
preview maps back to its media file under the same root (exact names, then one folder
|
||||
@@ -3878,7 +3929,7 @@ workspace never asks; previews are a cache the server keeps:
|
||||
`exclusive` job runs (`_auto_thumbnail_blocker`), checked when a write starts (after
|
||||
its slot) and again just before its rename.
|
||||
- Shutdown is bounded and logged: the worker stops at a file boundary, ffmpeg is
|
||||
killed, an idle worker leaves (the next output starts a fresh one), the view pool
|
||||
killed, an idle worker leaves (the next output starts a fresh one), the image pool
|
||||
shuts down, and after the app's cleanup a late monitor event
|
||||
starts nothing. A package reload stops the workers the previous import left.
|
||||
- Cards and strips load previews only: Images in every layout and size, Library cards,
|
||||
@@ -4006,7 +4057,11 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
"Save preferences" is `aria-disabled` (opacity .55, `not-allowed`) while clean or
|
||||
saving; a setting the last save could not save gets a red outline. CSS under D92.
|
||||
- **D93** (Overview): real values. The metrics are the Library's eligible positives
|
||||
and active saved negatives and the indexed `generated_images` rows (Q2). "Database
|
||||
and active saved negatives and the Images count (#555, amending Q2's indexed
|
||||
`generated_images` rows): the shell's Images sidebar count, the Images page's own
|
||||
total, which also lists files no row links, as the concept's `images.length` is
|
||||
both. A visit re-reads it unless a read is in flight; "–" until read or after a
|
||||
failed read. `images.indexed` stays in the API and is not shown. "Database
|
||||
integrity" is "Not checked" until a check ran this session, then "N issues" (checks
|
||||
that found issues, even beside a check that could not run), "Passed" or "Incomplete".
|
||||
The review row is "N ambiguous image links" (the Needs review queue's pending count,
|
||||
@@ -4044,10 +4099,11 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
- **D96** (Statistics; amended by D153): "Usage statistics" without the sample note: the
|
||||
concept's "Runs by category" table without its Avg. rating column (Category / Runs)
|
||||
with the real runs of eligible parents per category ("Uncategorized"), then a second
|
||||
table "Library totals" (Total / Count: Prompts, Tags, Images from the legacy
|
||||
statistics; no Rated images). The API still reports `average_rating` and
|
||||
table "Database records" (Total / Count: Prompt records, Tags on prompts, Image
|
||||
records: the raw row counts of the legacy statistics, labelled as rows so they never
|
||||
read as the Library or Images page counts, #555; no Rated images). The API still reports `average_rating` and
|
||||
`rated_images`; the workspace does not show them.
|
||||
The Overview row says "Runs by category, with library totals." because
|
||||
The Overview row says "Runs by category, with database record counts." because
|
||||
that is what the dialog shows (the concept's own dialog shows only "Runs by
|
||||
category"); per-prompt run counts, prompt tags and dated runs are recorded, but no
|
||||
table of them exists yet. A failed read toasts "Statistics could not
|
||||
@@ -4253,6 +4309,12 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
primary stays the concept's plain button; the plan it opens has the primary.
|
||||
`duplicates dialog matches the concept` and `duplicates comparison detects mutations`
|
||||
(system-maintenance-concept.spec) pin the mapping.
|
||||
- **D192** (Maintenance, #550, beyond the concept): Database adds "Pair category and
|
||||
tags" after "Unused prompt components", a plain Review plan row (no danger mark: it
|
||||
deletes nothing) with the line "Fill an empty category or tags on prompts older Pair
|
||||
node runs saved, from their saved pairs. Only empty fields of never-edited prompts."
|
||||
`system-maintenance-concept.spec.mjs` maps it (`mapPairRow`) as a copy of Optimize
|
||||
database's row.
|
||||
- **D102** (Maintenance, S7, Q5, #222): Metadata cache clears only the browser entries
|
||||
Prompt Manager owns as a cache (`web/js/cache-ownership.js`). The inventory of every
|
||||
storage writer has none (see "Browser cache ownership" below), so the row's line adds
|
||||
@@ -4350,6 +4412,23 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
choice. The host's checks still refuse a linked field or a changed workflow, and the
|
||||
sender also refuses if the destination's `revision` (node object plus graph version)
|
||||
moved since discovery; it is undefined on an older tab, which skips the comparison. A host error reads as a refusal, not "may have applied".
|
||||
- **D191** (#542, beyond the concept; maintainer 2026-10-31, Q542-1 A to 5 A): a window with two or more
|
||||
eligible nodes adds a last Destination node option "All N eligible nodes" (`ALL_NODES` in
|
||||
`prompt-sender.mjs`), never selected for you (D173). Eligible is D173's filter; the nodes it leaves out
|
||||
stay counted in the discovery sentence. One window only. Preview stays optional (D175): Preview changes
|
||||
shows one `section.prompt-sender-node` per node, headed "#<id> <title>", in send order (node id,
|
||||
numeric). Apply previews every node first and writes nothing if any refuses ("#<id> <title>: <reason>
|
||||
Nothing was sent."). Then it sends one node after another. Only the first node of a pass taken in that
|
||||
click keeps its ticket; every other node, and every node of a held Preview (host tickets expire after
|
||||
30 s), takes a fresh ticket that is refused unless its Before texts and composition equal the reviewed
|
||||
ones. That fresh preview checks the discovered revision until something has been posted, and not after,
|
||||
since our own write may move the graph version. Refusals are best effort: "Sent to 2 of 3 nodes in
|
||||
<window>. #12 <title>: <host's refusal>."; Apply stays available and a retry sends to every node again
|
||||
(rewriting the same text). A lost reply stops the run, names that node as unknown and lists the nodes
|
||||
not sent. Closing the dialog stops after the current node. The review sections are labelled by their
|
||||
`h4` node heading. Unit tests in `tests/js/prompt-sender.test.js`, a fake three-node host in
|
||||
`tests/browser/prompt-inspector.spec.mjs`, real Classic and Nodes 2.0 widgets in
|
||||
`tests/comfyui/prompt-sender.spec.mjs`.
|
||||
- **D106** (Integrations, S10, Q8): the send bridge always asks for the ComfyUI window
|
||||
and node, and the nodes always follow ComfyUI's own renderer, so "Default send
|
||||
destination" offers only "Ask each time" and "Host compatibility" only "Follow host
|
||||
@@ -4535,9 +4614,9 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
engine and policy (read-only) and opens "Review tag output images" in the concept's
|
||||
record-changing review (its `maintenancePlan` shell): `h3` "Tag output images", the
|
||||
`dl.meta-grid` Model, Existing tags, Images, Already auto-tagged, Images to tag
|
||||
("about N"), Time, Settings ("General <g> · Character <c>" or "Template of N
|
||||
characters", then " · up to L tags per image" with the preview's saved
|
||||
`suggestion_limit`) and Backup, the estimate sentence ("Nothing is written until you
|
||||
("about N"), Time, Settings ("General <g> · Character <c>", then " · up to L tags
|
||||
per image" with the preview's saved `suggestion_limit`; for a language model, since
|
||||
D205, an "Instructions" row and "Up to L tags per image") and Backup, the estimate sentence ("Nothing is written until you
|
||||
start. The number to tag is an estimate: images changed since they were tagged are
|
||||
tagged again, and missing files are skipped. Activity reports what was tagged.") in the
|
||||
`p.small`, the D55 backup box and Cancel / "Start tagging", which starts the
|
||||
@@ -4559,6 +4638,71 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
Cancel-only review matches the concept review shell` (system-tagging-concept.spec) and
|
||||
the batch tests of workspace-system-tagging.spec; the mutation check renames each grid
|
||||
label in turn.
|
||||
- **D202** (Auto-tagging, #431, beyond the concept): a "Skipped tags" block
|
||||
(`.setting-block`, `data-setting="tag-skipped"`, `data-key="autotag_skipped_tags"`)
|
||||
follows "Suggestions per image": the label and description in the concept's
|
||||
setting-row tokens ("Suggested tags that contain one of these words are hidden.
|
||||
Words are separated by spaces, _ or -; other characters such as ( ) belong to the
|
||||
word."), an "Add a word or words" input (`#tag-skipped-input`) with Add (Enter adds
|
||||
too), the entries as removable chips in the Tag image dialog's D83 chip markup (each
|
||||
with its count, title "Hides N tags", or muted "none" with "Hides no known tag"), and
|
||||
a `role="status"` preview line: "**Hides N tags from suggestions**: <up to 12>[ and K
|
||||
more]. Tags already on images are not changed." ("Checked against library tags only;
|
||||
no WD14 label list is downloaded." before the last sentence without a label list;
|
||||
"No tags are skipped." for an empty list; "Checking…" while waiting; "The preview
|
||||
could not be read." on failure). The client refuses an entry with "Enter a word.",
|
||||
"No commas or line breaks.", "At most 50 characters.", "Already in the list." or "The
|
||||
list is full (200)." under the input. The Tag image dialog's chip hint adds "N skipped
|
||||
tags were not suggested." when the server hid any (the static mock's copy). Pinned by
|
||||
the comparison's mapping (system-tagging-concept.spec: the block's tokens are the
|
||||
concept row's), `skipped tags add, preview, remove, save and reload (D202)`, `skipped
|
||||
tags preview checks library tags only …` (workspace-system-tagging.spec) and
|
||||
`suggestions omit skipped tags and say how many were hidden` (workspace-tagging.spec).
|
||||
- **D203** (Auto-tagging, #431, beyond the concept): "Saved instructions" replaces the
|
||||
concept's "LLM tag template" field (the mock's state 2): a `.setting-block`
|
||||
(`data-setting="tag-instructions"`, `data-key="autotag_instructions"`) with the label,
|
||||
"Instructions for the language-model engines. Built-in can’t be changed; duplicate it
|
||||
to make your own. N of 20 saved · Built‑in is not counted.", a `role="listbox"` list
|
||||
(Built-in first with a "Read-only" chip, then the saved entries; "Default for GGUF" /
|
||||
"Default for HuggingFace" chips), "+ New" ("New instructions", "(2)" if taken),
|
||||
"Duplicate" ("<name> copy"), "Delete" (`.danger`, disabled for Built-in), "At most 20
|
||||
saved instructions." at the cap, and the editor beside the list (under it below 700px):
|
||||
"Name" ("N of 60") and "Text" ("N of 4,000 characters"), read-only for Built-in with
|
||||
"Built-in instructions can’t be changed. Duplicate them to make your own.", else the
|
||||
entry's problem ("The name “X” is already used.", "The text must not be empty."…).
|
||||
Then two concept select rows, "Default for GGUF JoyCaption" and "Default for
|
||||
HuggingFace JoyCaption" ("The Tag image dialog and batches start from this.",
|
||||
`data-key="autotag_default_instructions"`), and the note a deleted default leaves
|
||||
("GGUF now uses Built-in."; nothing is written until Save preferences). Pinned by the
|
||||
comparison's mapping (system-tagging-concept.spec: the block in the concept row's
|
||||
tokens, the two rows on the concept's select row) and `saved instructions: new, edit,
|
||||
per-engine default, delete, save and reload (D203)`, `a refused instructions save names
|
||||
the entry and highlights the block (D203)` (workspace-system-tagging.spec).
|
||||
- **D204** (Tag image and Auto-tag selected dialogs, #431, beyond the concept): for a
|
||||
language-model engine a "Saved instructions" select (`#image-tag-instructions-pick`,
|
||||
`#bulk-tag-instructions-pick`) sits above "Tagging instructions" (the mock's state 3),
|
||||
on the concept's select field: Built-in, then the saved entries (`GET /autotag/models`
|
||||
`instructions`, full text), the engine's default marked "<name> · default". It starts
|
||||
at that default; a pick replaces the text. The text stays editable for the run: while
|
||||
it differs from the picked entry the select shows a disabled "Edited for this run" and
|
||||
a muted line under the textarea reads "Edits apply to this run only. Manage saved
|
||||
instructions in System › Auto-tagging." Choosing an entry again replaces the edit.
|
||||
Switching between GGUF and HF starts from the new engine's default and drops an edit;
|
||||
WD14 hides both. A preset (Preview tagging review, the bulk review) shows the entry
|
||||
with its text, else "Edited for this run". When the read fails the select holds only
|
||||
Built-in with the client's copy (`BUILTIN_INSTRUCTIONS`, pinned equal to
|
||||
`DEFAULT_TAG_TEMPLATE` by workspace-image-tagging-state.test). Request bodies are
|
||||
unchanged (`options.instructions` is the textarea's text); there is no "Save as new".
|
||||
The workspace no longer reads `default_prompt`. Pinned by the tagging comparison's
|
||||
mapping (tagging-canonical-comparison.mjs: the select before the instructions),
|
||||
`the saved instructions picker fills, edits one-off and resets per engine (D204)`
|
||||
(workspace-tagging.spec) and `Auto-tag selected picks saved instructions and sends the
|
||||
text shown (D204)` (workspace-tagging-bulk.spec).
|
||||
- **D205** (Auto-tagging, #431; amends D134): for a language-model engine the batch
|
||||
review adds a read-only "Instructions" row with the drafted engine default's name
|
||||
before "Settings", which reads "Up to L tags per image"; the batch and Preview tagging
|
||||
review send that entry's text. WD14 is unchanged. Pinned by `the batch review names the
|
||||
language model's default instructions (D205)` (workspace-system-tagging.spec).
|
||||
- **D158** (Sources, #247 D-Q4): the concept has no switch for prompts from workflows
|
||||
without a PromptManager node; the maintainer chose a visible switch over a config key.
|
||||
A `.setting-row` in the concept's toggle markup follows "Watch for new outputs":
|
||||
@@ -4790,6 +4934,47 @@ replaced the database since the review; the plan is dropped). The job:
|
||||
prompt ids only), logs the backup by its file name, and prunes safety backups to the
|
||||
newest 10. With no backup the log and the status line say "No safety backup".
|
||||
|
||||
#### Pair category and tags repair (#550)
|
||||
|
||||
`pair_metadata` ("Pair category and tags", D192; `database/pair_metadata_repair.py`) fills
|
||||
the empty category and the empty tag set of prompts that Pair node runs before #546
|
||||
saved, from their saved pairs. It is reviewed and applied like the operations above and
|
||||
follows the safety backup policy like them (no forced backup). Rules
|
||||
(`docs/egm/post-3.4.1/550-design.md`):
|
||||
|
||||
- Considered: prompts the Pair node created as a positive: their earliest pair (lowest
|
||||
id, either role, archived included) has them as the positive and its `created_at` is
|
||||
within 2 s of theirs. A prompt first saved by the PromptManager node, or first as a
|
||||
negative, is out of scope and not counted.
|
||||
- Sources: the prompt's non-archived pairs where it is the positive; pairs with a blank
|
||||
category or no tags are ignored for that field. Category values are trimmed and
|
||||
case-sensitive, tag sets compared by tag id; a field fills when exactly one value is
|
||||
left. A whitespace-only category counts as empty.
|
||||
- Kept, per field, in this order: archived prompt, edited prompt (`updated_at` not within
|
||||
2 s of `created_at`; NULL or unreadable counts as edited), pairs that disagree, a tag
|
||||
set over 20 tags. A prompt can fill one field and keep the other.
|
||||
- The plan lists protected rows first, then fills, then a "Record" row, within the 50
|
||||
detail rows; at most 20,000 prompts per plan ("review again for the rest").
|
||||
- The fingerprint covers each considered prompt's computed result (values, tag ids and
|
||||
names, source pair ids), never a pair's title, notes, rating, run count or use
|
||||
stamps, so a Pair run between review and apply still applies.
|
||||
- The apply's writes are guarded (`category` still blank, no `prompt_tags` row) and a
|
||||
guard that hits nothing rolls the whole transaction back as `plan-changed`. It never
|
||||
writes `prompt_pairs`, `prompt_pair_tags`, `updated_at`, a tag or Library state.
|
||||
- The job lists 200 changed rows ("Prompt 12: category “night”; tags a, b"); its
|
||||
Download report lists every one (`truncated: false`, `Job.record_counted(complete=True)`).
|
||||
Protected lists one row per kept prompt in the review's wording and order ("Prompt 7:
|
||||
kept, pairs disagree on category: Adult, asian"), also complete in the report.
|
||||
- Activity shows a row with no path as a record: its detail as plain text, never an open
|
||||
or locate action, and the filter reads the detail. A maintenance job of records (every
|
||||
operation but Missing-file links) says "No records in this group.", "No listed record
|
||||
matches this filter." and "Filter records…" (`groupCopy`); file jobs keep "files".
|
||||
|
||||
Tests: `tests/test_maintenance_pair_metadata.py`, `tests/test_api_maintenance_pair_metadata.py`,
|
||||
`tests/test_jobs.py`, `tests/js/workspace-system-maintenance.test.js`,
|
||||
`tests/browser/workspace-system-maintenance.spec.mjs` (hook `/__test__/system/pair-metadata`).
|
||||
A Python change: ComfyUI needs a restart.
|
||||
|
||||
The legacy restore (`POST /prompt_manager/restore`) takes a backup of any size the disks
|
||||
have room for, so a real library's safety backup restores: the upload, its staged copy
|
||||
and the swap (the upload twice in the temporary folder, once more plus the current
|
||||
@@ -5144,16 +5329,16 @@ server-side tagging configuration, `autotag.defaults` in config.json: `model_typ
|
||||
(`wd14-swinv2`, `wd14-vit`, `gguf`, `hf`), `existing` (`skip`, `merge`,
|
||||
`replace_after_review`), `general_threshold` and `character_threshold` (0–1 in
|
||||
steps of 0.05, #369),
|
||||
`template` (non-blank, at most 4,000 characters and 16 KiB) and `suggestion_limit`
|
||||
(1–100). They are host drafts (`autotag_model_type`, `autotag_existing`,
|
||||
`autotag_general_threshold`, `autotag_character_threshold`, `autotag_template`,
|
||||
`autotag_suggestion_limit`) saved by the one Save row through `POST
|
||||
`suggestion_limit` (1–100), and since #431 `skipped_tags`, `instructions` and
|
||||
`default_instructions` (below). They are host drafts (`autotag_model_type`, `autotag_existing`,
|
||||
`autotag_general_threshold`, `autotag_character_threshold`, `autotag_suggestion_limit`,
|
||||
`autotag_skipped_tags`, `autotag_instructions`, `autotag_default_instructions`) saved by the one Save row through `POST
|
||||
/prompt_manager/settings` with `expected_revision`; a bad value is refused with a 400
|
||||
naming the setting by its row label ("General-tag confidence must be between 0 and
|
||||
1."), nothing is written, the Save row shows that sentence (`invalid-setting`, set by
|
||||
`settingRefusal` in system.mjs only for a save that carried `autotag_*` keys; every
|
||||
other section's plain 400 keeps the generic status) and the control is highlighted.
|
||||
The template's limit counts code points, as Python's `len` does. A hand-edited bad value
|
||||
Instruction text limits count code points, as Python's `len` does. A hand-edited bad value
|
||||
falls back to its default on load, one key at a time. Every page shows the thresholds in
|
||||
steps of 0.05 (the System rows, the Tag image dialogs, the legacy sliders), so a saved
|
||||
threshold off that grid is refused ("General-tag confidence must be a multiple of
|
||||
@@ -5164,8 +5349,70 @@ stored exactly on the grid (#369). A per-request threshold keeps its own check:
|
||||
`validate_options` and starts the `autotag` job titled "Tag output images (legacy page)";
|
||||
the legacy `/autotag/single` only requires a number. `GET /autotag/models` reports the
|
||||
saved values (`wd14_general_threshold`, `wd14_character_threshold`, `suggestion_limit`,
|
||||
`default_prompt`, `model_type`, `existing`), so the Tag image and Auto-tag selected
|
||||
dialogs start from them; a preset (the bulk review, Preview tagging review) wins.
|
||||
`default_prompt`, `model_type`, `existing`, and since #431 `instructions` and
|
||||
`default_instructions`), so the Tag image and Auto-tag selected dialogs start from them;
|
||||
a preset (the bulk review, Preview tagging review) wins.
|
||||
|
||||
**Saved instructions** (#431, D203, D205): `autotag.defaults.instructions` holds the
|
||||
saved entries `[{id, name, text}]` in order (at most 20; `id` 12 lowercase hex made by
|
||||
the client, unique, `builtin` reserved; `name` trimmed, 1–60 characters, no control
|
||||
character, unique case-folded, not "Built-in"; `text` as `validate_template`), and
|
||||
`autotag.defaults.default_instructions` is `{gguf, hf}`, each `builtin` or a saved id.
|
||||
Built-in (`{id: "builtin", name: "Built-in", text: DEFAULT_TAG_TEMPLATE}`) is never
|
||||
stored; readers put it first (`AutotagConfig.all_instructions`), so a later release's
|
||||
built-in text reaches everyone who uses it. Refusals (400, the Save row shows them and
|
||||
highlights the editor): "Saved instructions: at most 20 entries besides Built-in.",
|
||||
"Saved instructions: entry N has an invalid id.", "… repeats the id of entry M.", "…
|
||||
needs a name.", "… name must be at most 60 characters.", "… name must not contain a line
|
||||
break.", "… name “X” is already used.", "Saved instructions: the name “Built-in” is
|
||||
reserved.", "Saved instructions: entry N text must be …", "Default for GGUF JoyCaption
|
||||
must be one of the saved instructions." (and HuggingFace). A save carrying either key
|
||||
without `expected_revision` is refused ("Reload the page and try again."); a save
|
||||
carrying one is paired with the other from the running configuration and both are
|
||||
written, plus the old `template` key as the GGUF default's text (deleted when that
|
||||
default is Built-in), so an older build reads that default as its one template. `POST`
|
||||
with `autotag_template` is refused ("LLM tag template was replaced by Saved instructions.
|
||||
Reload the page."); `GET` keeps returning it (the GGUF default's text, read-only) for one
|
||||
release and adds `autotag_builtin_instructions` (Built-in's text, read-only). Load: each
|
||||
key alone (a damaged list → Built-in only, a default naming nothing → Built-in, each with
|
||||
one warning), then the in-memory migration of the old `template`: with no `instructions`
|
||||
key, a template other than Built-in becomes "Custom template" (id = first 12 hex of its
|
||||
SHA-256), the default of both engines unless `default_instructions` is stored; with the
|
||||
key, a template matching no entry (edited on an older build) is appended as "Custom
|
||||
template (n)" with one INFO line (a full list: one WARNING, nothing imported). Nothing is
|
||||
written at load; the next save that carries either key writes the list.
|
||||
`AutotagConfig.default_instructions_text(engine)` feeds the legacy batch and single
|
||||
blank-prompt fallbacks (by the request's engine) and `default_prompt`. Tests:
|
||||
`tests/test_saved_instructions.py`, `tests/test_saved_instructions_routes.py`,
|
||||
`tests/js/workspace-system-instructions.test.js`.
|
||||
|
||||
**Skipped tags** (#431, D202): `autotag.defaults.skipped_tags`, host draft
|
||||
`autotag_skipped_tags`, a list of at most 200 entries, each trimmed, 1–50 code points,
|
||||
no comma or control character, at least one word, no two with the same words (refused
|
||||
with "Skipped tags: at most 200 entries.", "Skipped tags: entry N is empty.", "… must be
|
||||
at most 50 characters.", "… must not contain a comma or a line break.", "… repeats entry
|
||||
M."); a save carrying it without `expected_revision` is refused ("Reload the page and try
|
||||
again."). A damaged saved list falls back to none with one warning. Words are the
|
||||
NFC, case-folded text split on runs of space, `_` and `-` (`tag_words`); an entry hides
|
||||
a suggested tag whose words contain the entry's words consecutively (`SkipMatcher`:
|
||||
`blurry` hides `very_blurry` and `blurry_background`, not `unblurry`; `(fate)` hides
|
||||
`saber_(fate)`, `fate` does not). `generate_tag_suggestions(skip=)` drops a match after
|
||||
the name check and before the dedupe and the limit, so skipped tags use up no slot
|
||||
(WD14 fills from lower scores; a language model may return fewer). Every path filters:
|
||||
`/image-annotations/suggest` (compiled per request; the answer adds `skipped`, the
|
||||
number of distinct names it hid), `/autotag/single`, and the `image-tags` (Auto-tag
|
||||
selected) and `autotag` (System batch, legacy `/autotag/start`) jobs, which snapshot the
|
||||
list at their start and log "Skipped tags: N entries" or "Skipped tags: none"; a save
|
||||
during a run does not change it. Hand-typed and stored tags are never filtered; the
|
||||
language models' hard-coded exclude list (`_parse_tags`) runs first. `POST
|
||||
/prompt_manager/autotag/skipped-tags/preview {entries}` (validated as a save) answers
|
||||
`count`, `sample` (12, library tags first, each part sorted case-folded),
|
||||
`sample_limit`, `per_entry [{entry, count}]` and `sources {library, wd14}` over the
|
||||
library's tags plus the category 0/4 names of each downloaded WD14 `selected_tags.csv`
|
||||
(read as UTF-8, cached by mtime and size; a missing list is skipped). It loads no model,
|
||||
opens no image, writes nothing and is `409 busy` while an exclusive job runs. Tests:
|
||||
`tests/test_skipped_tags.py`, `tests/test_skipped_tags_routes.py`, the settings and
|
||||
config tests, `tests/js/workspace-system-skipped-tags.test.js`.
|
||||
|
||||
`GET /prompt_manager/autotag/models` reads the disk in the executor, never loads or
|
||||
downloads a model and carries no absolute path: per engine `name`, `label`,
|
||||
@@ -5413,7 +5660,7 @@ list stay unscoped: any row's control proves them.
|
||||
| Worker threads | `admin.html:437–445` | S4 Sources | covered | `sources` · slider · "Worker threads" | worker range max is cpu_count (workspace-system-sources.spec.mjs) |
|
||||
| LoraManager badge, enable, path, trigger words, CivitAI key, clear key | `admin.html:448–490`; `admin.js:1052–1156` | S10 Integrations | covered | `integrations` · switch · "LoraManager integration" | LoRA rows show detected / enabled / cache-loaded separately (workspace-system-integrations.spec.mjs) |
|
||||
| LoRA data import with progress | `admin.html:492–498,517–541`; `admin.js:1157` | S10 Integrations | covered | `integrations` · button · "Review import" | Review import shows replacement, blocked items and network effects with the backup box; Start opens the job (D107) (workspace-system-integrations.spec.mjs) |
|
||||
| Statistics | `admin.html:159`; `admin.js:1627` | S2 Overview | covered | `overview` · button · "Explore" | statistics dialog lists runs by category and library totals (workspace-system.spec.mjs) |
|
||||
| Statistics | `admin.html:159`; `admin.js:1627` | S2 Overview | covered | `overview` · button · "Explore" | statistics dialog lists runs by category and database record counts (workspace-system.spec.mjs) |
|
||||
| Export components JSON `/export` | `admin.html:163`; `admin.js:1607`; `py/api/prompts.py:238` | S9 Backup & export | covered | `backups` · button · "Export all prompts JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Saved pairs export | `py/api/pairs.py:84` | S9 Backup & export | covered | `backups` · button · "Export saved pairs JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Library export | `py/api/library.py:223` | S9 Backup & export | covered | `backups` · button · "Export JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
@@ -6797,7 +7044,7 @@ checkout or point this harness at a live ComfyUI checkout.
|
||||
|
||||
The package-qualified config import also corrects #211: the configured database
|
||||
path is honored when PromptManager runs as a ComfyUI custom node. Follow the
|
||||
[database path compatibility and safe backup sequence](README.md#database-path-compatibility-211)
|
||||
[database path compatibility and safe backup sequence](README-legacy.md#database-path-compatibility-211)
|
||||
before restarting an existing deployment.
|
||||
|
||||
The ledger records only new dequeued jobs and observed outcomes. Existing
|
||||
|
||||
@@ -120,6 +120,7 @@ except Exception as e:
|
||||
logger = get_logger("prompt_manager.init")
|
||||
logger.error(f"Failed to register API routes: {e}")
|
||||
except Exception:
|
||||
# logging failed too; a report must never stop the node loading
|
||||
pass
|
||||
|
||||
# Shared database, set up with the monitoring system below. Stays None when that
|
||||
@@ -279,11 +280,11 @@ def _teardown_logger():
|
||||
|
||||
|
||||
def _stop_background_work():
|
||||
"""Stop the image monitor, the preview worker and the prompt cleanup
|
||||
thread against one TEARDOWN_SECONDS deadline, once.
|
||||
"""Stop the image monitor, the preview worker, the job worker threads
|
||||
and the prompt cleanup thread against one TEARDOWN_SECONDS deadline, once.
|
||||
|
||||
Returns ``{"monitor", "thumbnails", "tracker"}`` booleans (True: it
|
||||
stopped), leaving out parts that never started. Later calls return a
|
||||
Returns ``{"monitor", "thumbnails", "workers", "tracker"}`` booleans
|
||||
(True: it stopped), leaving out parts that never started. Later calls return a
|
||||
copy of the first result. A Ctrl+C inside a step is logged and the
|
||||
step is False; one between steps is re-raised after the result (parts
|
||||
not reached are False) is stored and the ``atexit`` entry removed.
|
||||
@@ -297,11 +298,14 @@ def _stop_background_work():
|
||||
api,
|
||||
_global_prompt_tracker,
|
||||
)
|
||||
from .utils import parallel
|
||||
|
||||
# A part whose global is set has started; False until its step says
|
||||
# otherwise, so an interrupt between steps leaves an honest result.
|
||||
started = (
|
||||
("monitor", monitor),
|
||||
("thumbnails", thumbnails),
|
||||
("workers", parallel.job_pool_started() or None),
|
||||
("tracker", tracker),
|
||||
)
|
||||
result = {name: False for name, part in started if part is not None}
|
||||
@@ -344,6 +348,16 @@ def _stop_background_work():
|
||||
result["thumbnails"] = _teardown_step(
|
||||
"the preview worker", close_previews, logger
|
||||
)
|
||||
if "workers" in result:
|
||||
|
||||
def stop_workers():
|
||||
from .utils.parallel import JOB_POOL_JOIN_SECONDS
|
||||
|
||||
return parallel.shutdown_job_pool(left(JOB_POOL_JOIN_SECONDS))
|
||||
|
||||
result["workers"] = _teardown_step(
|
||||
"the job worker threads", stop_workers, logger
|
||||
)
|
||||
if tracker is not None:
|
||||
|
||||
def stop_tracker():
|
||||
@@ -411,6 +425,7 @@ except Exception as e:
|
||||
f"Failed to register the exit signal: {e}"
|
||||
)
|
||||
except Exception:
|
||||
# logging failed too; a report must never stop the node loading
|
||||
pass
|
||||
|
||||
try:
|
||||
@@ -428,16 +443,14 @@ except Exception as e:
|
||||
f"Failed to register the background work teardown: {e}"
|
||||
)
|
||||
except Exception:
|
||||
# logging failed too; a report must never stop the node loading
|
||||
pass
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
||||
|
||||
# Print startup message with loaded tools
|
||||
print()
|
||||
print(f"\033[94m[ComfyUI-PromptManager] Version:\033[0m {get_version()}")
|
||||
for node_key, display_name in NODE_DISPLAY_NAME_MAPPINGS.items():
|
||||
# ASCII only: a Windows console using a legacy code page cannot print emoji,
|
||||
# and a UnicodeEncodeError here would abort loading the whole node.
|
||||
print(f" \033[94mLoaded:\033[0m {display_name}")
|
||||
print(f"\033[94mTotal: {len(NODE_CLASS_MAPPINGS)} tools loaded\033[0m")
|
||||
print()
|
||||
# One startup line. ASCII only: a Windows console using a legacy code page
|
||||
# cannot print emoji, and a UnicodeEncodeError here would abort loading the node.
|
||||
print(
|
||||
f"\033[94m[ComfyUI-PromptManager] Version:\033[0m {get_version()}"
|
||||
f" - {len(NODE_CLASS_MAPPINGS)} tools loaded"
|
||||
)
|
||||
|
||||
|
After Width: | Height: | Size: 35 KiB |
|
After Width: | Height: | Size: 88 KiB |
|
After Width: | Height: | Size: 153 KiB |
|
After Width: | Height: | Size: 82 KiB |
|
After Width: | Height: | Size: 72 KiB |
|
After Width: | Height: | Size: 156 KiB |
|
After Width: | Height: | Size: 159 KiB |
|
After Width: | Height: | Size: 96 KiB |
|
After Width: | Height: | Size: 165 KiB |
|
After Width: | Height: | Size: 60 KiB |
|
After Width: | Height: | Size: 38 KiB |
|
After Width: | Height: | Size: 41 KiB |
|
After Width: | Height: | Size: 42 KiB |
|
After Width: | Height: | Size: 39 KiB |
|
After Width: | Height: | Size: 42 KiB |
|
After Width: | Height: | Size: 40 KiB |
|
After Width: | Height: | Size: 41 KiB |
|
After Width: | Height: | Size: 43 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 237 KiB |
|
After Width: | Height: | Size: 66 KiB |
|
After Width: | Height: | Size: 53 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 181 KiB |
|
After Width: | Height: | Size: 51 KiB |
|
After Width: | Height: | Size: 110 KiB |
@@ -590,6 +590,7 @@ class _Move:
|
||||
try:
|
||||
os.rmdir(self.plan.target_dir)
|
||||
except OSError:
|
||||
# not empty or already gone: the undo leaves it alone
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
Four operations, each the native owner of a legacy ``/maintenance`` one and
|
||||
built on the same queries, so a preview counts what the legacy cleanup would
|
||||
change:
|
||||
change (``workflow_storage`` and #550's ``pair_metadata``, which fills old
|
||||
Pair prompts' empty category and tags, live in their own modules):
|
||||
|
||||
- ``duplicate_prompts``: prompts whose text is equal ignoring case and outer
|
||||
spaces (``DUPLICATE_PROMPTS_SQL``, legacy ``cleanup_duplicates``); each
|
||||
@@ -60,6 +61,9 @@ MAINTENANCE_OPERATIONS = (
|
||||
"missing_links",
|
||||
# Applied as its own batched job (database/workflow_storage.py).
|
||||
"workflow_storage",
|
||||
# #550: fill old Pair prompts' empty category and tags
|
||||
# (database/pair_metadata_repair.py).
|
||||
"pair_metadata",
|
||||
)
|
||||
MAX_DETAILS = 50 # detail rows a plan lists
|
||||
MAX_RESULT_ROWS = 200 # changed records an apply describes (the job lists 200)
|
||||
@@ -533,6 +537,10 @@ class MaintenanceOperations:
|
||||
lambda conn: (self._link_rows(conn), _mark_rows(conn)),
|
||||
)
|
||||
return self._plan_missing_links(rows, roots, _stale_marks(marks, roots))
|
||||
if operation == "pair_metadata":
|
||||
from .pair_metadata_repair import plan_pair_metadata
|
||||
|
||||
return _read_snapshot(self.model.db_path, plan_pair_metadata)
|
||||
reader = (
|
||||
self._plan_duplicate_prompts
|
||||
if operation == "duplicate_prompts"
|
||||
@@ -800,7 +808,8 @@ class MaintenanceOperations:
|
||||
records changed and up to ``MAX_RESULT_ROWS`` descriptions of them
|
||||
(``{prompt_id[, kept_id]}`` or ``{image_path}``). ``marks`` are a
|
||||
missing-links plan's stale removal marks; that apply also returns
|
||||
``marks``, how many it forgot.
|
||||
``marks``, how many it forgot. ``pair_metadata`` returns every filled
|
||||
prompt, uncapped (its job keeps the complete record, #550).
|
||||
"""
|
||||
if operation not in MAINTENANCE_OPERATIONS or operation == "workflow_storage":
|
||||
raise ValueError(f"Unknown maintenance operation {operation!r}")
|
||||
@@ -815,6 +824,10 @@ class MaintenanceOperations:
|
||||
"unused_components": self._apply_unused_components,
|
||||
}
|
||||
with self._pair_transaction() as conn:
|
||||
if operation == "pair_metadata":
|
||||
from .pair_metadata_repair import apply_pair_metadata
|
||||
|
||||
return apply_pair_metadata(conn, fingerprint, targets)
|
||||
if operation == "missing_links":
|
||||
return self._apply_missing_links(
|
||||
conn, fingerprint, targets, roots, list(marks)
|
||||
|
||||
@@ -35,38 +35,51 @@ from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import operator
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
from typing import Iterable, Iterator, List, Optional
|
||||
from collections import deque
|
||||
from typing import Any, Callable, Iterable, Iterator, List, Optional
|
||||
|
||||
import sqlite3
|
||||
|
||||
CLOSED_MESSAGE = "Cannot operate on a closed database."
|
||||
# Rows fetched per step when a cursor is iterated: the connection is marked
|
||||
# in use once per chunk instead of once per row (iteration costs about what
|
||||
# a plain cursor costs). A loop that stops early leaves up to this many rows
|
||||
# fetched; ``fetchone``/``fetchmany`` after a partial loop skip them.
|
||||
# a plain cursor costs). Prefetched rows belong to the cursor so iteration
|
||||
# and explicit fetches always advance the same position.
|
||||
ITER_CHUNK = 128
|
||||
|
||||
|
||||
class ManagedCursor(sqlite3.Cursor):
|
||||
"""A cursor whose calls into SQLite count as its connection being in use."""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._pending = deque()
|
||||
self._closed = False
|
||||
|
||||
def execute(self, sql, parameters=(), /):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().execute(sql, parameters)
|
||||
finally:
|
||||
conn._leave()
|
||||
return self._run_statement(super().execute, sql, parameters)
|
||||
|
||||
def executemany(self, sql, seq_of_parameters, /):
|
||||
return self._run_statement(super().executemany, sql, seq_of_parameters)
|
||||
|
||||
def _run_statement(self, operation: Callable, *args: Any) -> ManagedCursor:
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
description = self.description
|
||||
completed = False
|
||||
try:
|
||||
return super().executemany(sql, seq_of_parameters)
|
||||
result = operation(*args)
|
||||
completed = True
|
||||
return result
|
||||
finally:
|
||||
# Argument parsing can fail before SQLite replaces the old statement.
|
||||
if completed or self.description is not description:
|
||||
self._pending.clear()
|
||||
conn._leave()
|
||||
|
||||
def executescript(self, sql_script, /):
|
||||
@@ -81,6 +94,8 @@ class ManagedCursor(sqlite3.Cursor):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
if self._pending and not self._closed:
|
||||
return self._pending.popleft()
|
||||
return super().fetchone()
|
||||
finally:
|
||||
conn._leave()
|
||||
@@ -89,7 +104,20 @@ class ManagedCursor(sqlite3.Cursor):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().fetchmany(*args, **kwargs)
|
||||
# Let SQLite validate the original arguments, including version-specific
|
||||
# handling of zero/negative sizes, before touching prefetched rows.
|
||||
rows = super().fetchmany(*args, **kwargs)
|
||||
if not self._pending:
|
||||
return rows
|
||||
size = operator.index(
|
||||
kwargs.get("size", args[0] if args else self.arraysize)
|
||||
)
|
||||
# Python 3.12 reads all rows for size=0; 3.13+ reads none.
|
||||
if size == 0 and sys.version_info >= (3, 13):
|
||||
return rows
|
||||
self._pending.extend(rows)
|
||||
count = len(self._pending) if size <= 0 else min(size, len(self._pending))
|
||||
return [self._pending.popleft() for _ in range(count)]
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
@@ -97,25 +125,31 @@ class ManagedCursor(sqlite3.Cursor):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().fetchall()
|
||||
rows = super().fetchall()
|
||||
pending = list(self._pending)
|
||||
self._pending.clear()
|
||||
return pending + rows
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def __iter__(self) -> Iterator:
|
||||
return self._chunks()
|
||||
|
||||
def _chunks(self) -> Iterator:
|
||||
while True:
|
||||
rows = self.fetchmany(ITER_CHUNK)
|
||||
if not rows:
|
||||
return
|
||||
yield from rows
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
conn = self.connection
|
||||
pending = self._pending
|
||||
# Buffered Python rows need no SQLite lock, but retirement and cursor
|
||||
# close must still stop an iterator obtained before either operation.
|
||||
if conn._retired or self._closed:
|
||||
raise sqlite3.ProgrammingError(CLOSED_MESSAGE)
|
||||
if pending:
|
||||
return pending.popleft()
|
||||
conn._enter()
|
||||
try:
|
||||
return super().__next__()
|
||||
pending.extend(super().fetchmany(ITER_CHUNK))
|
||||
if not pending:
|
||||
raise StopIteration
|
||||
return pending.popleft()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
@@ -126,7 +160,10 @@ class ManagedCursor(sqlite3.Cursor):
|
||||
except sqlite3.ProgrammingError:
|
||||
return super().close() # the connection is closed or closing
|
||||
try:
|
||||
return super().close()
|
||||
result = super().close()
|
||||
self._pending = deque()
|
||||
self._closed = True
|
||||
return result
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
@@ -293,6 +330,9 @@ class ManagedConnection(sqlite3.Connection):
|
||||
cursors = list(self._cursors)
|
||||
try:
|
||||
for cursor in cursors:
|
||||
if isinstance(cursor, ManagedCursor):
|
||||
cursor._pending = deque()
|
||||
cursor._closed = True
|
||||
with contextlib.suppress(sqlite3.Error):
|
||||
sqlite3.Cursor.close(cursor)
|
||||
super().close()
|
||||
|
||||
@@ -34,7 +34,7 @@ from .history_reads import HistoryReadOperations
|
||||
from .health import HealthOperations, iter_consistency_issues
|
||||
from .duplicate_copies import DuplicateCopyOperations
|
||||
from .workflow_storage import WorkflowStorageOperations
|
||||
from .category_import import CategoryImportOperations, EmptyReplacement # noqa: F401
|
||||
from .category_import import CategoryImportOperations
|
||||
from .image_categories import ImageCategoryOperations
|
||||
from .workflow_documents import (
|
||||
DocumentReader,
|
||||
|
||||
@@ -0,0 +1,452 @@
|
||||
"""Pair category and tags repair (#550): the ``pair_metadata`` maintenance plan.
|
||||
|
||||
Before #546 a Pair node run saved its category and tags on the saved pair
|
||||
only; the positive prompt it created stayed Uncategorized and untagged. This
|
||||
plan fills the **empty** category and the **empty** tag set of those prompts
|
||||
from their saved pairs (``docs/egm/post-3.4.1/550-design.md``):
|
||||
|
||||
- Considered: prompts the Pair node created as a positive (their earliest
|
||||
pair, in either role, has them as the positive and its ``created_at`` is
|
||||
within ``PAIR_CREATED_TOLERANCE`` of theirs) whose category is blank
|
||||
and/or that have no tags.
|
||||
- Sources: the prompt's non-archived pairs where it is the positive. Pairs
|
||||
with a blank category or no tags are ignored for that field. A field fills
|
||||
when exactly one value remains (category: trimmed, case-sensitive; tags:
|
||||
the tag-id set).
|
||||
- Kept (protected), in this order: archived prompt, edited prompt
|
||||
(``updated_at`` not within the tolerance of ``created_at``), pairs that
|
||||
disagree, a tag set over ``MAX_LIBRARY_TAGS``.
|
||||
|
||||
The apply runs in the caller's ``BEGIN IMMEDIATE`` transaction, recomputes
|
||||
the fingerprint, then writes guarded statements (the field must still be
|
||||
empty) that never touch a pair, a negative or ``updated_at``. A guard that
|
||||
hits nothing raises ``PlanChanged`` so the whole transaction rolls back.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
|
||||
from . import maintenance as _maintenance
|
||||
from .library_commands import MAX_LIBRARY_TAGS
|
||||
from .maintenance import (
|
||||
MAX_DETAILS,
|
||||
MaintenanceDraft,
|
||||
PlanChanged,
|
||||
_excerpt,
|
||||
_Fingerprint,
|
||||
_marks,
|
||||
_plural,
|
||||
)
|
||||
|
||||
OPERATION = "pair_metadata"
|
||||
# Seconds: the Pair save stamps its new prompt and its pair with separate
|
||||
# utc_now_iso() calls in one transaction (microseconds apart).
|
||||
PAIR_CREATED_TOLERANCE = 2.0
|
||||
# Fill prompts one plan applies; the rest are applied by the next review.
|
||||
# Bounds the complete record a job keeps in memory (about 100 bytes a row).
|
||||
MAX_PLAN_FILLS = 20000
|
||||
DETAIL_VALUES = 3
|
||||
DETAIL_TAGS = 5
|
||||
MEDIA_LABEL = "No image file or saved pair is changed."
|
||||
EMPTY_LABEL = "No Pair prompts are missing a category or tags"
|
||||
RECORD_NOTE = (
|
||||
"The job's report lists every filled field. Download it to keep it: "
|
||||
"Activity forgets jobs after a restart."
|
||||
)
|
||||
# Protected classes in precedence order: (key, detail, reason one, many).
|
||||
EDITED_DETAIL = (
|
||||
"Kept: edited after it was saved; its empty category or tags may be intentional"
|
||||
)
|
||||
PROTECTED = (
|
||||
(
|
||||
"archived",
|
||||
"Kept: archived in the Library",
|
||||
"archived in the Library",
|
||||
"archived in the Library",
|
||||
),
|
||||
(
|
||||
"edited",
|
||||
EDITED_DETAIL,
|
||||
"edited after it was saved",
|
||||
"edited after they were saved",
|
||||
),
|
||||
("conflict", None, "where pairs disagree", "where pairs disagree"),
|
||||
(
|
||||
"too-many-tags",
|
||||
f"Kept: its pair has more than {MAX_LIBRARY_TAGS} tags",
|
||||
f"whose pair has more than {MAX_LIBRARY_TAGS} tags",
|
||||
f"whose pair has more than {MAX_LIBRARY_TAGS} tags",
|
||||
),
|
||||
)
|
||||
_PROTECTED_ORDER = [key for key, *_rest in PROTECTED]
|
||||
# Detail rows: what the user can act on (a conflict) first.
|
||||
DETAIL_ORDER = ("conflict", "edited", "archived", "too-many-tags")
|
||||
|
||||
_CANDIDATES_SQL = """
|
||||
SELECT p.id, p.hash, p.text, p.created_at, p.updated_at,
|
||||
COALESCE(TRIM(p.category), '') = '' AS blank_category,
|
||||
NOT EXISTS (SELECT 1 FROM prompt_tags pt WHERE pt.prompt_id = p.id)
|
||||
AS untagged,
|
||||
EXISTS (SELECT 1 FROM prompt_archive pa WHERE pa.prompt_id = p.id)
|
||||
AS archived,
|
||||
(SELECT CASE WHEN pp.positive_prompt_id = p.id
|
||||
THEN pp.created_at END
|
||||
FROM prompt_pairs pp
|
||||
WHERE pp.positive_prompt_id = p.id OR pp.negative_prompt_id = p.id
|
||||
ORDER BY pp.id LIMIT 1) AS first_pair_at
|
||||
FROM prompts p
|
||||
WHERE EXISTS (SELECT 1 FROM prompt_pairs pp WHERE pp.positive_prompt_id = p.id)
|
||||
AND (COALESCE(TRIM(p.category), '') = ''
|
||||
OR NOT EXISTS (SELECT 1 FROM prompt_tags pt WHERE pt.prompt_id = p.id))
|
||||
ORDER BY p.id
|
||||
"""
|
||||
_SOURCES_SQL = (
|
||||
"SELECT id, positive_prompt_id, TRIM(COALESCE(category, '')) FROM prompt_pairs "
|
||||
"WHERE archived = 0 AND positive_prompt_id IN ({marks}) ORDER BY id"
|
||||
)
|
||||
_PAIR_TAGS_SQL = (
|
||||
"SELECT ppt.pair_id, t.id, t.name FROM prompt_pair_tags ppt "
|
||||
"JOIN tags t ON t.id = ppt.tag_id WHERE ppt.pair_id IN ({marks}) "
|
||||
"ORDER BY ppt.pair_id, t.id"
|
||||
)
|
||||
|
||||
|
||||
def parse_stamp(value):
|
||||
"""A stored timestamp as an aware UTC datetime, or None when unreadable."""
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
try:
|
||||
stamp = datetime.datetime.fromisoformat(value.strip())
|
||||
except ValueError:
|
||||
return None
|
||||
if stamp.tzinfo is None:
|
||||
stamp = stamp.replace(tzinfo=datetime.timezone.utc)
|
||||
return stamp
|
||||
|
||||
|
||||
def _close(first, second):
|
||||
"""Whether two stored stamps parse and lie within the tolerance."""
|
||||
a, b = parse_stamp(first), parse_stamp(second)
|
||||
if a is None or b is None:
|
||||
return False
|
||||
return abs((a - b).total_seconds()) <= PAIR_CREATED_TOLERANCE
|
||||
|
||||
|
||||
def _batches(ids):
|
||||
size = _maintenance.ID_BATCH
|
||||
for start in range(0, len(ids), size):
|
||||
yield ids[start : start + size]
|
||||
|
||||
|
||||
def _sources(conn, ids):
|
||||
"""``{prompt_id: [(pair_id, category, ((tag_id, name), ...)), ...]}``."""
|
||||
pairs = {}
|
||||
for batch in _batches(ids):
|
||||
for pair_id, prompt_id, category in conn.execute(
|
||||
_SOURCES_SQL.format(marks=_marks(batch)), batch
|
||||
):
|
||||
pairs.setdefault(prompt_id, []).append([pair_id, category, []])
|
||||
by_pair = {pair[0]: pair for group in pairs.values() for pair in group}
|
||||
pair_ids = sorted(by_pair)
|
||||
for batch in _batches(pair_ids):
|
||||
for pair_id, tag_id, name in conn.execute(
|
||||
_PAIR_TAGS_SQL.format(marks=_marks(batch)), batch
|
||||
):
|
||||
by_pair[pair_id][2].append((tag_id, name))
|
||||
return {
|
||||
prompt_id: [(pid, category, tuple(tags)) for pid, category, tags in group]
|
||||
for prompt_id, group in pairs.items()
|
||||
}
|
||||
|
||||
|
||||
def _field(values, guard):
|
||||
"""``(class, value)`` of one field from its distinct values, or None."""
|
||||
if not values:
|
||||
return None
|
||||
if guard:
|
||||
return (guard, None)
|
||||
if len(values) > 1:
|
||||
return ("conflict", sorted(values))
|
||||
return ("fill", next(iter(values)))
|
||||
|
||||
|
||||
def _classify(row, sources):
|
||||
"""The result of one considered prompt, or None when nothing applies."""
|
||||
guard = None
|
||||
if row["archived"]:
|
||||
guard = "archived"
|
||||
elif not _close(row["updated_at"], row["created_at"]):
|
||||
guard = "edited"
|
||||
category = tags = None
|
||||
tag_pair = None
|
||||
if row["blank_category"]:
|
||||
category = _field({c for _p, c, _t in sources if c}, guard)
|
||||
if row["untagged"]:
|
||||
sets = {}
|
||||
for pair_id, _category, pair_tags in sources:
|
||||
if pair_tags:
|
||||
sets.setdefault(frozenset(pair_tags), pair_id)
|
||||
tags = _field(set(sets), guard)
|
||||
if tags and tags[0] == "fill":
|
||||
tag_pair = sets[tags[1]]
|
||||
if len(tags[1]) > MAX_LIBRARY_TAGS:
|
||||
tags = ("too-many-tags", None)
|
||||
else:
|
||||
tags = ("fill", tuple(sorted(tags[1])))
|
||||
elif tags and tags[0] == "conflict":
|
||||
tags = ("conflict", [sorted(s) for s in sorted(sets, key=sorted)])
|
||||
if category is None and tags is None:
|
||||
return None
|
||||
return {
|
||||
"id": row["id"],
|
||||
"hash": row["hash"],
|
||||
"text": row["text"],
|
||||
"category": category,
|
||||
"tags": tags,
|
||||
"tag_pair": tag_pair if tags and tags[0] == "fill" else None,
|
||||
"pairs": [pair_id for pair_id, _c, _t in sources],
|
||||
}
|
||||
|
||||
|
||||
def _state(conn):
|
||||
"""``(results, fingerprint)``: every considered prompt's result, by id."""
|
||||
considered = []
|
||||
for row in conn.execute(_CANDIDATES_SQL):
|
||||
row = dict(
|
||||
zip(
|
||||
(
|
||||
"id",
|
||||
"hash",
|
||||
"text",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"blank_category",
|
||||
"untagged",
|
||||
"archived",
|
||||
"first_pair_at",
|
||||
),
|
||||
tuple(row),
|
||||
)
|
||||
)
|
||||
if _close(row["created_at"], row["first_pair_at"]):
|
||||
considered.append(row)
|
||||
sources = _sources(conn, [row["id"] for row in considered])
|
||||
fingerprint = _Fingerprint(OPERATION)
|
||||
results = []
|
||||
for row in considered:
|
||||
result = _classify(row, sources.get(row["id"], []))
|
||||
if result is None:
|
||||
continue
|
||||
results.append(result)
|
||||
fingerprint.add(
|
||||
[
|
||||
result["id"],
|
||||
result["hash"],
|
||||
result["category"],
|
||||
result["tags"],
|
||||
result["tag_pair"],
|
||||
result["pairs"],
|
||||
]
|
||||
)
|
||||
return results, fingerprint.hexdigest()
|
||||
|
||||
|
||||
def _fills(result):
|
||||
return (
|
||||
bool(result["category"] and result["category"][0] == "fill"),
|
||||
bool(result["tags"] and result["tags"][0] == "fill"),
|
||||
)
|
||||
|
||||
|
||||
def _targets(results):
|
||||
targets = []
|
||||
for result in results:
|
||||
category, tags = _fills(result)
|
||||
if category or tags:
|
||||
targets.append([result["id"], category, tags])
|
||||
return targets[:MAX_PLAN_FILLS]
|
||||
|
||||
|
||||
def _protected_classes(result):
|
||||
"""The protected classes of a result's fields, in precedence order."""
|
||||
found = {
|
||||
field[0]
|
||||
for field in (result["category"], result["tags"])
|
||||
if field and field[0] != "fill"
|
||||
}
|
||||
return [key for key in _PROTECTED_ORDER if key in found]
|
||||
|
||||
|
||||
def _more(values, limit):
|
||||
shown = ", ".join(values[:limit])
|
||||
return shown + (", …" if len(values) > limit else "")
|
||||
|
||||
|
||||
def _fill_detail(result):
|
||||
parts = []
|
||||
category, tags = _fills(result)
|
||||
if category:
|
||||
parts.append(f"category “{result['category'][1]}”")
|
||||
if tags:
|
||||
names = sorted(name for _id, name in result["tags"][1])
|
||||
noun = "tag" if len(names) == 1 else "tags"
|
||||
parts.append(f"{len(names)} {noun}: {_more(names, DETAIL_TAGS)}")
|
||||
pairs = [str(pair_id) for pair_id in result["pairs"]]
|
||||
label = "pair" if len(pairs) == 1 else "pairs"
|
||||
return f"Fills {' and '.join(parts)} ({label} {_more(pairs, DETAIL_VALUES)})"
|
||||
|
||||
|
||||
def _protected_detail(result, key):
|
||||
if key != "conflict":
|
||||
return next(detail for k, detail, *_r in PROTECTED if k == key)
|
||||
if result["category"] and result["category"][0] == "conflict":
|
||||
values = result["category"][1]
|
||||
return f"Kept: pairs disagree on category: {_more(values, DETAIL_VALUES)}"
|
||||
return "Kept: pairs disagree on tags"
|
||||
|
||||
|
||||
def _kept(results):
|
||||
"""One row per kept prompt, its review reasons without "Kept: " (#550)."""
|
||||
rows = []
|
||||
for result in results:
|
||||
classes = set(_protected_classes(result))
|
||||
if classes:
|
||||
rows.append(
|
||||
{
|
||||
"prompt_id": result["id"],
|
||||
"reasons": [
|
||||
_protected_detail(result, key).removeprefix("Kept: ")
|
||||
for key in DETAIL_ORDER
|
||||
if key in classes
|
||||
],
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _details(results, targets):
|
||||
rows = []
|
||||
for key in DETAIL_ORDER:
|
||||
for result in results:
|
||||
if key in _protected_classes(result):
|
||||
rows.append(
|
||||
{
|
||||
"label": _excerpt(result["text"]),
|
||||
"detail": _protected_detail(result, key),
|
||||
}
|
||||
)
|
||||
planned = {target[0] for target in targets}
|
||||
for result in results:
|
||||
if result["id"] in planned:
|
||||
rows.append(
|
||||
{"label": _excerpt(result["text"]), "detail": _fill_detail(result)}
|
||||
)
|
||||
if not targets:
|
||||
return rows[:MAX_DETAILS]
|
||||
return rows[: MAX_DETAILS - 1] + [{"label": "Record", "detail": RECORD_NOTE}]
|
||||
|
||||
|
||||
def _scope_label(targets, total):
|
||||
if not targets:
|
||||
return EMPTY_LABEL
|
||||
categories = sum(1 for _id, category, _tags in targets if category)
|
||||
tag_sets = sum(1 for _id, _category, tags in targets if tags)
|
||||
label = (
|
||||
f"{_plural(len(targets), 'prompt')} to fill: "
|
||||
f"{_plural(categories, 'category', 'categories')}, "
|
||||
f"{_plural(tag_sets, 'tag set')}"
|
||||
)
|
||||
if total > len(targets):
|
||||
label += (
|
||||
f" (the first {len(targets):,} of {total:,}; review again for the rest)"
|
||||
)
|
||||
return label
|
||||
|
||||
|
||||
def plan_pair_metadata(conn):
|
||||
"""The reviewed ``pair_metadata`` plan on ``conn``'s snapshot."""
|
||||
results, fingerprint = _state(conn)
|
||||
targets = _targets(results)
|
||||
total = sum(1 for result in results if any(_fills(result)))
|
||||
counts = {key: 0 for key in _PROTECTED_ORDER}
|
||||
held = 0
|
||||
for result in results:
|
||||
classes = _protected_classes(result)
|
||||
held += bool(classes)
|
||||
for key in classes:
|
||||
counts[key] += 1
|
||||
reasons = [
|
||||
f"{counts[key]:,} {one if counts[key] == 1 else many}"
|
||||
for key, _detail, one, many in PROTECTED
|
||||
if counts[key]
|
||||
]
|
||||
return MaintenanceDraft(
|
||||
operation=OPERATION,
|
||||
fingerprint=fingerprint,
|
||||
targets=targets,
|
||||
scope={"count": len(targets), "label": _scope_label(targets, total)},
|
||||
protected={"count": held, "reasons": reasons},
|
||||
media={"deletes_files": False, "label": MEDIA_LABEL},
|
||||
details=_details(results, targets),
|
||||
)
|
||||
|
||||
|
||||
def _fill_categories(conn, fills):
|
||||
"""Set each blank category; raises PlanChanged when one is no longer blank."""
|
||||
cursor = conn.executemany(
|
||||
"UPDATE prompts SET category = ? WHERE id = ? "
|
||||
"AND COALESCE(TRIM(category), '') = ''",
|
||||
fills,
|
||||
)
|
||||
if cursor.rowcount != len(fills):
|
||||
raise PlanChanged(OPERATION)
|
||||
|
||||
|
||||
def _fill_tags(conn, fills, expected):
|
||||
"""Copy each source pair's tags onto its untagged prompt."""
|
||||
cursor = conn.executemany(
|
||||
"INSERT INTO prompt_tags(prompt_id, tag_id) "
|
||||
"SELECT ?, ppt.tag_id FROM prompt_pair_tags ppt "
|
||||
"JOIN tags t ON t.id = ppt.tag_id WHERE ppt.pair_id = ? "
|
||||
"AND NOT EXISTS (SELECT 1 FROM prompt_tags WHERE prompt_id = ?)",
|
||||
fills,
|
||||
)
|
||||
if cursor.rowcount != expected:
|
||||
raise PlanChanged(OPERATION)
|
||||
|
||||
|
||||
def apply_pair_metadata(conn, fingerprint, targets):
|
||||
"""Apply the reviewed fills on ``conn`` (inside BEGIN IMMEDIATE).
|
||||
|
||||
Returns ``{changed, categories, tag_sets, rows, kept}``; ``rows`` lists
|
||||
every filled prompt (``{prompt_id, category?, tags?}``) and ``kept`` every
|
||||
prompt the plan keeps (``{prompt_id, reasons}``, in the review's wording
|
||||
and order), both never capped.
|
||||
"""
|
||||
results, current = _state(conn)
|
||||
if current != fingerprint or _targets(results) != targets:
|
||||
raise PlanChanged(OPERATION)
|
||||
by_id = {result["id"]: result for result in results}
|
||||
categories, tags, rows = [], [], []
|
||||
tag_count = 0
|
||||
for prompt_id, fill_category, fill_tags in targets:
|
||||
result = by_id[prompt_id]
|
||||
row = {"prompt_id": prompt_id}
|
||||
if fill_category:
|
||||
row["category"] = result["category"][1]
|
||||
categories.append((row["category"], prompt_id))
|
||||
if fill_tags:
|
||||
row["tags"] = sorted(name for _id, name in result["tags"][1])
|
||||
tags.append((prompt_id, result["tag_pair"], prompt_id))
|
||||
tag_count += len(result["tags"][1])
|
||||
rows.append(row)
|
||||
if categories:
|
||||
_fill_categories(conn, categories)
|
||||
if tags:
|
||||
_fill_tags(conn, tags, tag_count)
|
||||
return {
|
||||
"changed": len(rows),
|
||||
"categories": len(categories),
|
||||
"tag_sets": len(tags),
|
||||
"rows": rows,
|
||||
"kept": _kept(results),
|
||||
}
|
||||
@@ -46,6 +46,7 @@ def remove_quietly(*paths):
|
||||
if path and os.path.exists(path):
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
# cleanup after a failure must not replace that failure
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -66,7 +66,6 @@ def _tag_rows(page: str, order: str = "page.id") -> str:
|
||||
)
|
||||
|
||||
|
||||
_TAG_PAGE = _tag_rows("SELECT id,name FROM tags WHERE id>? ORDER BY id LIMIT ?")
|
||||
# #444 tag sorts: sort -> (key expression on tags/counted, direction). Ties and
|
||||
# "created" follow the id, so every order is total and keyset pages are exact.
|
||||
TAG_SORTS = {
|
||||
|
||||
@@ -85,6 +85,8 @@ try:
|
||||
from ...database.operations import PromptDatabase
|
||||
from ...utils.file_prompt import parse_comfyui_prompt
|
||||
from ...utils.logging_config import get_logger
|
||||
from ...utils.native_memory import release_free_memory
|
||||
from ...utils.png_text import png_text
|
||||
from ...utils.video_metadata import is_video_path, read_video_metadata
|
||||
except ImportError:
|
||||
import sys
|
||||
@@ -95,6 +97,8 @@ except ImportError:
|
||||
from database.operations import PromptDatabase
|
||||
from utils.file_prompt import parse_comfyui_prompt
|
||||
from utils.logging_config import get_logger
|
||||
from utils.native_memory import release_free_memory
|
||||
from utils.png_text import png_text
|
||||
from utils.video_metadata import is_video_path, read_video_metadata
|
||||
|
||||
|
||||
@@ -526,6 +530,14 @@ class PromptManagerAPI(
|
||||
return await loop.run_in_executor(None, call)
|
||||
return await loop.run_in_executor(None, func, *args)
|
||||
|
||||
async def _release_job_memory(self):
|
||||
"""After a scan or thumbnail job: glibc's free heap pages go back to
|
||||
the OS once (``malloc_trim(0)``; nothing elsewhere, #523)."""
|
||||
try:
|
||||
await self._run_in_executor(release_free_memory)
|
||||
except Exception: # best effort: the job's result already stands
|
||||
self.logger.debug("Releasing free memory failed", exc_info=True)
|
||||
|
||||
def _root_listing(self, root) -> RootListing:
|
||||
"""The shared listing record of one output root (created empty)."""
|
||||
return self._root_listings.setdefault(str(root), RootListing())
|
||||
@@ -1048,6 +1060,8 @@ class PromptManagerAPI(
|
||||
def _extract_comfyui_metadata(self, image_path):
|
||||
"""Embedded ComfyUI metadata of an image (PNG text chunks) or a video.
|
||||
|
||||
PNG text is read without decoding the pixels (``utils.png_text``, #523).
|
||||
|
||||
Videos go through ffprobe (see utils/video_metadata). Unreadable files
|
||||
are a warning, not an error: a scan over a large output tree meets
|
||||
truncated and foreign files, and it carries on past them.
|
||||
@@ -1056,7 +1070,7 @@ class PromptManagerAPI(
|
||||
return read_video_metadata(image_path)
|
||||
try:
|
||||
with Image.open(image_path) as img:
|
||||
return dict(getattr(img, "text", None) or {})
|
||||
return png_text(img)
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
f"Unreadable image {os.path.basename(str(image_path))}: {e}"
|
||||
|
||||
@@ -176,7 +176,7 @@ def _physical_file_key(path, stat):
|
||||
volumes on Windows) falls back to the resolved, case-folded path.
|
||||
"""
|
||||
if stat.st_ino:
|
||||
return ("inode", stat.st_dev, stat.st_ino)
|
||||
return ("inode", (stat.st_dev, stat.st_ino))
|
||||
return ("path", os.path.normcase(os.path.realpath(str(path))))
|
||||
|
||||
|
||||
@@ -539,6 +539,7 @@ def _diagnose_output_dirs(configured_dirs, public_path=None):
|
||||
|
||||
candidates.append(folder_paths.get_output_directory())
|
||||
except ImportError:
|
||||
# outside ComfyUI there is no output folder to add
|
||||
pass
|
||||
|
||||
output_dirs = []
|
||||
@@ -1453,6 +1454,7 @@ class AdminRoutesMixin:
|
||||
verb = "cancelled" if state == "cancelled" else "finished"
|
||||
job.log_line(f"Scan {verb} · {_plural(job.processed, 'file')} processed")
|
||||
job.finish(state)
|
||||
await self._release_job_memory()
|
||||
if state == "finished":
|
||||
# After the scan, so the guarded thumbnail job is admitted (#249).
|
||||
await self._hand_off_scan_thumbnails(job, output_dirs)
|
||||
@@ -1667,9 +1669,10 @@ class AdminRoutesMixin:
|
||||
def _extract_batch_metadata_sync(self, file_batch):
|
||||
"""Extract metadata from a batch of files in one executor call.
|
||||
|
||||
Files are read on ``PromptManagerConfig.WORKER_THREADS`` threads:
|
||||
Pillow decodes the whole image to reach trailing text chunks and
|
||||
releases the GIL while doing so.
|
||||
Files are read on the long-lived job pool of
|
||||
``PromptManagerConfig.WORKER_THREADS`` threads (#523).
|
||||
PNG text is read chunk by chunk without decoding pixels (#523); file
|
||||
reads and ffprobe release the GIL.
|
||||
"""
|
||||
from ..config import PromptManagerConfig
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ import math
|
||||
from ..tagging_defaults import ( # noqa: F401 - re-exported for the route
|
||||
LLM_MODEL_TYPES as LLM_MODELS,
|
||||
MAX_INSTRUCTION_BYTES,
|
||||
MAX_INSTRUCTION_CHARS,
|
||||
MODEL_TYPES,
|
||||
WD14_MODEL_TYPES as WD14_MODELS,
|
||||
validate_template,
|
||||
@@ -103,11 +102,14 @@ def model_refusal(status: dict, model_type: str, model_name: str):
|
||||
return None
|
||||
|
||||
|
||||
def run_suggestions(service, capture_path: str, model_type: str, options: dict):
|
||||
def run_suggestions(
|
||||
service, capture_path: str, model_type: str, options: dict, skip=None
|
||||
):
|
||||
"""Blocking: load (or swap to) *model_type*, then suggest for the capture.
|
||||
|
||||
The caller holds the model gate for the whole call; nothing is ever
|
||||
downloaded (``load_model`` refuses a model that is not on disk).
|
||||
downloaded (``load_model`` refuses a model that is not on disk). ``skip``
|
||||
is the request's or job's compiled Skipped tags (#431).
|
||||
"""
|
||||
if not service.is_model_loaded() or service.get_loaded_model_type() != model_type:
|
||||
service.load_model(model_type, options["use_gpu"])
|
||||
@@ -117,6 +119,7 @@ def run_suggestions(service, capture_path: str, model_type: str, options: dict):
|
||||
prompt=options.get("instructions"),
|
||||
general_threshold=options.get("general_threshold"),
|
||||
character_threshold=options.get("character_threshold"),
|
||||
skip=skip,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ from .media_target import media_target as _target
|
||||
from .images import _unavailable_link, _verified_title
|
||||
from .prompts import bad_request
|
||||
from .. import model_gate
|
||||
from ..tagging_defaults import SkipMatcher
|
||||
|
||||
try:
|
||||
from ...database.annotations import (
|
||||
@@ -640,6 +641,8 @@ class AnnotationRoutesMixin:
|
||||
owner = model_gate.new_owner("image-annotations")
|
||||
if not model_gate.try_acquire(owner):
|
||||
return model_busy_response()
|
||||
# #431: the Skipped tags as they are when the request is accepted.
|
||||
skip = SkipMatcher(AutotagConfig.SKIPPED_TAGS).counting()
|
||||
try:
|
||||
# From here the pin is the gate's to end: the worker ends it,
|
||||
# or ``on_skipped`` does when the worker never starts.
|
||||
@@ -653,6 +656,7 @@ class AnnotationRoutesMixin:
|
||||
capture,
|
||||
model_type,
|
||||
options,
|
||||
skip,
|
||||
on_skipped=functools.partial(store.release_use, token),
|
||||
)
|
||||
except MissingBackendPackages as exc:
|
||||
@@ -677,6 +681,8 @@ class AnnotationRoutesMixin:
|
||||
"model_type": model_type,
|
||||
"options": options,
|
||||
"suggestions": suggestion_rows(pairs),
|
||||
# How many distinct suggestions the Skipped tags hid (#431).
|
||||
"skipped": len(skip.hidden),
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
@@ -732,10 +738,10 @@ def _byte_range(request: web.Request, size: int):
|
||||
return start, stop
|
||||
|
||||
|
||||
def _pinned_suggestions(store, token, service, capture, model_type, options):
|
||||
def _pinned_suggestions(store, token, service, capture, model_type, options, skip=None):
|
||||
"""Worker thread: infer on the capture, then end its pin (even on error)."""
|
||||
try:
|
||||
return run_suggestions(service, capture.path, model_type, options)
|
||||
return run_suggestions(service, capture.path, model_type, options, skip)
|
||||
finally:
|
||||
store.release_use(token)
|
||||
|
||||
|
||||
@@ -40,6 +40,12 @@ BATCH_PREVIEW_BODY_MAX = 1024
|
||||
APPLY_BODY_MAX = 64 * 1024 # POST /autotag/apply: one image and its tags
|
||||
_HEX64 = re.compile(r"[0-9a-f]{64}")
|
||||
BATCH_JOB_BODY_MAX = 32 * 1024
|
||||
# Skipped tags preview (#431): 200 entries of 50 characters, with room for JSON.
|
||||
SKIPPED_PREVIEW_BODY_MAX = 64 * 1024
|
||||
SKIPPED_PREVIEW_LEASE = "Skipped tags preview"
|
||||
SKIPPED_PREVIEW_SAMPLE = 12
|
||||
# WD14 label lists the preview found missing or unreadable (logged once each).
|
||||
_MISSING_LABEL_LISTS = set()
|
||||
REVIEW_REQUIRED = (
|
||||
"Replacing tags needs a review of each image. Use Preview tagging review, "
|
||||
"or choose Skip or Merge for a batch run."
|
||||
@@ -72,6 +78,12 @@ MODEL_DOWNLOAD_BODY_MAX = 1024
|
||||
_download_titles = {}
|
||||
|
||||
|
||||
def _instruction_engine(model_type: str) -> str:
|
||||
"""The language-model engine whose default instructions ``model_type``
|
||||
reads: itself, or GGUF for a WD14 engine (which takes none)."""
|
||||
return model_type if model_type in ("gguf", "hf") else "gguf"
|
||||
|
||||
|
||||
def _holder_label(holder: Optional[str]) -> str:
|
||||
if not holder:
|
||||
return ANOTHER_OPERATION
|
||||
@@ -247,6 +259,10 @@ class AutotagRoutesMixin:
|
||||
async def preview_autotag_batch_route(request):
|
||||
return await self.preview_autotag_batch(request)
|
||||
|
||||
@routes.post("/prompt_manager/autotag/skipped-tags/preview")
|
||||
async def preview_skipped_tags_route(request):
|
||||
return await self.preview_skipped_tags(request)
|
||||
|
||||
@routes.post("/prompt_manager/autotag/start")
|
||||
async def start_autotag_route(request):
|
||||
return await self.start_autotag(request)
|
||||
@@ -284,7 +300,15 @@ class AutotagRoutesMixin:
|
||||
"success": True,
|
||||
"models": public_models(status, loaded_type),
|
||||
# The saved tagging defaults (#328, System › Auto-tagging).
|
||||
"default_prompt": AutotagConfig.TEMPLATE,
|
||||
# The saved engine's default instructions (GGUF for a
|
||||
# WD14 engine); kept one release for cached pages (#431).
|
||||
"default_prompt": AutotagConfig.default_instructions_text(
|
||||
_instruction_engine(AutotagConfig.MODEL_TYPE)
|
||||
),
|
||||
# #431: Built-in first, with every text, and each
|
||||
# language-model engine's default.
|
||||
"instructions": AutotagConfig.all_instructions(),
|
||||
"default_instructions": dict(AutotagConfig.DEFAULT_INSTRUCTIONS),
|
||||
"model_loaded": loaded,
|
||||
"loaded_model_type": loaded_type,
|
||||
"wd14_general_threshold": AutotagConfig.GENERAL_THRESHOLD,
|
||||
@@ -424,6 +448,7 @@ class AutotagRoutesMixin:
|
||||
title = f"Tagging model download · {label}"
|
||||
_download_titles.clear()
|
||||
_download_titles[owner] = title
|
||||
admitted = False
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
MODEL_DOWNLOAD_JOB_KIND,
|
||||
@@ -439,13 +464,13 @@ class AutotagRoutesMixin:
|
||||
self._run_model_download, service, model_type, owner
|
||||
),
|
||||
)
|
||||
except BaseException as refused:
|
||||
# JobBusy, or anything else the registry raised: give the gate back.
|
||||
model_gate.release(owner)
|
||||
_download_titles.pop(owner, None)
|
||||
if isinstance(refused, JobBusy):
|
||||
return None, busy_response(refused)
|
||||
raise
|
||||
admitted = True
|
||||
except JobBusy as refused:
|
||||
return None, busy_response(refused)
|
||||
finally:
|
||||
if not admitted: # JobBusy, or anything else: give the gate back.
|
||||
model_gate.release(owner)
|
||||
_download_titles.pop(owner, None)
|
||||
|
||||
def ended(_task):
|
||||
model_gate.release(owner)
|
||||
@@ -538,7 +563,10 @@ class AutotagRoutesMixin:
|
||||
else:
|
||||
prompt = params.get("prompt")
|
||||
template = prompt if isinstance(prompt, str) and prompt.strip() else None
|
||||
options = {"instructions": template or AutotagConfig.TEMPLATE}
|
||||
options = {
|
||||
"instructions": template
|
||||
or AutotagConfig.default_instructions_text(model_type)
|
||||
}
|
||||
try:
|
||||
_model, options = validate_options(
|
||||
model_type,
|
||||
@@ -698,6 +726,86 @@ class AutotagRoutesMixin:
|
||||
return None, "existing must be one of " + ", ".join(EXISTING_POLICIES)
|
||||
return MODELS[model_type], None
|
||||
|
||||
async def preview_skipped_tags(self, request):
|
||||
"""Which known tags a Skipped tags list would hide (#431); no write.
|
||||
|
||||
``{entries}`` is checked as a save checks it (the same 400 sentence).
|
||||
The known tags are the library's plus the general and character names
|
||||
of each downloaded WD14 label list; no model is loaded and no image is
|
||||
opened. Refused (409 ``busy``) while an exclusive job runs.
|
||||
"""
|
||||
from ..tagging_defaults import validate_skipped_tags
|
||||
|
||||
data, error = await self._batch_body(
|
||||
request, ("entries",), SKIPPED_PREVIEW_BODY_MAX
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
try:
|
||||
entries = validate_skipped_tags(data["entries"])
|
||||
except ValueError as exc:
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=400)
|
||||
try:
|
||||
async with self._jobs.lease(SKIPPED_PREVIEW_LEASE):
|
||||
result = await self._run_in_executor(self._skipped_preview, entries)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
except Exception:
|
||||
self.logger.exception("The Skipped tags preview failed")
|
||||
return web.json_response(
|
||||
{"success": False, "error": _INTERNAL_ERROR}, status=500
|
||||
)
|
||||
return web.json_response({"success": True, **result})
|
||||
|
||||
def _skipped_preview(self, entries):
|
||||
"""Worker thread: the preview's counts over the known tags."""
|
||||
from ..autotag import MODELS, get_autotag_service
|
||||
from ..tagging_defaults import (
|
||||
WD14_MODEL_TYPES,
|
||||
SkipMatcher,
|
||||
wd14_label_names,
|
||||
)
|
||||
|
||||
library = self.db.get_all_tags()
|
||||
known = {} # case-folded -> (0 library / 1 label list, spelling)
|
||||
for name in library:
|
||||
known.setdefault(name.casefold(), (0, name))
|
||||
lists = []
|
||||
models_dir = getattr(get_autotag_service(), "models_dir", None)
|
||||
for model_type in sorted(WD14_MODEL_TYPES, key=lambda t: MODELS[t]["name"]):
|
||||
if models_dir is None:
|
||||
break
|
||||
path = Path(models_dir) / MODELS[model_type]["subdir"] / "selected_tags.csv"
|
||||
try:
|
||||
names = wd14_label_names(path)
|
||||
except (OSError, ValueError) as exc:
|
||||
if path not in _MISSING_LABEL_LISTS: # logged once per list
|
||||
_MISSING_LABEL_LISTS.add(path)
|
||||
self.logger.debug(f"No WD14 label list for {model_type}: {exc}")
|
||||
continue
|
||||
_MISSING_LABEL_LISTS.discard(path)
|
||||
lists.append(MODELS[model_type]["name"])
|
||||
for name in names:
|
||||
known.setdefault(name.casefold(), (1, name))
|
||||
matchers = [(entry, SkipMatcher([entry])) for entry in entries]
|
||||
whole = SkipMatcher(entries)
|
||||
hidden = [item for item in known.values() if whole.matches(item[1])]
|
||||
hidden.sort(key=lambda item: (item[0], item[1].casefold()))
|
||||
per_entry = [
|
||||
{
|
||||
"entry": entry,
|
||||
"count": sum(1 for _, name in hidden if matcher.matches(name)),
|
||||
}
|
||||
for entry, matcher in matchers
|
||||
]
|
||||
return {
|
||||
"count": len(hidden),
|
||||
"sample": [name for _, name in hidden[:SKIPPED_PREVIEW_SAMPLE]],
|
||||
"sample_limit": SKIPPED_PREVIEW_SAMPLE,
|
||||
"per_entry": per_entry,
|
||||
"sources": {"library": len(library), "wd14": lists},
|
||||
}
|
||||
|
||||
async def preview_autotag_batch(self, request):
|
||||
"""Review Tag output images: counts, model, backup and disk; no write.
|
||||
|
||||
@@ -1001,7 +1109,10 @@ class AutotagRoutesMixin:
|
||||
else:
|
||||
prompt = data.get("prompt")
|
||||
template = prompt if isinstance(prompt, str) and prompt.strip() else None
|
||||
options = {"instructions": template or AutotagConfig.TEMPLATE}
|
||||
options = {
|
||||
"instructions": template
|
||||
or AutotagConfig.default_instructions_text(model_type)
|
||||
}
|
||||
use_gpu = data.get("use_gpu", True)
|
||||
options["use_gpu"] = use_gpu if isinstance(use_gpu, bool) else True
|
||||
return validate_options(
|
||||
@@ -1041,12 +1152,12 @@ class AutotagRoutesMixin:
|
||||
return self._bad_request('existing must be "skip" or "merge"')
|
||||
try:
|
||||
options = self._single_options(data, model_type)
|
||||
except OptionError as error:
|
||||
return self._bad_request(str(error))
|
||||
except (ValueError, TypeError):
|
||||
return self._bad_request(
|
||||
"general_threshold and character_threshold must be numeric"
|
||||
)
|
||||
except OptionError as error:
|
||||
return self._bad_request(str(error))
|
||||
target, error = await self._select_autotag_target(data)
|
||||
if error is not None:
|
||||
return error
|
||||
@@ -1095,6 +1206,8 @@ class AutotagRoutesMixin:
|
||||
|
||||
async def _suggest_single(self, target, model_type, options, base):
|
||||
from ..autotag import MODELS, MissingBackendPackages, get_autotag_service
|
||||
from ..config import AutotagConfig
|
||||
from ..tagging_defaults import SkipMatcher
|
||||
from .annotation_suggest import model_refusal
|
||||
from .annotations import _pinned_suggestions, _settled, _submit
|
||||
|
||||
@@ -1111,6 +1224,8 @@ class AutotagRoutesMixin:
|
||||
owner = model_gate.new_owner("legacy-single")
|
||||
if not model_gate.try_acquire(owner):
|
||||
return model_busy_response()
|
||||
# #431: the legacy review hides the Skipped tags too (19A).
|
||||
skip = SkipMatcher(AutotagConfig.SKIPPED_TAGS)
|
||||
store = self._capture_store()
|
||||
try:
|
||||
epoch = await self._run_in_executor(self.db.get_execution_epoch)
|
||||
@@ -1150,6 +1265,7 @@ class AutotagRoutesMixin:
|
||||
capture,
|
||||
model_type,
|
||||
options,
|
||||
skip,
|
||||
on_skipped=functools.partial(store.release_use, capture.token),
|
||||
)
|
||||
except MissingBackendPackages as exc:
|
||||
|
||||
@@ -971,6 +971,7 @@ class BackupRestoreRoutesMixin:
|
||||
line = restored_line(staged.summary)
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=line)
|
||||
return None
|
||||
|
||||
|
||||
async def _outlive_cancel(future):
|
||||
|
||||
@@ -574,12 +574,13 @@ class DuplicateFilesMixin:
|
||||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return
|
||||
return None
|
||||
try:
|
||||
await self._remove_duplicates(job, db, plan, backup)
|
||||
finally:
|
||||
if backup:
|
||||
await self._prune_backup_pool(job, db)
|
||||
return None
|
||||
|
||||
async def _remove_duplicates(self, job, db, plan, backup):
|
||||
targets = plan.payload["targets"]
|
||||
|
||||
@@ -27,7 +27,7 @@ def _identity(path):
|
||||
except OSError:
|
||||
return None
|
||||
if info.st_ino:
|
||||
return ("inode", info.st_dev, info.st_ino)
|
||||
return ("inode", (info.st_dev, info.st_ino))
|
||||
return ("path", os.path.normcase(os.path.realpath(str(path))))
|
||||
|
||||
|
||||
|
||||
@@ -843,7 +843,10 @@ class HistoryRoutesMixin:
|
||||
raise
|
||||
|
||||
async def execute():
|
||||
pending = asyncio.create_task(self._run_in_executor(project))
|
||||
# An inspection reads the file's header and text: the
|
||||
# interactive image pool (#523); media stays on the default.
|
||||
run = self._run_image if kind == "inspection" else self._run_in_executor
|
||||
pending = asyncio.create_task(run(project))
|
||||
try:
|
||||
return await asyncio.shield(pending)
|
||||
except asyncio.CancelledError:
|
||||
|
||||
@@ -330,12 +330,13 @@ class ImageDeleteMixin:
|
||||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return
|
||||
return None
|
||||
try:
|
||||
await self._delete_images(job, db, targets, mode, backup)
|
||||
finally:
|
||||
if backup:
|
||||
await self._prune_backup_pool(job, db)
|
||||
return None
|
||||
|
||||
async def _delete_images(self, job, db, targets, mode, backup):
|
||||
job.update(total=len(targets), status_line=TITLES[mode])
|
||||
|
||||
@@ -39,6 +39,7 @@ from typing import NamedTuple
|
||||
from aiohttp import web
|
||||
|
||||
from .. import model_gate
|
||||
from ..tagging_defaults import SkipMatcher
|
||||
from .admin import _plural, _time_left
|
||||
from .annotation_suggest import (
|
||||
OptionError,
|
||||
@@ -170,6 +171,8 @@ class _Run:
|
||||
owner: object
|
||||
service: object = None
|
||||
roots: dict = field(default_factory=dict)
|
||||
# The job's Skipped tags, compiled once at its start (#431).
|
||||
skip: object = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -183,6 +186,9 @@ class _Spec:
|
||||
options: dict | None = None
|
||||
# A listing filter the start resolves into ``targets`` (#392).
|
||||
filter: dict | None = None
|
||||
# The Skipped tags when the job was requested (#431); a later save does
|
||||
# not change a running job.
|
||||
skip: tuple = ()
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
@@ -238,7 +244,11 @@ def _parse_auto(targets, data):
|
||||
)
|
||||
except OptionError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
return _Spec("auto", targets, model_type=model_type, options=options), None
|
||||
skip = tuple(AutotagConfig.SKIPPED_TAGS)
|
||||
return (
|
||||
_Spec("auto", targets, model_type=model_type, options=options, skip=skip),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def parse_image_tag_request(data: dict):
|
||||
@@ -329,6 +339,8 @@ class AutotagRun:
|
||||
keep_in_memory: bool
|
||||
backup_on: bool
|
||||
owner: object
|
||||
# The Skipped tags when the job was started (#431).
|
||||
skip: tuple = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -400,6 +412,14 @@ def _load_model(service, model_type: str, use_gpu: bool) -> None:
|
||||
raise RuntimeError(f"{model_type} could not be loaded")
|
||||
|
||||
|
||||
def _skipped_tags_line(entries: tuple) -> str:
|
||||
"""The job log's one line about the Skipped tags it runs with (#431)."""
|
||||
if not entries:
|
||||
return "Skipped tags: none"
|
||||
count = len(entries)
|
||||
return f"Skipped tags: {count} {'entry' if count == 1 else 'entries'}"
|
||||
|
||||
|
||||
def _refusal(status: int, code: str, message: str) -> Refusal:
|
||||
response = web.json_response(
|
||||
{"success": False, "code": code, "error": message}, status=status
|
||||
@@ -507,6 +527,7 @@ class ImageTagJobsMixin:
|
||||
released when the task ends, also before its runner ever starts.
|
||||
"""
|
||||
from ..autotag import MODELS, get_autotag_service
|
||||
from ..config import AutotagConfig
|
||||
|
||||
backup_on = self._safety_backup_policy() == "on"
|
||||
status = await self._run_in_executor(get_autotag_service().get_models_status)
|
||||
@@ -529,8 +550,15 @@ class ImageTagJobsMixin:
|
||||
response = model_busy_response()
|
||||
return None, Refusal(409, "model-busy", MODEL_BUSY_ERROR, response)
|
||||
run = AutotagRun(
|
||||
model_type, existing, options, keep_in_memory, backup_on, owner
|
||||
model_type,
|
||||
existing,
|
||||
options,
|
||||
keep_in_memory,
|
||||
backup_on,
|
||||
owner,
|
||||
skip=tuple(AutotagConfig.SKIPPED_TAGS),
|
||||
)
|
||||
admitted = False
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
AUTOTAG_BATCH_JOB_KIND,
|
||||
@@ -541,11 +569,12 @@ class ImageTagJobsMixin:
|
||||
groups=AUTOTAG_GROUP_NAMES,
|
||||
runner=functools.partial(self._run_autotag_batch, run),
|
||||
)
|
||||
except BaseException as refused:
|
||||
model_gate.release(owner)
|
||||
if isinstance(refused, JobBusy):
|
||||
return None, _busy_refusal(refused)
|
||||
raise
|
||||
admitted = True
|
||||
except JobBusy as refused:
|
||||
return None, _busy_refusal(refused)
|
||||
finally:
|
||||
if not admitted:
|
||||
model_gate.release(owner)
|
||||
job.task.add_done_callback(lambda _task: model_gate.release(owner))
|
||||
return job, None
|
||||
|
||||
@@ -596,7 +625,9 @@ class ImageTagJobsMixin:
|
||||
return _batch_cancelled(job)
|
||||
job.publish({"type": "phase", "phase": "model"})
|
||||
job.update(status_line="Loading the tagging model")
|
||||
spec = _Spec("auto", (), model_type=run.model_type, options=run.options)
|
||||
spec = _Spec(
|
||||
"auto", (), model_type=run.model_type, options=run.options, skip=run.skip
|
||||
)
|
||||
service = await self._prepare_tagging_model(job, spec, run.owner)
|
||||
if service is None:
|
||||
return "failed"
|
||||
@@ -622,6 +653,7 @@ class ImageTagJobsMixin:
|
||||
f"Tagging started · {job.scope.get('label') or run.model_type} · "
|
||||
f"{_plural(total, 'image')} · {policy}"
|
||||
)
|
||||
job.log_line(_skipped_tags_line(spec.skip))
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
remaining = await self._autotag_prepass(job, targets, run.existing == "skip")
|
||||
@@ -631,7 +663,9 @@ class ImageTagJobsMixin:
|
||||
spec = dataclasses.replace(
|
||||
spec, targets=tuple(item.target for item in remaining)
|
||||
)
|
||||
loop_run = _Run(job, spec, epoch, run.owner, service)
|
||||
loop_run = _Run(
|
||||
job, spec, epoch, run.owner, service, skip=SkipMatcher(spec.skip)
|
||||
)
|
||||
if not await self._tag_each_file(loop_run, done_before=done):
|
||||
job.update(status_line=STORAGE_RESTORED)
|
||||
job.log_line(f"Tagging stopped · {STORAGE_RESTORED}")
|
||||
@@ -737,6 +771,9 @@ class ImageTagJobsMixin:
|
||||
started += f" · {spec.model_type}"
|
||||
job.log_line(started)
|
||||
run = _Run(job, spec, epoch, owner)
|
||||
if spec.mode == "auto":
|
||||
job.log_line(_skipped_tags_line(spec.skip))
|
||||
run.skip = SkipMatcher(spec.skip)
|
||||
try:
|
||||
if spec.mode == "auto":
|
||||
run.service = await self._prepare_tagging_model(job, spec, owner)
|
||||
@@ -892,6 +929,7 @@ class ImageTagJobsMixin:
|
||||
capture,
|
||||
spec.model_type,
|
||||
spec.options,
|
||||
run.skip,
|
||||
on_skipped=functools.partial(store.release_use, capture.token),
|
||||
)
|
||||
except Exception:
|
||||
|
||||
@@ -39,10 +39,8 @@ from .thumbnail_encode import (
|
||||
PREVIEW_HELD,
|
||||
EncoderClosed,
|
||||
PreviewWriteHeld,
|
||||
replace_unless_held,
|
||||
run_encoder,
|
||||
sweep_stray_temps,
|
||||
temp_sibling,
|
||||
thumbnail_frame,
|
||||
writer_settings,
|
||||
write_thumbnail,
|
||||
@@ -339,11 +337,6 @@ def _thumbnail_batch_size(workers):
|
||||
return max(4, workers * 2)
|
||||
|
||||
|
||||
# Preview files are written by thumbnail_encode in the configured format
|
||||
# (thumbnail_settings, #317); these names stay importable from here.
|
||||
_temp_sibling = temp_sibling
|
||||
_replace_unless_held = replace_unless_held
|
||||
|
||||
# A preview that could not be replaced (#249 S5 M6). Windows refuses to rename
|
||||
# over a file another process has open (WinError 5 or 32), such as a preview
|
||||
# the gallery is serving; the old preview stays and the file counts as an error.
|
||||
@@ -396,6 +389,7 @@ def _video_placeholder_image(thumbnail_size):
|
||||
font=font,
|
||||
)
|
||||
except (OSError, AttributeError):
|
||||
# a placeholder without its label still works
|
||||
pass
|
||||
return img
|
||||
|
||||
@@ -499,6 +493,7 @@ def _preview_names(folders, folder):
|
||||
except OSError:
|
||||
continue
|
||||
except OSError:
|
||||
# an unreadable folder lists as empty; nothing there to cache
|
||||
pass
|
||||
folders[folder] = names
|
||||
return names
|
||||
@@ -2980,6 +2975,7 @@ class ImageRoutesMixin:
|
||||
result = await self._run_in_executor(
|
||||
self._generate_gallery_thumbnails_sync, output_dirs
|
||||
)
|
||||
await self._release_job_memory()
|
||||
return web.json_response(result)
|
||||
|
||||
except Exception as e:
|
||||
@@ -4002,7 +3998,8 @@ class ImageRoutesMixin:
|
||||
**observed,
|
||||
)
|
||||
|
||||
body = await self._run_in_executor(inspect)
|
||||
# Header and text reads: the interactive image pool (#523).
|
||||
body = await self._run_image(inspect)
|
||||
if body is None:
|
||||
return web.json_response(
|
||||
{
|
||||
@@ -4215,6 +4212,22 @@ class ImageRoutesMixin:
|
||||
# (which Windows refuses on open files) is not held up by it.
|
||||
return _CardFacts(width, height, *self._card_parse(path, records))
|
||||
|
||||
def _cached_card_facts(self, entry: tuple) -> _CardFacts | None:
|
||||
path, mtime, root, index = entry[:4]
|
||||
return self._workspace_card_cache.get((index, str(root)), {}).get((path, mtime))
|
||||
|
||||
def _card_facts_of(self, entries: list, *, parse: bool) -> list[_CardFacts]:
|
||||
"""Card facts of *entries*, in order; files not cached yet are read on
|
||||
the interactive image pool (#523), cached ones answer here."""
|
||||
found = [self._cached_card_facts(entry) for entry in entries]
|
||||
missing = [i for i, facts in enumerate(found) if facts is None]
|
||||
read = self._map_image(
|
||||
lambda i: self._card_facts(entries[i], parse=parse), missing
|
||||
)
|
||||
for i, facts in zip(missing, read):
|
||||
found[i] = facts
|
||||
return found
|
||||
|
||||
def _card_facts(self, entry: tuple, *, parse: bool) -> _CardFacts:
|
||||
"""One listed image's card facts, read once per ``(path, mtime)``.
|
||||
|
||||
@@ -4223,11 +4236,11 @@ class ImageRoutesMixin:
|
||||
and drops the rest, and a roots change drops them all. Only files of
|
||||
the current scan are stored, so the cache is bounded by the listing.
|
||||
"""
|
||||
path, mtime, root, index = entry[:4]
|
||||
root_key, key = (index, str(root)), (path, mtime)
|
||||
cached = self._workspace_card_cache.get(root_key, {}).get(key)
|
||||
cached = self._cached_card_facts(entry)
|
||||
if cached is not None:
|
||||
return cached
|
||||
path, mtime, root, index = entry[:4]
|
||||
root_key, key = (index, str(root)), (path, mtime)
|
||||
facts = self._read_card_facts(path, parse=parse)
|
||||
if facts is None:
|
||||
# Not stored: a locked or half-written file may read next time.
|
||||
@@ -4253,15 +4266,17 @@ class ImageRoutesMixin:
|
||||
if linked
|
||||
else {}
|
||||
)
|
||||
# Videos: the card reads no video metadata. Library previews show
|
||||
# no card facts, so they skip the parse (``parse=linked``).
|
||||
stills = [image for image in images if not image["is_video"]]
|
||||
read = self._card_facts_of(
|
||||
[entries[(image["root_index"], image["path"])] for image in stills],
|
||||
parse=linked,
|
||||
)
|
||||
facts_of = {id(image): facts for image, facts in zip(stills, read)}
|
||||
for image in images:
|
||||
entry = entries[(image["root_index"], image["path"])]
|
||||
# Videos: the card reads no video metadata. Library previews show
|
||||
# no card facts, so they skip the parse (``parse=linked``).
|
||||
facts = (
|
||||
_NO_CARD_FACTS
|
||||
if image["is_video"]
|
||||
else self._card_facts(entry, parse=linked)
|
||||
)
|
||||
facts = facts_of.get(id(image), _NO_CARD_FACTS)
|
||||
image["width"], image["height"] = facts.width, facts.height
|
||||
if not linked:
|
||||
continue
|
||||
|
||||
@@ -250,7 +250,7 @@ class JobRoutesMixin:
|
||||
try:
|
||||
await response.write(_summary_event(job))
|
||||
async with contextlib.aclosing(job.events()) as events:
|
||||
async for _payload in events:
|
||||
async for _ in events:
|
||||
await response.write(_summary_event(job))
|
||||
# The last event is published before the job takes its final state.
|
||||
await response.write(_summary_event(job))
|
||||
|
||||
@@ -204,6 +204,8 @@ class Job:
|
||||
self.total = 0
|
||||
self.counters = dict.fromkeys(groups, 0)
|
||||
self.outcomes = {group: [] for group in groups}
|
||||
# Groups whose report lists every row, beyond MAX_OUTCOME_ROWS (#550).
|
||||
self.complete = {}
|
||||
self.log = []
|
||||
self._cancel_requested = False
|
||||
self.last_event = None
|
||||
@@ -297,31 +299,36 @@ class Job:
|
||||
)
|
||||
self.log_line(log_line)
|
||||
|
||||
def record_counted(self, group, count, rows=()):
|
||||
def record_counted(self, group, count, rows=(), *, complete=False):
|
||||
"""Count ``count`` outcomes in ``group`` at once, listing ``rows``.
|
||||
|
||||
For outcomes that need no log line each (a thumbnail job's already
|
||||
fresh previews): the counter is exact, ``rows`` (``record``'s row
|
||||
shape) are listed up to MAX_OUTCOME_ROWS, and nothing is logged.
|
||||
``complete`` also keeps every row for the report (#550: a record of
|
||||
each change that must not be cut short; the caller bounds it).
|
||||
"""
|
||||
if group not in self.counters:
|
||||
raise ValueError(f"Unknown outcome group {group!r} for {self.kind}")
|
||||
self.counters[group] += max(0, int(count))
|
||||
listed = self.outcomes[group]
|
||||
kept = self.complete.setdefault(group, []) if complete else None
|
||||
for row in rows:
|
||||
if len(listed) >= MAX_OUTCOME_ROWS:
|
||||
if kept is None and len(listed) >= MAX_OUTCOME_ROWS:
|
||||
break
|
||||
index = row.get("root_index")
|
||||
if isinstance(index, bool) or not isinstance(index, int) or index < 0:
|
||||
index = None
|
||||
listed.append(
|
||||
{
|
||||
"path": _relative_path(row.get("path")),
|
||||
"root": str(row.get("root") or ""),
|
||||
"root_index": index,
|
||||
"detail": str(row.get("detail") or ""),
|
||||
}
|
||||
)
|
||||
shaped = {
|
||||
"path": _relative_path(row.get("path")),
|
||||
"root": str(row.get("root") or ""),
|
||||
"root_index": index,
|
||||
"detail": str(row.get("detail") or ""),
|
||||
}
|
||||
if kept is not None:
|
||||
kept.append(shaped)
|
||||
if len(listed) < MAX_OUTCOME_ROWS:
|
||||
listed.append(dict(shaped))
|
||||
|
||||
def finish(self, state="finished"):
|
||||
"""End the job in ``state`` and release every subscriber.
|
||||
@@ -406,8 +413,9 @@ class Job:
|
||||
"""The downloadable report: the summary, every outcome group and the log.
|
||||
|
||||
Each group carries its exact count, its listed rows (at most
|
||||
MAX_OUTCOME_ROWS) and whether the list is cut short. Rows and log lines
|
||||
already hold root-relative paths only.
|
||||
MAX_OUTCOME_ROWS, or every row of a ``complete`` group) and whether
|
||||
the list is cut short. Rows and log lines already hold root-relative
|
||||
paths only.
|
||||
"""
|
||||
summary = self.summary()
|
||||
summary.pop("listed", None)
|
||||
@@ -417,16 +425,21 @@ class Job:
|
||||
"generated_at": self._clock(),
|
||||
"job": summary,
|
||||
"outcomes": {
|
||||
group: {
|
||||
"count": self.counters.get(group, len(rows)),
|
||||
"rows": [dict(row) for row in rows],
|
||||
"truncated": self.counters.get(group, len(rows)) > len(rows),
|
||||
}
|
||||
group: self._report_group(group, rows)
|
||||
for group, rows in self.outcomes.items()
|
||||
},
|
||||
"log": list(self.log),
|
||||
}
|
||||
|
||||
def _report_group(self, group, listed):
|
||||
rows = self.complete.get(group, listed)
|
||||
count = self.counters.get(group, len(rows))
|
||||
return {
|
||||
"count": count,
|
||||
"rows": [dict(row) for row in rows],
|
||||
"truncated": count > len(rows),
|
||||
}
|
||||
|
||||
def report_filename(self):
|
||||
"""The download name: prompt-manager-KIND-YYYYMMDD-HHMM.json (local time)."""
|
||||
stamp = time.strftime("%Y%m%d-%H%M", time.localtime(self._clock()))
|
||||
|
||||
@@ -207,6 +207,7 @@ class LibraryLocationRoutesMixin:
|
||||
try:
|
||||
size = os.path.getsize(location.db_path)
|
||||
except OSError:
|
||||
# no library file yet: report size 0
|
||||
pass
|
||||
return {
|
||||
"path": self._library_public_path(location.db_path),
|
||||
@@ -373,6 +374,7 @@ class LibraryLocationRoutesMixin:
|
||||
line += ". Logs move at the next restart."
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=line)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _fail_library_move(job, code, message):
|
||||
|
||||
@@ -205,3 +205,4 @@ class LogCleanupMixin:
|
||||
line += f", {failed} could not be deleted"
|
||||
job.log_line(line)
|
||||
job.update(processed=len(files), progress=100, status_line=line)
|
||||
return None
|
||||
|
||||
@@ -436,7 +436,7 @@ class LoraImportRoutesMixin:
|
||||
job.log_line(f"{CANCELLED} {CACHE_NOTE}")
|
||||
job.update(status_line=CANCELLED)
|
||||
job.finish("cancelled")
|
||||
return
|
||||
return None
|
||||
except Exception:
|
||||
self.logger.exception("LoRA import staging failed")
|
||||
return _fail_job(job, "import-failed", f"{FAILED} {CACHE_NOTE}")
|
||||
@@ -455,10 +455,10 @@ class LoraImportRoutesMixin:
|
||||
job.log_line(f"{CANCELLED} {CACHE_NOTE}")
|
||||
job.update(status_line=CANCELLED)
|
||||
job.finish("cancelled")
|
||||
return
|
||||
return None
|
||||
ok, _backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return
|
||||
return None
|
||||
# I1: the files are listed and fingerprinted again right before the
|
||||
# replace, so a file changed, removed or added while the downloads ran
|
||||
# (LoraManager refreshing its CivitAI info) aborts with nothing deleted.
|
||||
@@ -466,6 +466,7 @@ class LoraImportRoutesMixin:
|
||||
return _fail_job(job, "plan-changed", f"{SOURCE_CHANGED} {CACHE_NOTE}")
|
||||
job.update(status_line="Replacing the LoRA collection", progress=95)
|
||||
await self._replace_in_job(job, db, plan, staged)
|
||||
return None
|
||||
|
||||
def _count_downloads(self, job, staged):
|
||||
downloaded = staged.stats.get("downloaded", 0)
|
||||
@@ -518,3 +519,4 @@ class LoraImportRoutesMixin:
|
||||
staged.stats.get("download_errors", 0),
|
||||
)
|
||||
job.update(progress=100, processed=job.total, status_line=line)
|
||||
return None
|
||||
|
||||
@@ -44,6 +44,7 @@ _FILES_CHANGED = (
|
||||
"The LoRA metadata files changed during the scan; the previous imports "
|
||||
"were kept. Run the scan again."
|
||||
)
|
||||
_LORA_SCAN_FAILED = "The LoRA import failed. Check the server log for details."
|
||||
_IMPORTS_CHANGED = (
|
||||
"The imported LoRA prompts changed during the scan; the previous imports "
|
||||
"were kept. Run the scan again."
|
||||
@@ -407,10 +408,21 @@ class LoraIntegrationMixin:
|
||||
return
|
||||
if isinstance(error, PlanChanged):
|
||||
message = _IMPORTS_CHANGED
|
||||
elif isinstance(error, (LoraFilesChanged, LoraSettingsUnusable)):
|
||||
elif isinstance(
|
||||
error,
|
||||
(
|
||||
LoraFilesChanged,
|
||||
LoraSettingsUnusable,
|
||||
EmptyReplacement,
|
||||
PairDependencyError,
|
||||
),
|
||||
):
|
||||
message = str(error)
|
||||
else:
|
||||
elif isinstance(error, OSError):
|
||||
# Reduced to its reason and the file's basename.
|
||||
message = self._public_error(error)
|
||||
else:
|
||||
message = _LORA_SCAN_FAILED
|
||||
if isinstance(
|
||||
error,
|
||||
(
|
||||
@@ -536,6 +548,7 @@ class LoraIntegrationMixin:
|
||||
except asyncio.CancelledError:
|
||||
interrupted = True
|
||||
except Exception:
|
||||
# the await after this loop raises it for the caller
|
||||
pass
|
||||
if interrupted:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""Maintenance routes: reviewed, read-only plans (#249 System S6).
|
||||
|
||||
``POST /prompt_manager/maintenance/preview {operation, options:{}}`` reviews
|
||||
one operation (``duplicate_prompts``, ``optimize``, ``unused_components`` or
|
||||
``missing_links``; ``database/maintenance.py``; ``duplicate_files``,
|
||||
one operation (``duplicate_prompts``, ``optimize``, ``unused_components``,
|
||||
``missing_links`` or ``pair_metadata``, #550; ``database/maintenance.py``;
|
||||
``duplicate_files``,
|
||||
``py/api/duplicate_files.py``; ``rotated_logs``, the rotated log copies,
|
||||
``py/api/log_cleanup.py``, which has its own apply job) off the event loop and
|
||||
answers a ``Plan``: ``{success, token, operation, expires_at, scope:{count,
|
||||
@@ -97,7 +98,11 @@ MAINTENANCE_TITLES = {
|
||||
"unused_components": "Unused prompt components",
|
||||
"missing_links": "Missing-file links",
|
||||
"workflow_storage": "Store each workflow once",
|
||||
# #550, beyond the concept: fills old Pair prompts' empty category and tags.
|
||||
"pair_metadata": "Pair category and tags",
|
||||
}
|
||||
# Operations whose job keeps every changed row for its report (#550).
|
||||
COMPLETE_RECORD = frozenset({"pair_metadata"})
|
||||
_OPERATION_ERROR = "operation must be one of: " + ", ".join(PREVIEW_OPERATIONS)
|
||||
MAINTENANCE_GROUPS = ("changed", "protected", "errors")
|
||||
MAX_TOKEN_LENGTH = 128
|
||||
@@ -281,9 +286,27 @@ def short_space_message(short):
|
||||
)
|
||||
|
||||
|
||||
def _changed_line(operation, count, marks=0):
|
||||
def _pair_metadata_line(result):
|
||||
count = result["changed"]
|
||||
noun = (
|
||||
"prompt from its saved pairs"
|
||||
if count == 1
|
||||
else "prompts from their saved pairs"
|
||||
)
|
||||
categories = result.get("categories", 0)
|
||||
tag_sets = result.get("tag_sets", 0)
|
||||
return (
|
||||
f"Filled {count:,} {noun} "
|
||||
f"({categories:,} {'category' if categories == 1 else 'categories'}, "
|
||||
f"{tag_sets:,} {'tag set' if tag_sets == 1 else 'tag sets'})"
|
||||
)
|
||||
|
||||
|
||||
def _changed_line(operation, count, marks=0, result=None):
|
||||
if operation == "optimize":
|
||||
return "Optimized the database"
|
||||
if operation == "pair_metadata":
|
||||
return _pair_metadata_line(result or {"changed": count})
|
||||
if operation == "duplicate_prompts":
|
||||
noun = "duplicate prompt" if count == 1 else "duplicate prompts"
|
||||
return f"Merged {count:,} {noun}"
|
||||
@@ -668,11 +691,31 @@ class MaintenanceRoutesMixin:
|
||||
rows = await self._run_in_executor(
|
||||
self._maintenance_rows, plan.operation, result["rows"]
|
||||
)
|
||||
job.record_counted("changed", result["changed"], rows)
|
||||
job.record_counted("protected", payload["protected"])
|
||||
line = _changed_line(plan.operation, result["changed"], result.get("marks", 0))
|
||||
job.record_counted(
|
||||
"changed",
|
||||
result["changed"],
|
||||
rows,
|
||||
complete=plan.operation in COMPLETE_RECORD,
|
||||
)
|
||||
if "kept" in result:
|
||||
# #550: each kept record with its reason, like the changed ones.
|
||||
kept = [
|
||||
{
|
||||
"path": "",
|
||||
"detail": f"Prompt {item['prompt_id']}: kept, "
|
||||
+ "; ".join(item["reasons"]),
|
||||
}
|
||||
for item in result["kept"]
|
||||
]
|
||||
job.record_counted("protected", len(kept), kept, complete=True)
|
||||
else:
|
||||
job.record_counted("protected", payload["protected"])
|
||||
line = _changed_line(
|
||||
plan.operation, result["changed"], result.get("marks", 0), result
|
||||
)
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
return None
|
||||
|
||||
def _maintenance_rows(self, operation, described):
|
||||
"""Outcome rows for the changed records (root-relative paths only)."""
|
||||
@@ -690,6 +733,8 @@ class MaintenanceRoutesMixin:
|
||||
}
|
||||
)
|
||||
return rows
|
||||
if operation == "pair_metadata":
|
||||
return [{"path": "", "detail": _filled_detail(item)} for item in described]
|
||||
if operation == "duplicate_prompts":
|
||||
return [
|
||||
{
|
||||
@@ -707,6 +752,16 @@ class MaintenanceRoutesMixin:
|
||||
]
|
||||
|
||||
|
||||
def _filled_detail(item):
|
||||
"""``Prompt 12: category “portrait”; tags a, b``: one filled prompt (#550)."""
|
||||
parts = []
|
||||
if "category" in item:
|
||||
parts.append(f"category “{item['category']}”")
|
||||
if item.get("tags"):
|
||||
parts.append("tags " + ", ".join(item["tags"]))
|
||||
return f"Prompt {item['prompt_id']}: " + "; ".join(parts)
|
||||
|
||||
|
||||
def backup_note(backup):
|
||||
"""The status line's backup part: the file name, or that there is none."""
|
||||
return f"Safety backup {backup}" if backup else NO_BACKUP
|
||||
|
||||
@@ -54,9 +54,25 @@ _SETTINGS_FILE_KEYS = {
|
||||
"AUTOTAG_EXISTING": ("autotag", "defaults", "existing"),
|
||||
"AUTOTAG_GENERAL_THRESHOLD": ("autotag", "defaults", "general_threshold"),
|
||||
"AUTOTAG_CHARACTER_THRESHOLD": ("autotag", "defaults", "character_threshold"),
|
||||
"AUTOTAG_TEMPLATE": ("autotag", "defaults", "template"),
|
||||
"AUTOTAG_SUGGESTION_LIMIT": ("autotag", "defaults", "suggestion_limit"),
|
||||
"AUTOTAG_SKIPPED_TAGS": ("autotag", "defaults", "skipped_tags"),
|
||||
# #431: the saved instructions and each engine's default, always written
|
||||
# together (server-side pairing).
|
||||
"AUTOTAG_INSTRUCTIONS": ("autotag", "defaults", "instructions"),
|
||||
"AUTOTAG_DEFAULT_INSTRUCTIONS": ("autotag", "defaults", "default_instructions"),
|
||||
}
|
||||
# The old single template: written only as the GGUF default's mirror for a
|
||||
# downgrade (#431 B1.1), deleted when that default is Built-in.
|
||||
_TEMPLATE_MIRROR_KEY = ("autotag", "defaults", "template")
|
||||
TEMPLATE_REPLACED_ERROR = (
|
||||
"LLM tag template was replaced by Saved instructions. Reload the page."
|
||||
)
|
||||
# Whole-list settings (#431): a save without ``expected_revision`` would merge
|
||||
# a stale copy over a list changed elsewhere, so it is refused.
|
||||
_REVISION_REQUIRED = frozenset(
|
||||
{"autotag_skipped_tags", "autotag_instructions", "autotag_default_instructions"}
|
||||
)
|
||||
REVISION_REQUIRED_ERROR = "Reload the page and try again."
|
||||
# The thumbnail settings a running Convert outdated job fixes (#317).
|
||||
_THUMBNAIL_SETTINGS = ("THUMBNAIL_FORMAT", "THUMBNAIL_QUALITY", "THUMBNAIL_EDGE")
|
||||
# Attributes that live on GalleryConfig; the others on PromptManagerConfig.
|
||||
@@ -137,13 +153,20 @@ def _settings_mutation(updates, new_roots):
|
||||
"""The config commit for a settings save: only the submitted keys change.
|
||||
|
||||
Every other section and key (integrations and the CivitAI key, database,
|
||||
unknown sections) is kept as it is on disk.
|
||||
unknown sections) is kept as it is on disk. A save of the instructions
|
||||
also writes the old ``template`` key as the GGUF default's mirror.
|
||||
"""
|
||||
|
||||
def mutate(config):
|
||||
for name, value in updates.items():
|
||||
*sections, key = _SETTINGS_FILE_KEYS[name]
|
||||
_set_nested(config, sections, key, value)
|
||||
_set_nested(config, sections, key, _jsonable(value))
|
||||
if "AUTOTAG_INSTRUCTIONS" in updates:
|
||||
*sections, key = _TEMPLATE_MIRROR_KEY
|
||||
mirror = _mirror_text(
|
||||
updates["AUTOTAG_INSTRUCTIONS"], updates["AUTOTAG_DEFAULT_INSTRUCTIONS"]
|
||||
)
|
||||
_set_nested(config, sections, key, _DELETE if mirror is None else mirror)
|
||||
if new_roots is not None:
|
||||
gallery = _section(config, "gallery")
|
||||
monitoring = _section(gallery, "monitoring")
|
||||
@@ -155,10 +178,37 @@ def _settings_mutation(updates, new_roots):
|
||||
return mutate
|
||||
|
||||
|
||||
# ``_set_nested``'s value that removes the key instead of setting it.
|
||||
_DELETE = object()
|
||||
|
||||
|
||||
def _jsonable(value):
|
||||
"""A runtime value as config.json holds it (tuples become lists)."""
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_jsonable(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {key: _jsonable(item) for key, item in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
def _mirror_text(entries, defaults):
|
||||
"""The GGUF default's text, or None when it is Built-in (#431 B1.1)."""
|
||||
from ..tagging_defaults import BUILTIN_INSTRUCTIONS_ID
|
||||
|
||||
for item in entries:
|
||||
if item["id"] == defaults["gguf"] != BUILTIN_INSTRUCTIONS_ID:
|
||||
return item["text"]
|
||||
return None
|
||||
|
||||
|
||||
def _set_nested(config, sections, key, value):
|
||||
"""``config[s1][s2]...[key] = value``, each section a fresh dict."""
|
||||
"""``config[s1][s2]...[key] = value``, each section a fresh dict;
|
||||
``_DELETE`` removes the key."""
|
||||
if not sections:
|
||||
config[key] = value
|
||||
if value is _DELETE:
|
||||
config.pop(key, None)
|
||||
else:
|
||||
config[key] = value
|
||||
return
|
||||
merged = _section(config, sections[0])
|
||||
_set_nested(merged, sections[1:], key, value)
|
||||
@@ -176,7 +226,7 @@ def _validate_autotag_settings(data):
|
||||
MAX_SUGGESTION_LIMIT,
|
||||
MODEL_TYPES,
|
||||
validate_saved_threshold,
|
||||
validate_template,
|
||||
validate_skipped_tags,
|
||||
)
|
||||
|
||||
updates = {}
|
||||
@@ -205,12 +255,7 @@ def _validate_autotag_settings(data):
|
||||
raise ValueError(f"{error}.") from None
|
||||
updates[key.upper()] = value
|
||||
if "autotag_template" in data:
|
||||
try:
|
||||
updates["AUTOTAG_TEMPLATE"] = validate_template(
|
||||
data["autotag_template"], "LLM tag template"
|
||||
)
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{error}.") from None
|
||||
raise ValueError(TEMPLATE_REPLACED_ERROR)
|
||||
if "autotag_suggestion_limit" in data:
|
||||
value = data["autotag_suggestion_limit"]
|
||||
if type(value) is not int or not 1 <= value <= MAX_SUGGESTION_LIMIT:
|
||||
@@ -219,9 +264,41 @@ def _validate_autotag_settings(data):
|
||||
f"{MAX_SUGGESTION_LIMIT}."
|
||||
)
|
||||
updates["AUTOTAG_SUGGESTION_LIMIT"] = value
|
||||
if "autotag_skipped_tags" in data:
|
||||
updates["AUTOTAG_SKIPPED_TAGS"] = tuple(
|
||||
validate_skipped_tags(data["autotag_skipped_tags"])
|
||||
)
|
||||
updates.update(_validate_instruction_settings(data))
|
||||
return updates
|
||||
|
||||
|
||||
def _validate_instruction_settings(data):
|
||||
"""Server-side pairing (#431): a save that carries either instruction key
|
||||
gets the other from the running configuration (which holds the migrated
|
||||
list), the defaults are checked against the merged list, and both are
|
||||
written. Empty when the save carries neither."""
|
||||
from ..config import AutotagConfig
|
||||
from ..tagging_defaults import (
|
||||
validate_default_instructions,
|
||||
validate_instructions,
|
||||
)
|
||||
|
||||
if not {"autotag_instructions", "autotag_default_instructions"} & set(data):
|
||||
return {}
|
||||
if "autotag_instructions" in data:
|
||||
entries = validate_instructions(data["autotag_instructions"])
|
||||
else:
|
||||
entries = [dict(item) for item in AutotagConfig.INSTRUCTIONS]
|
||||
defaults = data.get(
|
||||
"autotag_default_instructions", dict(AutotagConfig.DEFAULT_INSTRUCTIONS)
|
||||
)
|
||||
defaults = validate_default_instructions(defaults, entries)
|
||||
return {
|
||||
"AUTOTAG_INSTRUCTIONS": tuple(entries),
|
||||
"AUTOTAG_DEFAULT_INSTRUCTIONS": defaults,
|
||||
}
|
||||
|
||||
|
||||
async def _read_json(request):
|
||||
"""Parse a JSON object body through the package's capped reader."""
|
||||
from . import _read_json_body
|
||||
@@ -249,6 +326,7 @@ class SettingsRoutesMixin:
|
||||
"""Get current settings."""
|
||||
try:
|
||||
from ..config import AutotagConfig, PromptManagerConfig, GalleryConfig
|
||||
from ..tagging_defaults import DEFAULT_TAG_TEMPLATE
|
||||
|
||||
root_paths = [
|
||||
self._public_path(d) for d in GalleryConfig.MONITORING_DIRECTORIES
|
||||
@@ -281,8 +359,21 @@ class SettingsRoutesMixin:
|
||||
"autotag_character_threshold": (
|
||||
AutotagConfig.CHARACTER_THRESHOLD
|
||||
),
|
||||
"autotag_template": AutotagConfig.TEMPLATE,
|
||||
# Read-only for one release (#431): a page cached
|
||||
# from before the upgrade still renders.
|
||||
"autotag_template": AutotagConfig.default_instructions_text(
|
||||
"gguf"
|
||||
),
|
||||
"autotag_instructions": [
|
||||
dict(item) for item in AutotagConfig.INSTRUCTIONS
|
||||
],
|
||||
"autotag_default_instructions": dict(
|
||||
AutotagConfig.DEFAULT_INSTRUCTIONS
|
||||
),
|
||||
# Built-in's text, which System shows read-only.
|
||||
"autotag_builtin_instructions": DEFAULT_TAG_TEMPLATE,
|
||||
"autotag_suggestion_limit": AutotagConfig.SUGGESTION_LIMIT,
|
||||
"autotag_skipped_tags": list(AutotagConfig.SKIPPED_TAGS),
|
||||
"cpu_count": PromptManagerConfig.max_worker_threads(),
|
||||
"gallery_root_paths": root_paths,
|
||||
"gallery_root_path": root_paths[0] if root_paths else "",
|
||||
@@ -383,6 +474,10 @@ class SettingsRoutesMixin:
|
||||
{"success": False, "error": "expected_revision must be a string"},
|
||||
status=400,
|
||||
)
|
||||
if expected_revision is None and _REVISION_REQUIRED & set(data):
|
||||
return web.json_response(
|
||||
{"success": False, "error": REVISION_REQUIRED_ERROR}, status=400
|
||||
)
|
||||
try:
|
||||
updates = self._validate_settings(data)
|
||||
except ValueError as ve:
|
||||
@@ -511,6 +606,12 @@ class SettingsRoutesMixin:
|
||||
if name in _AUTOTAG_SETTINGS:
|
||||
setattr(AutotagConfig, name[len(_AUTOTAG_PREFIX) :], value)
|
||||
continue
|
||||
if "AUTOTAG_INSTRUCTIONS" in updates:
|
||||
# What config.json now holds as the old key (the mirror).
|
||||
AutotagConfig._LEGACY_TEMPLATE = AutotagConfig.mirror_template()
|
||||
for name, value in updates.items():
|
||||
if name in _AUTOTAG_SETTINGS:
|
||||
continue
|
||||
owner = GalleryConfig if name in _GALLERY_SETTINGS else PromptManagerConfig
|
||||
setattr(owner, name, value)
|
||||
restart_fields = []
|
||||
|
||||
@@ -8,7 +8,8 @@ previews behind those URLs are made here, in three ways:
|
||||
- **On first view.** ``GET /prompt_manager/images/serve/thumbnails/<rel>_thumb
|
||||
<ext>`` for a preview that does not exist yet is admitted on the event loop
|
||||
(``_PreviewGate``): at most ``ON_DEMAND_LIMIT`` previews are written at once,
|
||||
on this API's own small pool (never the default executor), one flight runs
|
||||
on this API's interactive image pool (``IMAGE_POOL_SIZE`` threads, shared
|
||||
with card facts and inspector reads; never the default executor), one flight runs
|
||||
per file, and the requests waiting are capped (``MAX_WAITING``); past the
|
||||
cap a request gets its fallback at once. Inside the admission the preview is
|
||||
mapped back to its media file under the same root (the request text is
|
||||
@@ -24,7 +25,8 @@ previews behind those URLs are made here, in three ways:
|
||||
- **On capture.** The image monitor hands every new output it settles to
|
||||
``queue_capture_thumbnail``, which queues its preview for one background
|
||||
worker thread (``AutoThumbnailer``). Each wake takes at most Worker threads
|
||||
(``WORKER_THREADS``) files and writes them on that many threads; enqueuing
|
||||
(``WORKER_THREADS``) files and writes them on the long-lived job pool
|
||||
(``utils.parallel``, shared with scans and thumbnail jobs); enqueuing
|
||||
is O(1), so the monitor and ComfyUI's queue never wait. A file already
|
||||
queued is not queued again, the queue is bounded (``MAX_QUEUED``) and a
|
||||
preview left out is made on its first view instead.
|
||||
@@ -49,7 +51,7 @@ write starts (after its slot) and again just before its rename.
|
||||
|
||||
Shutdown: ``close_auto_thumbnails`` stops the worker at a file boundary,
|
||||
kills a running ffmpeg, waits a bounded time (and logs when the wait ran
|
||||
out) and shuts the view pool down; after the app's cleanup
|
||||
out) and shuts the image pool down; after the app's cleanup
|
||||
(``final=True``) nothing starts a worker again. Importing this module again
|
||||
(a package reload) stops the workers the previous import left.
|
||||
"""
|
||||
@@ -60,7 +62,7 @@ import functools
|
||||
import io
|
||||
import os
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import CancelledError, ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
@@ -104,8 +106,11 @@ PLACEHOLDER_SIZE = (thumbnail_settings.DEFAULT_EDGE, thumbnail_settings.DEFAULT_
|
||||
PLACEHOLDER_QUALITY = 85
|
||||
# Source extensions in the order a shared preview name looks for them (R2).
|
||||
_SOURCE_ORDER = thumbnail_settings.IMAGE_SOURCE_ORDER + tuple(sorted(VIDEO_EXTENSIONS))
|
||||
# Previews views may write at once (the size of the view pool).
|
||||
ON_DEMAND_LIMIT = 3
|
||||
# Threads of the interactive image pool (#523): view previews, card facts and
|
||||
# inspector reads. Fixed, so image buffers stay in a few threads' arenas.
|
||||
IMAGE_POOL_SIZE = 3
|
||||
# Previews views may write at once (the whole image pool).
|
||||
ON_DEMAND_LIMIT = IMAGE_POOL_SIZE
|
||||
# View requests that may wait on the gate; one more gets its fallback at once.
|
||||
MAX_WAITING = 64
|
||||
# How long a pool thread waits for the capture worker's flight of the same file.
|
||||
@@ -124,7 +129,7 @@ AUTO_REFUSED_KINDS = frozenset(
|
||||
{RESTORE_JOB_KIND, DUPLICATE_CLEANUP_JOB_KIND, SOURCE_CHANGE_JOB_KIND}
|
||||
)
|
||||
WORKER_NAME = "PromptManagerThumbnailWorker"
|
||||
POOL_PREFIX = "pm-preview"
|
||||
POOL_PREFIX = "pm-image"
|
||||
# Identifies this import's workers: a reload stops the ones it did not start.
|
||||
_MODULE_TOKEN = object()
|
||||
|
||||
@@ -294,7 +299,7 @@ class AutoThumbnailer:
|
||||
``generate(src, dst, is_video, *, stop, guard, track)`` writes one
|
||||
preview and returns ``_generate_one``'s result dict; ``may_write()``
|
||||
says whether automatic previews may be written now. Views call
|
||||
``ensure`` on the view pool, the monitor calls ``enqueue`` from its
|
||||
``ensure`` on the image pool, the monitor calls ``enqueue`` from its
|
||||
thread.
|
||||
"""
|
||||
|
||||
@@ -491,12 +496,15 @@ class AutoThumbnailer:
|
||||
self._worker_running = False
|
||||
self._changed.notify_all()
|
||||
return
|
||||
take = min(len(self._queue), max(1, int(self._workers() or 1)))
|
||||
workers = max(1, int(self._workers() or 1))
|
||||
take = min(len(self._queue), workers)
|
||||
batch = [self._queue.popleft() for _ in range(take)]
|
||||
self._batch = len(batch)
|
||||
self._changed.notify_all()
|
||||
try:
|
||||
map_parallel(self._write_queued, batch, len(batch))
|
||||
# The job pool's size is the setting; a smaller batch uses
|
||||
# fewer of its threads (#523: one long-lived pool).
|
||||
map_parallel(self._write_queued, batch, workers)
|
||||
finally:
|
||||
with self._changed:
|
||||
self._batch = 0
|
||||
@@ -533,6 +541,7 @@ class AutoThumbnailer:
|
||||
try:
|
||||
process.kill()
|
||||
except OSError:
|
||||
# the process has already exited
|
||||
pass
|
||||
if worker is None:
|
||||
return True
|
||||
@@ -566,7 +575,7 @@ class _PreviewGate:
|
||||
"""Admission of view requests on the event loop (#249 I2).
|
||||
|
||||
Waiters are coroutines: at most ``limit`` previews are written at once
|
||||
on the view pool, one flight runs per key, and past ``MAX_WAITING``
|
||||
on the image pool, one flight runs per key, and past ``MAX_WAITING``
|
||||
waiting requests ``run`` answers ``OVER`` at once.
|
||||
"""
|
||||
|
||||
@@ -621,13 +630,17 @@ class _PreviewGate:
|
||||
flight.set_result(result)
|
||||
|
||||
|
||||
def _on_image_thread():
|
||||
return threading.current_thread().name.startswith(POOL_PREFIX + "_")
|
||||
|
||||
|
||||
class AutoThumbnailMixin:
|
||||
"""Automatic previews on the API: views, the monitor's queue, scans."""
|
||||
|
||||
_auto_thumbnailer = None
|
||||
_auto_thumbnails_shut = False
|
||||
_preview_gate_obj = None
|
||||
_preview_pool_obj = None
|
||||
_image_pool_obj = None
|
||||
_auto_thumbnailer_lock = threading.Lock()
|
||||
|
||||
def _auto_thumbnails(self):
|
||||
@@ -650,7 +663,7 @@ class AutoThumbnailMixin:
|
||||
self._auto_thumbnails_shut = True
|
||||
|
||||
def close_auto_thumbnails(self, final=False, timeout=None):
|
||||
"""Stop the capture worker and the view pool, and forget them.
|
||||
"""Stop the capture worker and the image pool, and forget them.
|
||||
|
||||
Bounded by ``timeout`` (see AutoThumbnailer.close). ``final`` (the
|
||||
package teardown) also keeps a late monitor event from starting a
|
||||
@@ -661,7 +674,7 @@ class AutoThumbnailMixin:
|
||||
if final:
|
||||
self._auto_thumbnails_shut = True
|
||||
thumbnailer, self._auto_thumbnailer = self._auto_thumbnailer, None
|
||||
pool, self._preview_pool_obj = self._preview_pool_obj, None
|
||||
pool, self._image_pool_obj = self._image_pool_obj, None
|
||||
self._preview_gate_obj = None
|
||||
closed = True
|
||||
if thumbnailer is not None:
|
||||
@@ -705,16 +718,67 @@ class AutoThumbnailMixin:
|
||||
gate = self._preview_gate_obj = _PreviewGate(ON_DEMAND_LIMIT, self.logger)
|
||||
return gate
|
||||
|
||||
def _preview_pool(self):
|
||||
with AutoThumbnailMixin._auto_thumbnailer_lock:
|
||||
if self._preview_pool_obj is None:
|
||||
self._preview_pool_obj = ThreadPoolExecutor(
|
||||
ON_DEMAND_LIMIT, thread_name_prefix=POOL_PREFIX
|
||||
)
|
||||
return self._preview_pool_obj
|
||||
def _image_pool(self):
|
||||
"""The interactive image pool (#523): IMAGE_POOL_SIZE threads for view
|
||||
previews (behind their gate), card facts and inspector reads, so
|
||||
image buffers never spread over the default executor's threads.
|
||||
|
||||
def _preview_pool_threads(self):
|
||||
pool = self._preview_pool_obj
|
||||
None after the package teardown (``close_auto_thumbnails(final=True)``):
|
||||
a late request then runs on the default executor (``run_in_executor``
|
||||
with None) and starts no thread the teardown no longer stops.
|
||||
"""
|
||||
with AutoThumbnailMixin._auto_thumbnailer_lock:
|
||||
if self._image_pool_obj is None and not self._auto_thumbnails_shut:
|
||||
self._image_pool_obj = ThreadPoolExecutor(
|
||||
IMAGE_POOL_SIZE, thread_name_prefix=POOL_PREFIX
|
||||
)
|
||||
return self._image_pool_obj
|
||||
|
||||
async def _run_image(self, func, *args, **kwargs):
|
||||
"""``_run_in_executor`` for interactive image work: the image pool.
|
||||
|
||||
After the teardown, or with the pool shut down in between (a test
|
||||
reset), the call runs on the default executor as before.
|
||||
"""
|
||||
call = functools.partial(func, *args, **kwargs)
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
future = loop.run_in_executor(self._image_pool(), call)
|
||||
except RuntimeError: # submit refused: the pool was shut down
|
||||
future = loop.run_in_executor(None, call)
|
||||
return await future
|
||||
|
||||
def _map_image(self, func, items):
|
||||
"""``func`` over *items* on the image pool, in order (blocking).
|
||||
|
||||
Called from an image pool thread, after the teardown, or with the
|
||||
pool shut down in between (its queued items are cancelled), the
|
||||
items it did not run run on the calling thread.
|
||||
"""
|
||||
items = list(items)
|
||||
pool = None if not items or _on_image_thread() else self._image_pool()
|
||||
if pool is None:
|
||||
return [func(item) for item in items]
|
||||
futures = []
|
||||
try:
|
||||
for item in items:
|
||||
futures.append(pool.submit(func, item))
|
||||
except RuntimeError:
|
||||
pass # shut down in between: the rest run here
|
||||
results = []
|
||||
for index, item in enumerate(items):
|
||||
future = futures[index] if index < len(futures) else None
|
||||
if future is None or future.cancelled():
|
||||
results.append(func(item))
|
||||
continue
|
||||
try:
|
||||
results.append(future.result())
|
||||
except CancelledError: # cancelled while this caller waited
|
||||
results.append(func(item))
|
||||
return results
|
||||
|
||||
def _image_pool_threads(self):
|
||||
pool = self._image_pool_obj
|
||||
return [] if pool is None else list(getattr(pool, "_threads", ()))
|
||||
|
||||
def _video_placeholder_bytes(self):
|
||||
@@ -729,7 +793,7 @@ class AutoThumbnailMixin:
|
||||
return await self._preview_gate().run(
|
||||
flight_key(Path(dst).parent, (Path(dst).name,)),
|
||||
functools.partial(self._ensure_view, src, dst, is_video),
|
||||
self._preview_pool(),
|
||||
self._image_pool(),
|
||||
)
|
||||
|
||||
def _ensure_view(self, src, dst, is_video):
|
||||
@@ -773,13 +837,13 @@ class AutoThumbnailMixin:
|
||||
The flight answers only the outcome (a history flight may be the one
|
||||
joined); the file it previews is derived here: the preview path is
|
||||
the request's own, and a fallback looks its source up again on the
|
||||
view pool (bounded by it, never the default executor).
|
||||
image pool (bounded by it, never the default executor).
|
||||
"""
|
||||
if _has_traversal(filepath) or not _is_preview_request(filepath):
|
||||
return None
|
||||
parts = tuple(filepath.split("/"))
|
||||
gate = self._preview_gate()
|
||||
pool = self._preview_pool()
|
||||
pool = self._image_pool()
|
||||
for root in output_dirs:
|
||||
root = Path(root)
|
||||
outcome = await gate.run(
|
||||
|
||||
@@ -153,6 +153,7 @@ class ThumbnailConvertMixin:
|
||||
job.log_line("Conversion failed · internal error")
|
||||
state = "failed"
|
||||
job.finish(state)
|
||||
await self._release_job_memory()
|
||||
|
||||
async def _convert_job_body(self, job, options):
|
||||
output_dirs = await self._run_in_executor(self._get_all_output_dirs)
|
||||
|
||||
@@ -158,6 +158,7 @@ def _unlink_quietly(path):
|
||||
try:
|
||||
Path(path).unlink()
|
||||
except FileNotFoundError:
|
||||
# already gone, which is what was wanted
|
||||
pass
|
||||
except OSError as exc:
|
||||
logger.debug(f"Could not remove temp file {Path(path).name}: {exc}")
|
||||
@@ -221,6 +222,7 @@ def shutdown_encoders():
|
||||
try:
|
||||
process.kill()
|
||||
except OSError:
|
||||
# the process has already exited
|
||||
pass
|
||||
|
||||
|
||||
@@ -359,10 +361,12 @@ def thumbnail_frame(img, edge):
|
||||
try:
|
||||
img.seek(0)
|
||||
except (EOFError, AttributeError, ValueError):
|
||||
# not seekable or a single frame: use the current one
|
||||
pass
|
||||
try:
|
||||
img.draft("RGB", (edge, edge)) # JPEG: decode at a reduced scale
|
||||
except (AttributeError, ValueError, OSError):
|
||||
# draft is only a speedup; the full decode below still works
|
||||
pass
|
||||
frame = img.convert("RGB") if img.mode != "RGB" else img.copy()
|
||||
frame.thumbnail((edge, edge), Image.Resampling.LANCZOS)
|
||||
|
||||
@@ -226,6 +226,7 @@ class ThumbnailJobsMixin:
|
||||
)
|
||||
state = "failed"
|
||||
job.finish(state)
|
||||
await self._release_job_memory()
|
||||
|
||||
async def _thumbnail_job_body(self, job, options):
|
||||
"""Plan, then generate in worker batches; returns the final state.
|
||||
|
||||
@@ -275,3 +275,4 @@ class WorkflowStorageRoutesMixin:
|
||||
job.record_counted("changed", 1)
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=line)
|
||||
return None
|
||||
|
||||
@@ -30,9 +30,9 @@ except ImportError:
|
||||
# The tag template and thresholds live with the saved tagging defaults
|
||||
# (#328), so config.py never imports PIL.
|
||||
try:
|
||||
from .tagging_defaults import DEFAULT_TAG_TEMPLATE
|
||||
from .tagging_defaults import DEFAULT_TAG_TEMPLATE, read_wd14_label_rows
|
||||
except ImportError:
|
||||
from py.tagging_defaults import DEFAULT_TAG_TEMPLATE
|
||||
from py.tagging_defaults import DEFAULT_TAG_TEMPLATE, read_wd14_label_rows
|
||||
|
||||
try:
|
||||
from ..utils import exit_signal
|
||||
@@ -559,7 +559,6 @@ class AutoTagService:
|
||||
# A missing onnxruntime is refused by load_model before this runs
|
||||
# (#424); a ModuleNotFoundError here becomes the same refusal.
|
||||
import onnxruntime as ort
|
||||
import csv
|
||||
|
||||
config = MODELS[model_type]
|
||||
model_dir = self.models_dir / config["subdir"]
|
||||
@@ -576,12 +575,8 @@ class AutoTagService:
|
||||
|
||||
session = ort.InferenceSession(str(model_path), providers=providers)
|
||||
|
||||
# Read tag list from CSV
|
||||
tags_list = []
|
||||
with open(tags_path, "r", encoding="utf-8") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
tags_list.append(row)
|
||||
# Read tag list from CSV (the reader the Skipped tags preview shares)
|
||||
tags_list = read_wd14_label_rows(tags_path)
|
||||
|
||||
self.logger.info(
|
||||
f"WD14 model loaded ({len(tags_list)} tags, "
|
||||
@@ -992,6 +987,7 @@ class AutoTagService:
|
||||
prompt: Optional[str] = None,
|
||||
general_threshold: Optional[float] = None,
|
||||
character_threshold: Optional[float] = None,
|
||||
skip: Any = None,
|
||||
) -> List[Tuple[str, Optional[float]]]:
|
||||
"""The top ``limit`` suggestions as ``(name, confidence)`` pairs.
|
||||
|
||||
@@ -1000,7 +996,9 @@ class AutoTagService:
|
||||
``None``. Names are distinct case-insensitively (first one wins).
|
||||
A name Save would reject (over 50 characters, a control character
|
||||
such as a newline, a comma) is skipped before the dedupe and the
|
||||
``limit`` count. ``prompt`` is used for this call only;
|
||||
``limit`` count. A name ``skip`` matches (a ``SkipMatcher``, the
|
||||
Skipped tags compiled for this request or job, #431) is dropped next,
|
||||
so it uses up no slot. ``prompt`` is used for this call only;
|
||||
``custom_prompt`` is never changed.
|
||||
"""
|
||||
if type(limit) is not int or not 1 <= limit <= MAX_SUGGESTION_LIMIT:
|
||||
@@ -1017,6 +1015,8 @@ class AutoTagService:
|
||||
name = validate_vocabulary_tag_name(raw)
|
||||
except ValueError:
|
||||
continue
|
||||
if skip is not None and skip.matches(name):
|
||||
continue
|
||||
key = name.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
|
||||
@@ -30,6 +30,7 @@ routes = server_instance.routes
|
||||
# Extension info
|
||||
extension_uri = None # Will be set in __init__.py
|
||||
|
||||
import hashlib # noqa: E402
|
||||
import os # noqa: E402
|
||||
from typing import Any, Dict, List, Optional, Tuple # noqa: E402
|
||||
|
||||
@@ -587,8 +588,15 @@ try:
|
||||
MODEL_TYPES,
|
||||
WD14_MODEL_TYPES,
|
||||
validate_saved_threshold,
|
||||
validate_skipped_tags,
|
||||
validate_template,
|
||||
validate_threshold,
|
||||
BUILTIN_INSTRUCTIONS_ID,
|
||||
INSTRUCTION_ENGINES,
|
||||
MAX_SAVED_INSTRUCTIONS,
|
||||
builtin_instructions,
|
||||
validate_default_instructions,
|
||||
validate_instructions,
|
||||
)
|
||||
except ImportError:
|
||||
from py.tagging_defaults import ( # noqa: F401 - re-exported
|
||||
@@ -600,8 +608,15 @@ except ImportError:
|
||||
MODEL_TYPES,
|
||||
WD14_MODEL_TYPES,
|
||||
validate_saved_threshold,
|
||||
validate_skipped_tags,
|
||||
validate_template,
|
||||
validate_threshold,
|
||||
BUILTIN_INSTRUCTIONS_ID,
|
||||
INSTRUCTION_ENGINES,
|
||||
MAX_SAVED_INSTRUCTIONS,
|
||||
builtin_instructions,
|
||||
validate_default_instructions,
|
||||
validate_instructions,
|
||||
)
|
||||
|
||||
|
||||
@@ -617,6 +632,31 @@ def _valid_existing(value: Any) -> str:
|
||||
return value
|
||||
|
||||
|
||||
def _valid_skipped_tags(value: Any) -> Tuple[str, ...]:
|
||||
return tuple(validate_skipped_tags(value))
|
||||
|
||||
|
||||
def _reason(error: ValueError) -> str:
|
||||
"""A settings sentence as a load warning's reason ("entry 2 is empty")."""
|
||||
text = str(error).rstrip(".")
|
||||
for prefix in ("Skipped tags", "Saved instructions"):
|
||||
if text.startswith(prefix):
|
||||
return text.removeprefix(prefix).lstrip(": ")
|
||||
return text
|
||||
|
||||
|
||||
def _text_id(text: str) -> str:
|
||||
"""A migrated entry's id: deterministic, so every load gives the same."""
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:12]
|
||||
|
||||
|
||||
def _fallback_warning(key: str, error: ValueError, default: Any) -> str:
|
||||
"""The one load warning of a saved tagging default that fails its check."""
|
||||
if key == "skipped_tags":
|
||||
return f"autotag.defaults.skipped_tags {_reason(error)}; using no skipped tags"
|
||||
return f"autotag.defaults.{key} {error}; using {default!r}"
|
||||
|
||||
|
||||
def _valid_suggestion_limit(value: Any) -> int:
|
||||
if type(value) is not int or not 1 <= value <= MAX_SUGGESTION_LIMIT:
|
||||
raise ValueError(f"must be an integer from 1 to {MAX_SUGGESTION_LIMIT}")
|
||||
@@ -637,8 +677,18 @@ class AutotagConfig:
|
||||
EXISTING = "skip"
|
||||
GENERAL_THRESHOLD = 0.35
|
||||
CHARACTER_THRESHOLD = 0.85
|
||||
TEMPLATE = DEFAULT_TAG_TEMPLATE
|
||||
SUGGESTION_LIMIT = DEFAULT_SUGGESTION_LIMIT
|
||||
SKIPPED_TAGS: Tuple[str, ...] = ()
|
||||
# #431: the saved instructions (Built-in is never stored) and the default
|
||||
# of each language-model engine, by id.
|
||||
INSTRUCTIONS: Tuple[Dict[str, str], ...] = ()
|
||||
DEFAULT_INSTRUCTIONS: Dict[str, str] = {
|
||||
"gguf": BUILTIN_INSTRUCTIONS_ID,
|
||||
"hf": BUILTIN_INSTRUCTIONS_ID,
|
||||
}
|
||||
# The old single template as read at load (None when absent or invalid);
|
||||
# only the migration reads it. Saves keep it as the GGUF default's mirror.
|
||||
_LEGACY_TEMPLATE: Optional[str] = None
|
||||
|
||||
# config.json key -> (attribute, validator, default)
|
||||
_KEYS = {
|
||||
@@ -654,12 +704,13 @@ class AutotagConfig:
|
||||
lambda v: validate_saved_threshold("value", v),
|
||||
0.85,
|
||||
),
|
||||
"template": ("TEMPLATE", validate_template, DEFAULT_TAG_TEMPLATE),
|
||||
"suggestion_limit": (
|
||||
"SUGGESTION_LIMIT",
|
||||
_valid_suggestion_limit,
|
||||
DEFAULT_SUGGESTION_LIMIT,
|
||||
),
|
||||
# #431: words whose suggested tags are never offered.
|
||||
"skipped_tags": ("SKIPPED_TAGS", _valid_skipped_tags, ()),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -672,17 +723,142 @@ class AutotagConfig:
|
||||
type(value) is int and 1 <= value <= MAX_SUGGESTION_LIMIT
|
||||
) # bool, float and str are rejected
|
||||
|
||||
@classmethod
|
||||
def all_instructions(cls) -> List[Dict[str, str]]:
|
||||
"""Built-in first, then the saved entries (copies)."""
|
||||
return [builtin_instructions()] + [dict(item) for item in cls.INSTRUCTIONS]
|
||||
|
||||
@classmethod
|
||||
def default_instructions_text(cls, engine: str) -> str:
|
||||
"""The text of ``engine``'s (``gguf`` or ``hf``) default instructions."""
|
||||
if engine not in INSTRUCTION_ENGINES:
|
||||
raise ValueError(f"{engine} takes no instructions")
|
||||
wanted = cls.DEFAULT_INSTRUCTIONS.get(engine, BUILTIN_INSTRUCTIONS_ID)
|
||||
for item in cls.INSTRUCTIONS:
|
||||
if item["id"] == wanted:
|
||||
return item["text"]
|
||||
return DEFAULT_TAG_TEMPLATE
|
||||
|
||||
@classmethod
|
||||
def mirror_template(cls) -> Optional[str]:
|
||||
"""What an older build reads as its one template (#431 B1.1): the
|
||||
GGUF default's text, or None (no key) when that default is Built-in."""
|
||||
if cls.DEFAULT_INSTRUCTIONS.get("gguf") == BUILTIN_INSTRUCTIONS_ID:
|
||||
return None
|
||||
return cls.default_instructions_text("gguf")
|
||||
|
||||
@classmethod
|
||||
def get_config(cls) -> Dict[str, Any]:
|
||||
"""The saved defaults; the template only when it is not the built-in
|
||||
one, so a later built-in template still reaches who never set one."""
|
||||
"""The saved defaults; ``template`` only as the GGUF default's mirror."""
|
||||
defaults = {
|
||||
key: getattr(cls, attr) for key, (attr, _v, _d) in cls._KEYS.items()
|
||||
}
|
||||
if defaults["template"] == DEFAULT_TAG_TEMPLATE:
|
||||
del defaults["template"]
|
||||
defaults["skipped_tags"] = list(defaults["skipped_tags"])
|
||||
defaults["instructions"] = [dict(item) for item in cls.INSTRUCTIONS]
|
||||
defaults["default_instructions"] = dict(cls.DEFAULT_INSTRUCTIONS)
|
||||
mirror = cls.mirror_template()
|
||||
if mirror is not None:
|
||||
defaults["template"] = mirror
|
||||
return {"defaults": defaults}
|
||||
|
||||
@classmethod
|
||||
def _load_instructions(cls, defaults: Dict[str, Any]) -> None:
|
||||
"""Each instruction key alone, then the cross-key default check, then
|
||||
the in-memory migration of the old template (#431). Nothing is
|
||||
written here, so the revision never changes at load."""
|
||||
cls._LEGACY_TEMPLATE = None
|
||||
if "template" in defaults:
|
||||
try:
|
||||
cls._LEGACY_TEMPLATE = validate_template(defaults["template"])
|
||||
except ValueError as error:
|
||||
config_logger.warning(
|
||||
f"autotag.defaults.template {error}; using the built-in one"
|
||||
)
|
||||
entries: List[Dict[str, str]] = []
|
||||
damaged = False
|
||||
if "instructions" in defaults:
|
||||
try:
|
||||
entries = validate_instructions(defaults["instructions"])
|
||||
except ValueError as error:
|
||||
damaged = True
|
||||
config_logger.warning(
|
||||
f"autotag.defaults.instructions {_reason(error)}; "
|
||||
"using Built-in only"
|
||||
)
|
||||
chosen = {engine: BUILTIN_INSTRUCTIONS_ID for engine in INSTRUCTION_ENGINES}
|
||||
stored = defaults.get("default_instructions")
|
||||
if damaged:
|
||||
# Both defaults fall back with the list (its one warning says so);
|
||||
# the old template is migrated as on a first upgrade.
|
||||
pass
|
||||
elif "default_instructions" in defaults and (
|
||||
not isinstance(stored, dict) or set(stored) != set(INSTRUCTION_ENGINES)
|
||||
):
|
||||
config_logger.warning(
|
||||
"autotag.defaults.default_instructions must name gguf and hf; "
|
||||
"using Built-in"
|
||||
)
|
||||
elif isinstance(stored, dict):
|
||||
ids = {item["id"] for item in entries}
|
||||
for engine in INSTRUCTION_ENGINES:
|
||||
value = stored[engine]
|
||||
if isinstance(value, str) and (
|
||||
value == BUILTIN_INSTRUCTIONS_ID or value in ids
|
||||
):
|
||||
chosen[engine] = value
|
||||
else:
|
||||
config_logger.warning(
|
||||
f"autotag.defaults.default_instructions.{engine} names no "
|
||||
"saved instructions; using Built-in"
|
||||
)
|
||||
cls._migrate_template(
|
||||
entries,
|
||||
chosen,
|
||||
first=damaged or "instructions" not in defaults,
|
||||
keep_defaults=not damaged and "default_instructions" in defaults,
|
||||
)
|
||||
cls.INSTRUCTIONS = tuple(entries)
|
||||
cls.DEFAULT_INSTRUCTIONS = chosen
|
||||
|
||||
@classmethod
|
||||
def _migrate_template(cls, entries, chosen, *, first, keep_defaults) -> None:
|
||||
"""Bring a template saved by an older build in as a named entry."""
|
||||
legacy = cls._LEGACY_TEMPLATE
|
||||
if legacy is None or legacy == DEFAULT_TAG_TEMPLATE:
|
||||
return
|
||||
if first:
|
||||
# The first upgrade: it becomes "Custom template", the default for
|
||||
# both engines unless defaults are stored already (20A).
|
||||
entries.append(
|
||||
{"id": _text_id(legacy), "name": "Custom template", "text": legacy}
|
||||
)
|
||||
if not keep_defaults:
|
||||
for engine in INSTRUCTION_ENGINES:
|
||||
chosen[engine] = entries[-1]["id"]
|
||||
return
|
||||
if any(item["text"] == legacy for item in entries):
|
||||
return # the mirror of a saved default
|
||||
# Edited on an older build after an upgrade (B1.2): import it.
|
||||
if len(entries) >= MAX_SAVED_INSTRUCTIONS:
|
||||
config_logger.warning(
|
||||
"The tagging template edited on an older version was not imported: "
|
||||
f"the saved instructions are full ({MAX_SAVED_INSTRUCTIONS})"
|
||||
)
|
||||
return
|
||||
names = {item["name"].casefold() for item in entries}
|
||||
number = 2
|
||||
while f"custom template ({number})" in names:
|
||||
number += 1
|
||||
name = f"Custom template ({number})"
|
||||
ids = {item["id"] for item in entries}
|
||||
new_id, n = _text_id(legacy), 2
|
||||
while new_id in ids:
|
||||
new_id, n = _text_id(f"{legacy}\n{n}"), n + 1
|
||||
entries.append({"id": new_id, "name": name, "text": legacy})
|
||||
config_logger.info(
|
||||
"Imported the tagging template edited on an older version as " f"'{name}'"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def update_config(cls, new_config: Any) -> None:
|
||||
if not isinstance(new_config, dict):
|
||||
@@ -698,12 +874,9 @@ class AutotagConfig:
|
||||
try:
|
||||
setattr(cls, attr, validate(defaults[key]))
|
||||
except ValueError as error:
|
||||
config_logger.warning(
|
||||
f"autotag.defaults.{key} {error}; using {default!r}"
|
||||
if key != "template"
|
||||
else f"autotag.defaults.template {error}; using the built-in one"
|
||||
)
|
||||
config_logger.warning(_fallback_warning(key, error, default))
|
||||
setattr(cls, attr, default)
|
||||
cls._load_instructions(defaults)
|
||||
|
||||
|
||||
class PromptManagerConfig:
|
||||
|
||||
@@ -109,6 +109,7 @@ def _write_atomically(path: str, payload: bytes) -> None:
|
||||
try:
|
||||
os.unlink(temp_path)
|
||||
except OSError:
|
||||
# nothing to clean up; the original error is raised below
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
@@ -642,6 +642,7 @@ def layout_summary(layout: LoraManagerLayout, home: Path) -> Dict:
|
||||
relative = Path(settings_file).relative_to(home)
|
||||
settings_file = os.path.join("~", str(relative))
|
||||
except ValueError:
|
||||
# outside the home folder: show the full path
|
||||
pass
|
||||
return {
|
||||
"source": layout.source,
|
||||
|
||||
@@ -585,6 +585,7 @@ def _read_capped(resp, cap: int) -> Optional[bytes]:
|
||||
if int(declared) > cap:
|
||||
return None
|
||||
except ValueError:
|
||||
# a bad header says nothing; the read below enforces the cap
|
||||
pass
|
||||
|
||||
chunks = []
|
||||
@@ -707,6 +708,7 @@ def _save_atomically(image, local_path: Path) -> None:
|
||||
try:
|
||||
os.remove(temp)
|
||||
except OSError:
|
||||
# nothing to clean up; the original error is raised below
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
@@ -4,8 +4,13 @@ Server-free on purpose: ``py.api`` must import without ComfyUI's ``server``
|
||||
module, which ``py/config.py`` needs; config re-exports all of this.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import math
|
||||
from typing import Any
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import unicodedata
|
||||
from typing import Any, Dict, Iterable, List, Set, Tuple
|
||||
|
||||
# The tagging engines (``py/autotag.py`` MODELS) and the options they take.
|
||||
WD14_MODEL_TYPES = frozenset({"wd14-swinv2", "wd14-vit"})
|
||||
@@ -88,3 +93,230 @@ def validate_template(value: Any, name: str = "instructions") -> str:
|
||||
if len(encoded) > MAX_INSTRUCTION_BYTES:
|
||||
raise ValueError(f"{name} {'are' if name == 'instructions' else 'is'} too long")
|
||||
return value
|
||||
|
||||
|
||||
# Skipped tags (#431): words whose suggested tags are never offered.
|
||||
MAX_SKIPPED_TAGS = 200
|
||||
MAX_SKIPPED_TAG_CHARS = 50
|
||||
_WORD_SEPARATORS = re.compile(r"[ _-]+")
|
||||
_FORBIDDEN_IN_ENTRY = re.compile(r"[,\x00-\x1f\x7f-\x9f\u2028\u2029]")
|
||||
|
||||
|
||||
def tag_words(text: str) -> Tuple[str, ...]:
|
||||
"""The words of a tag or a skipped entry: NFC, case-folded, split on runs
|
||||
of space, ``_`` and ``-``. Every other character belongs to a word."""
|
||||
folded = unicodedata.normalize("NFC", text).casefold()
|
||||
return tuple(word for word in _WORD_SEPARATORS.split(folded) if word)
|
||||
|
||||
|
||||
def validate_skipped_tags(value: Any) -> List[str]:
|
||||
"""The Skipped tags list, each entry trimmed, or ValueError with the
|
||||
sentence the System page shows (entries are numbered from 1)."""
|
||||
if not isinstance(value, list):
|
||||
raise ValueError("Skipped tags must be a list of words.")
|
||||
if len(value) > MAX_SKIPPED_TAGS:
|
||||
raise ValueError(f"Skipped tags: at most {MAX_SKIPPED_TAGS} entries.")
|
||||
entries: List[str] = []
|
||||
seen: Dict[Tuple[str, ...], int] = {}
|
||||
for number, raw in enumerate(value, 1):
|
||||
if not isinstance(raw, str):
|
||||
raise ValueError(f"Skipped tags: entry {number} must be text.")
|
||||
entry = raw.strip()
|
||||
if not entry:
|
||||
raise ValueError(f"Skipped tags: entry {number} is empty.")
|
||||
if len(entry) > MAX_SKIPPED_TAG_CHARS:
|
||||
raise ValueError(
|
||||
f"Skipped tags: entry {number} must be at most "
|
||||
f"{MAX_SKIPPED_TAG_CHARS} characters."
|
||||
)
|
||||
if _FORBIDDEN_IN_ENTRY.search(entry):
|
||||
raise ValueError(
|
||||
f"Skipped tags: entry {number} must not contain a comma "
|
||||
"or a line break."
|
||||
)
|
||||
words = tag_words(entry)
|
||||
if not words:
|
||||
raise ValueError(f"Skipped tags: entry {number} is empty.")
|
||||
if words in seen:
|
||||
raise ValueError(
|
||||
f"Skipped tags: entry {number} repeats entry {seen[words]}."
|
||||
)
|
||||
seen[words] = number
|
||||
entries.append(entry)
|
||||
return entries
|
||||
|
||||
|
||||
class SkipMatcher:
|
||||
"""Whether a suggested tag contains a skipped entry's words, consecutive
|
||||
and whole (#431, 17B). Compiled once per request or job; the cost of a
|
||||
check depends on the tag's words, not on the number of entries."""
|
||||
|
||||
def __init__(self, entries: Iterable[str] = ()):
|
||||
self._entries = frozenset(filter(None, (tag_words(e) for e in entries)))
|
||||
self._longest = max((len(words) for words in self._entries), default=0)
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
return bool(self._entries)
|
||||
|
||||
def matches(self, name: str) -> bool:
|
||||
if not self._entries:
|
||||
return False
|
||||
words = tag_words(name)
|
||||
for start in range(len(words)):
|
||||
stop = min(len(words), start + self._longest)
|
||||
for end in range(start + 1, stop + 1):
|
||||
if words[start:end] in self._entries:
|
||||
return True
|
||||
return False
|
||||
|
||||
def counting(self) -> "CountingSkip":
|
||||
"""A per-request view that also remembers which names it hid."""
|
||||
return CountingSkip(self)
|
||||
|
||||
|
||||
class CountingSkip:
|
||||
"""A :class:`SkipMatcher` that records the distinct names it hid, so the
|
||||
Tag image dialog can say how many skipped tags were not suggested."""
|
||||
|
||||
def __init__(self, matcher: SkipMatcher):
|
||||
self._matcher = matcher
|
||||
self.hidden: Set[str] = set()
|
||||
|
||||
def matches(self, name: str) -> bool:
|
||||
if self._matcher.matches(name):
|
||||
self.hidden.add(name.casefold())
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# A WD14 label list's tags the engine can suggest: general (0) and character (4).
|
||||
WD14_SUGGESTED_CATEGORIES = frozenset({"0", "4"})
|
||||
_label_cache: Dict[str, Tuple[Tuple[int, int], Tuple[str, ...]]] = {}
|
||||
_label_cache_lock = threading.Lock()
|
||||
|
||||
|
||||
def read_wd14_label_rows(path) -> List[Dict[str, str]]:
|
||||
"""Every row of a WD14 ``selected_tags.csv``, read as UTF-8 and closed
|
||||
before returning (the handle never outlives the call, for Windows)."""
|
||||
with open(path, "r", encoding="utf-8", newline="") as handle:
|
||||
return list(csv.DictReader(handle))
|
||||
|
||||
|
||||
def wd14_label_names(path) -> Tuple[str, ...]:
|
||||
"""The names a WD14 label list can suggest (categories 0 and 4), cached
|
||||
by ``(mtime_ns, size)``. OSError (missing) and ValueError (unreadable)
|
||||
propagate; the caller skips that list."""
|
||||
key = os.fspath(path)
|
||||
stat = os.stat(key)
|
||||
stamp = (stat.st_mtime_ns, stat.st_size)
|
||||
with _label_cache_lock:
|
||||
cached = _label_cache.get(key)
|
||||
if cached is not None and cached[0] == stamp:
|
||||
return cached[1]
|
||||
try:
|
||||
rows = read_wd14_label_rows(key)
|
||||
except csv.Error as exc: # a damaged list (a NUL byte, an oversized field)
|
||||
raise ValueError(f"unreadable label list: {exc}") from None
|
||||
names = tuple(
|
||||
name
|
||||
for row in rows
|
||||
if row.get("category", "0") in WD14_SUGGESTED_CATEGORIES
|
||||
and (name := (row.get("name") or "").strip())
|
||||
)
|
||||
with _label_cache_lock:
|
||||
_label_cache[key] = (stamp, names)
|
||||
return names
|
||||
|
||||
|
||||
# Saved instructions (#431): named language-model instructions with a default
|
||||
# per engine. Built-in is never stored; readers put it first.
|
||||
BUILTIN_INSTRUCTIONS_ID = "builtin"
|
||||
BUILTIN_INSTRUCTIONS_NAME = "Built-in"
|
||||
MAX_SAVED_INSTRUCTIONS = 20
|
||||
MAX_INSTRUCTION_NAME_CHARS = 60
|
||||
INSTRUCTION_ENGINES = ("gguf", "hf")
|
||||
ENGINE_LABELS = {"gguf": "GGUF JoyCaption", "hf": "HuggingFace JoyCaption"}
|
||||
_INSTRUCTION_ID = re.compile(r"[0-9a-f]{12}")
|
||||
_CONTROL = re.compile(r"[\x00-\x1f\x7f-\x9f
]")
|
||||
|
||||
|
||||
def builtin_instructions() -> Dict[str, str]:
|
||||
"""The read-only Built-in entry, always the shipped template."""
|
||||
return {
|
||||
"id": BUILTIN_INSTRUCTIONS_ID,
|
||||
"name": BUILTIN_INSTRUCTIONS_NAME,
|
||||
"text": DEFAULT_TAG_TEMPLATE,
|
||||
}
|
||||
|
||||
|
||||
def _instruction_entry(raw: Any, number: int) -> Dict[str, str]:
|
||||
prefix = f"Saved instructions: entry {number}"
|
||||
if (
|
||||
not isinstance(raw, dict)
|
||||
or set(raw) != {"id", "name", "text"}
|
||||
or not isinstance(raw["id"], str)
|
||||
or _INSTRUCTION_ID.fullmatch(raw["id"]) is None
|
||||
):
|
||||
raise ValueError(f"{prefix} has an invalid id.")
|
||||
name = raw["name"].strip() if isinstance(raw["name"], str) else ""
|
||||
if not name:
|
||||
raise ValueError(f"{prefix} needs a name.")
|
||||
if len(name) > MAX_INSTRUCTION_NAME_CHARS:
|
||||
raise ValueError(
|
||||
f"{prefix} name must be at most {MAX_INSTRUCTION_NAME_CHARS} characters."
|
||||
)
|
||||
if _CONTROL.search(name):
|
||||
raise ValueError(f"{prefix} name must not contain a line break.")
|
||||
if name.casefold() == BUILTIN_INSTRUCTIONS_NAME.casefold():
|
||||
raise ValueError("Saved instructions: the name “Built-in” is reserved.")
|
||||
try:
|
||||
text = validate_template(raw["text"], f"{prefix} text")
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{error}.") from None
|
||||
return {"id": raw["id"], "name": name, "text": text}
|
||||
|
||||
|
||||
def validate_instructions(value: Any) -> List[Dict[str, str]]:
|
||||
"""The saved entries ``[{id, name, text}]`` in order (names trimmed), or
|
||||
ValueError with the sentence System shows (Built-in is not counted)."""
|
||||
if not isinstance(value, list):
|
||||
raise ValueError("Saved instructions must be a list.")
|
||||
if len(value) > MAX_SAVED_INSTRUCTIONS:
|
||||
raise ValueError(
|
||||
f"Saved instructions: at most {MAX_SAVED_INSTRUCTIONS} entries besides "
|
||||
"Built-in."
|
||||
)
|
||||
entries: List[Dict[str, str]] = []
|
||||
ids: Dict[str, int] = {}
|
||||
names: Dict[str, int] = {}
|
||||
for number, raw in enumerate(value, 1):
|
||||
item = _instruction_entry(raw, number)
|
||||
if item["id"] in ids:
|
||||
raise ValueError(
|
||||
f"Saved instructions: entry {number} repeats the id of entry "
|
||||
f"{ids[item['id']]}."
|
||||
)
|
||||
key = item["name"].casefold()
|
||||
if key in names:
|
||||
raise ValueError(
|
||||
f"Saved instructions: entry {number} name “{item['name']}” is "
|
||||
"already used."
|
||||
)
|
||||
ids[item["id"]] = number
|
||||
names[key] = number
|
||||
entries.append(item)
|
||||
return entries
|
||||
|
||||
|
||||
def validate_default_instructions(value: Any, entries) -> Dict[str, str]:
|
||||
"""``{gguf: id, hf: id}``, each Built-in or an id of ``entries``."""
|
||||
if not isinstance(value, dict) or set(value) != set(INSTRUCTION_ENGINES):
|
||||
raise ValueError("Default instructions must name GGUF and HuggingFace.")
|
||||
ids = {BUILTIN_INSTRUCTIONS_ID} | {item["id"] for item in entries}
|
||||
for engine in INSTRUCTION_ENGINES:
|
||||
if not isinstance(value[engine], str) or value[engine] not in ids:
|
||||
raise ValueError(
|
||||
f"Default for {ENGINE_LABELS[engine]} must be one of the saved "
|
||||
"instructions."
|
||||
)
|
||||
return {engine: value[engine] for engine in INSTRUCTION_ENGINES}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "promptmanager"
|
||||
description = "A powerful ComfyUI custom node that extends the standard text encoder with persistent prompt storage, advanced search capabilities, and an automatic image gallery system using SQLite."
|
||||
version = "3.3.1"
|
||||
version = "3.4.1"
|
||||
license = {file = "LICENSE"}
|
||||
requires-python = ">=3.12"
|
||||
dependencies = ["watchdog>=2.1.0", "Pillow>=8.0.0"]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Runtime dependencies for PromptManager.
|
||||
# aiohttp, numpy and torch come from ComfyUI itself and are not listed here.
|
||||
watchdog>=2.1.0 # Output directory monitoring
|
||||
watchdog>=2.1.0 # Output directory monitoring
|
||||
Pillow>=8.0.0 # Image metadata extraction and thumbnails
|
||||
|
||||
@@ -6,7 +6,6 @@ import { THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate
|
||||
// app.js 1428–1432) and its backup box with the concept's System `.notice`; the differences are D55. The job view
|
||||
// has no concept counterpart: it is compared with the approved mockup `activity-scan.html` (a local, ignored file),
|
||||
// whose structure and CSS values are restated below (D56). Each comparison is shown to notice a mutation.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
|
||||
@@ -320,3 +320,44 @@ test('copy toast and fallback dialog',async({page})=>{
|
||||
await expect(fallback).toContainText('Automatic clipboard access requires a secure browser context.');
|
||||
await page.keyboard.press('Escape');await expect(fallback).toBeHidden();await expect(copy).toBeFocused();
|
||||
});
|
||||
|
||||
// #542 D191: "All N eligible nodes" previews every node, sends one after another and names each refusal.
|
||||
test('All eligible nodes previews each node, sends in node id order and names a refusal',async({page,context},testInfo)=>{
|
||||
const host=await context.newPage();await host.goto('/prompt_manager/workspace#/overview');
|
||||
await host.evaluate(async()=>{
|
||||
const {createBridgeHost}=await import('/prompt_manager/js/shared/toolkit/prompt-bridge.js');
|
||||
const targets=['12','3','7'].map(id=>({id,label:id==='7'?'Edited in ComfyUI':'Prompt Manager (+/-)',type:'PromptManagerPair',roles:['positive','negative']}));
|
||||
window.hostValues=Object.fromEntries(targets.map(target=>[target.id,{positive:`Existing ${target.id}`,negative:'Existing negative'}]));window.applied=[];
|
||||
window.fixtureHost=createBridgeHost({origin:location.origin,namespace:'/',label:'Test ComfyUI',adapter:{
|
||||
listTargets:()=>targets,
|
||||
preview:(id,roles)=>{const target=targets.find(item=>item.id===id);return{preview:{target,fields:Object.entries(roles).map(([role,after])=>({role,before:window.hostValues[id][role],after}))},snapshot:{id,roles}};},
|
||||
apply:({id,roles})=>{if(id==='7')throw new Error('Workflow or prompt changed. Preview again.');Object.assign(window.hostValues[id],roles);window.applied.push(id);return{targetId:id,values:roles};},
|
||||
}});
|
||||
});
|
||||
try {
|
||||
await page.goto(workspace);await select(page,'0:shared/same.png');await details(page).getByRole('button',{name:/^Send prompt pair to ComfyUI: /}).click();
|
||||
const sender=dialog(page);const nodes=sender.getByRole('combobox',{name:'Destination node',exact:true});
|
||||
await sender.getByRole('combobox',{name:'ComfyUI window',exact:true}).selectOption({label:'Test ComfyUI'});
|
||||
await expect(nodes.locator('option')).toHaveText(['Choose a destination node','#12 Prompt Manager (+/-)','#3 Prompt Manager (+/-)','#7 Edited in ComfyUI','All 3 eligible nodes']);
|
||||
await expect(nodes).toHaveValue('');
|
||||
await nodes.selectOption({label:'All 3 eligible nodes'});
|
||||
await expect(sender.getByRole('status')).toHaveText('Sending to all 3 eligible nodes in Test ComfyUI. Apply to send it, or preview the changes first.');
|
||||
await page.screenshot({path:testInfo.outputPath('send-all-option.png')});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();
|
||||
const reviews=sender.locator('section.prompt-sender-node');
|
||||
await expect(reviews.locator('> h4')).toHaveText(['#3 Prompt Manager (+/-)','#7 Edited in ComfyUI','#12 Prompt Manager (+/-)']);
|
||||
await expect(sender.getByRole('region',{name:'#12 Prompt Manager (+/-)',exact:true})).toContainText('Existing 12');
|
||||
expect(await host.evaluate(()=>window.applied)).toEqual([]);
|
||||
await sender.getByRole('button',{name:'Apply to ComfyUI',exact:true}).click();
|
||||
await expect(sender.getByRole('alert')).toHaveText('Sent to 2 of 3 nodes in Test ComfyUI. #7 Edited in ComfyUI: Workflow or prompt changed. Preview again.');
|
||||
expect(await host.evaluate(()=>window.applied)).toEqual(['3','12']);
|
||||
expect(await host.evaluate(()=>window.hostValues['7'].positive)).toBe('Existing 7');
|
||||
await page.screenshot({path:testInfo.outputPath('send-all-result.png')});
|
||||
await page.setViewportSize({width:390,height:844});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();await expect(reviews).toHaveCount(3);
|
||||
await reviews.last().scrollIntoViewIfNeeded();
|
||||
expect(await page.evaluate(()=>document.querySelector('dialog.prompt-sender').scrollWidth<=document.querySelector('dialog.prompt-sender').clientWidth+1)).toBe(true);
|
||||
await page.screenshot({path:testInfo.outputPath('send-all-mobile.png')});
|
||||
const apply=sender.getByRole('button',{name:'Apply to ComfyUI',exact:true});await apply.focus();await expect(apply).toBeFocused();
|
||||
} finally {await host.evaluate(()=>window.fixtureHost.dispose());await host.close();}
|
||||
});
|
||||
|
||||
@@ -85,6 +85,9 @@ class TestModel:
|
||||
suggestions: list | None = None
|
||||
# The engine's model table (``py/autotag.py`` MODELS), set at startup.
|
||||
models_config: dict = {}
|
||||
# Where the WD14 label lists live (the Skipped tags preview, #431), set at
|
||||
# startup; the tagging fixture writes or removes a small list there.
|
||||
models_dir = None
|
||||
|
||||
def get_models_status(self) -> dict:
|
||||
from py.autotag import package_fields
|
||||
@@ -115,21 +118,26 @@ class TestModel:
|
||||
return ["generated", Path(image_path).parent.parent.name]
|
||||
|
||||
def generate_tag_suggestions(
|
||||
self, image_path: str, *, limit: int, **kwargs: object
|
||||
self, image_path: str, *, limit: int, skip=None, **kwargs: object
|
||||
) -> list[tuple[str, float | None]]:
|
||||
"""Fixed scored suggestions (WD14) or unscored ones (language models)."""
|
||||
"""Fixed scored suggestions (WD14) or unscored ones (language models),
|
||||
without the ones ``skip`` hides before the limit, as the engine does
|
||||
(#431)."""
|
||||
scored = self.loaded is not None and self.loaded.startswith("wd14")
|
||||
if self.suggestions is not None:
|
||||
return [
|
||||
pairs = [
|
||||
(name, confidence if scored else None)
|
||||
for name, confidence in self.suggestions[:limit]
|
||||
for name, confidence in self.suggestions
|
||||
]
|
||||
names = ["sandstone", "desert", "arch", "sky", "rock", "dune", "canyon"]
|
||||
names += [f"tag_{index:02d}" for index in range(100)]
|
||||
return [
|
||||
(name, round(0.98 - index * 0.008, 3) if scored else None)
|
||||
for index, name in enumerate(names[:limit])
|
||||
]
|
||||
else:
|
||||
names = ["sandstone", "desert", "arch", "sky", "rock", "dune", "canyon"]
|
||||
names += [f"tag_{index:02d}" for index in range(100)]
|
||||
pairs = [
|
||||
(name, round(0.98 - index * 0.008, 3) if scored else None)
|
||||
for index, name in enumerate(names)
|
||||
]
|
||||
kept = [pair for pair in pairs if skip is None or not skip.matches(pair[0])]
|
||||
return kept[:limit]
|
||||
|
||||
def download_model(self, model_type: str, progress_callback=None) -> bool:
|
||||
raise AssertionError("browser tests never download a model")
|
||||
@@ -233,6 +241,7 @@ def create_app(root: Path) -> web.Application:
|
||||
|
||||
TestModel.models_config = autotag.MODELS
|
||||
model = TestModel()
|
||||
model.models_dir = root / "models"
|
||||
autotag.get_autotag_service = lambda: model
|
||||
api = PromptManagerAPI()
|
||||
api.add_routes(routes)
|
||||
|
||||
@@ -109,6 +109,7 @@ class ResetWatch:
|
||||
stream.write(text)
|
||||
stream.flush()
|
||||
except (OSError, ValueError):
|
||||
# the stream closed; a watcher must not fail the server
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -162,13 +162,15 @@ async function serverJson(request, url) {
|
||||
}
|
||||
|
||||
const number = value => value.toLocaleString('en-US');
|
||||
// The sidebar's Images count (state.mjs), which System's Images metric shows (#555).
|
||||
const IMAGES_COUNT_URL = '/prompt_manager/images/output?mode=workspace&limit=1';
|
||||
|
||||
/** The concept's shell and Overview with the recorded deviations applied, given the server's real overview. */
|
||||
function mapOverviewConcept(reference, overview) {
|
||||
function mapOverviewConcept(reference, overview, images) {
|
||||
// D165: the discreet legacy-pages line follows the settings layout.
|
||||
reference = { ...reference, tree: [...reference.tree, 'p.legacy-pages.small'] };
|
||||
const relabel = { 'Database integrity': 'Not checked', 'Positive prompts': number(overview.library.positive_prompts),
|
||||
'Negative variants': number(overview.library.negative_variants), Images: number(overview.images.indexed) };
|
||||
'Negative variants': number(overview.library.negative_variants), Images: number(images) };
|
||||
const rows = {
|
||||
// D93: the real pending count; Review metadata opens Needs review (the concept's go-review).
|
||||
'3 ambiguous image links': row => ({ ...row,
|
||||
@@ -179,8 +181,9 @@ function mapOverviewConcept(reference, overview) {
|
||||
'One display configuration': row => ({ ...row, description: 'Grid and density settings are shared across views.' }),
|
||||
// D93: the real sources, not samples.
|
||||
'2 sample output sources': row => ({ ...row, title: `${number(overview.sources.effective)} output sources` }),
|
||||
// D96: the dialog shows runs by category and library totals, not prompt, tag or date usage (D153: no ratings).
|
||||
Statistics: row => ({ ...row, description: 'Runs by category, with library totals.' }),
|
||||
// D96: the dialog shows runs by category and database record counts, not prompt, tag or date usage (D153: no
|
||||
// ratings; #555: the counts are labelled as records, not as the Library or Images page counts).
|
||||
Statistics: row => ({ ...row, description: 'Runs by category, with database record counts.' }),
|
||||
};
|
||||
const surfaces = reference.columns.surfaces.map(surface => ({ ...surface,
|
||||
rows: surface.rows.map(row => (rows[row.title] ?? (value => value))(row)) }));
|
||||
@@ -218,6 +221,8 @@ async function conceptOverview(page, theme) {
|
||||
async function productionOverview(page, theme) {
|
||||
await openProduction(page, 'system/overview', theme);
|
||||
await expect(page.locator('#workspace-main .summary-metrics')).toBeVisible();
|
||||
// #555: the Images metric is the shell's Images count: compare once it was read.
|
||||
await expect(page.locator('#workspace-main .summary-metrics > .metric:nth-child(3) strong')).toHaveText(/^[\d,]+$/);
|
||||
// The thumbnail row is its own read (I2): compare once it answered.
|
||||
await expect(page.locator('#workspace-main .operation h3').filter({ hasText: 'thumbnails' })).toHaveText('No missing thumbnails');
|
||||
// So is the backup row (S9).
|
||||
@@ -231,6 +236,7 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`system shell and overview match the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const overview = await serverJson(request, '/prompt_manager/system/overview');
|
||||
const images = (await serverJson(request, IMAGES_COUNT_URL)).total;
|
||||
// The fixture's thumbnails all exist, so the concept's "No missing thumbnails" is also the real title.
|
||||
expect((await serverJson(request, '/prompt_manager/images/thumbnails/status')).pending).toBe(0);
|
||||
const concept = await conceptOverview(page, theme);
|
||||
@@ -239,7 +245,7 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-system-${theme}-${width}.png`), fullPage: true });
|
||||
const production = await productionOverview(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-system-${theme}-${width}.png`), fullPage: true });
|
||||
expect(diffInventory(mapOverviewConcept(concept, overview), production)).toEqual([]);
|
||||
expect(diffInventory(mapOverviewConcept(concept, overview, images), production)).toEqual([]);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
}
|
||||
@@ -265,8 +271,9 @@ const UNMAPPED_PATHS = [
|
||||
test('system comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const overview = await serverJson(request, '/prompt_manager/system/overview');
|
||||
const images = (await serverJson(request, IMAGES_COUNT_URL)).total;
|
||||
const concept = await conceptOverview(page, 'dark');
|
||||
const expected = mapOverviewConcept(concept, overview);
|
||||
const expected = mapOverviewConcept(concept, overview, images);
|
||||
const production = await productionOverview(page, 'dark');
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Unmapped, the concept differs exactly where a deviation is recorded, and nowhere else.
|
||||
@@ -290,7 +297,7 @@ test('system comparison detects mutations', async ({ page, request }) => {
|
||||
});
|
||||
|
||||
/**
|
||||
* D96: the concept's table, its rows the real ones; then "Library totals" in the same table markup. D153 (#447): the
|
||||
* D96: the concept's table, its rows the real ones; then "Database records" (#555) in the same table markup. D153 (#447): the
|
||||
* concept's "Avg. rating" column and a "Rated images" total are not drawn.
|
||||
*/
|
||||
function mapStatisticsConcept(reference, statistics) {
|
||||
@@ -302,9 +309,10 @@ function mapStatisticsConcept(reference, statistics) {
|
||||
tables: [
|
||||
{ ...runs, head: runs.head.filter(([, , text]) => text !== 'Avg. rating'),
|
||||
rows: statistics.by_category.map(row => [row.category ?? 'Uncategorized', number(row.runs)]) },
|
||||
{ caption: 'Library totals', head: [['th', 'col', 'Total'], ['th', 'col', 'Count']],
|
||||
rows: [['Prompts', number(statistics.totals.total_prompts)], ['Tags', number(statistics.totals.total_tags)],
|
||||
['Images', number(statistics.totals.total_images)]] },
|
||||
{ caption: 'Database records', head: [['th', 'col', 'Total'], ['th', 'col', 'Count']],
|
||||
rows: [['Prompt records', number(statistics.totals.total_prompts)],
|
||||
['Tags on prompts', number(statistics.totals.total_tags)],
|
||||
['Image records', number(statistics.totals.total_images)]] },
|
||||
],
|
||||
};
|
||||
}
|
||||
@@ -420,7 +428,7 @@ for (const theme of THEMES) test(`statistics and health dialogs match the concep
|
||||
expect(diffInventory(mapHealthConcept(concept.health), await page.evaluate(measureDialog)).map(item => item.path))
|
||||
.toEqual(['operations.rows.0.title']);
|
||||
await page.evaluate(() => { document.querySelector('#modal .operation h3').textContent = 'Integrity check'; });
|
||||
// And a removed "Library totals" table and a changed caption in the statistics dialog.
|
||||
// And a removed "Database records" table and a changed caption in the statistics dialog.
|
||||
await page.keyboard.press('Escape');
|
||||
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Explore', exact: true }).click();
|
||||
|
||||
@@ -64,11 +64,31 @@ function measureMaintenance(mainSelector) {
|
||||
};
|
||||
}
|
||||
|
||||
/** D102: the Metadata cache row says why its review is unavailable and its button is aria-disabled (D93's dimmed style). */
|
||||
function mapMaintenanceConcept(reference) {
|
||||
/**
|
||||
* D192 (#550, beyond the concept): Database adds "Pair category and tags" after "Unused prompt components", shaped like
|
||||
* the concept's plain Review plan row (Optimize database's).
|
||||
*/
|
||||
function mapPairRow(reference) {
|
||||
return {
|
||||
...reference,
|
||||
surfaces: reference.surfaces.map(surface => ({ ...surface, rows: surface.rows.map(row => row.title !== 'Metadata cache' ? row : {
|
||||
surfaces: reference.surfaces.map(surface => {
|
||||
const plain = surface.rows.find(row => row.title === 'Optimize database');
|
||||
const at = surface.rows.findIndex(row => row.title === 'Unused prompt components');
|
||||
if (!plain || at < 0) return surface;
|
||||
return { ...surface, tree: [...surface.tree, 'div.operation'], rows: [...surface.rows.slice(0, at + 1),
|
||||
{ ...plain, title: 'Pair category and tags', description: 'Fill an empty category or tags on prompts older ' +
|
||||
'Pair node runs saved, from their saved pairs. Only empty fields of never-edited prompts.' },
|
||||
...surface.rows.slice(at + 1)] };
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/** D102: the Metadata cache row says why its review is unavailable and its button is aria-disabled (D93's dimmed style). */
|
||||
function mapMaintenanceConcept(reference) {
|
||||
const mapped = mapPairRow(reference);
|
||||
return {
|
||||
...mapped,
|
||||
surfaces: mapped.surfaces.map(surface => ({ ...surface, rows: surface.rows.map(row => row.title !== 'Metadata cache' ? row : {
|
||||
...row, description: `${row.description} Nothing is cached in this browser.`,
|
||||
actions: row.actions.map(action => ({ ...action, opacity: '0.55', cursor: 'not-allowed' })) }) })),
|
||||
};
|
||||
@@ -209,8 +229,8 @@ test('maintenance comparison detects mutations', async ({ page, request }) => {
|
||||
const expected = mapMaintenanceConcept(concept);
|
||||
const production = await productionMaintenance(page, 'dark');
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Unmapped, the section differs only on the D102 row.
|
||||
expect(diffInventory(concept, production).map(item => item.path).sort()).toEqual([
|
||||
// Mapped only for the D192 row, the section differs only on the D102 row.
|
||||
expect(diffInventory(mapPairRow(concept), production).map(item => item.path).sort()).toEqual([
|
||||
'surfaces.1.rows.3.actions.0.cursor', 'surfaces.1.rows.3.actions.0.opacity', 'surfaces.1.rows.3.description']);
|
||||
const measure = () => page.evaluate(measureMaintenance, '#workspace-main');
|
||||
const paths = async () => diffInventory(expected, await measure()).map(item => item.path);
|
||||
|
||||
@@ -33,6 +33,10 @@ test.afterEach(async ({ page, request, browserName }) => {
|
||||
const PREVIEW_URL = '/prompt_manager/autotag/batch/preview';
|
||||
const MODEL_OPTIONS = ['WD14 SwinV2 · ~400 MB', 'WD14 ViT · ~400 MB', 'GGUF JoyCaption · ~4.5 GB', 'HuggingFace JoyCaption · ~16 GB'];
|
||||
const SUGGESTIONS = 'From 1 to 100: the most suggestions Generate shows for one image.';
|
||||
const SKIPPED_DESCRIPTION = 'Suggested tags that contain one of these words are hidden. Words are separated by spaces, _ or -; '
|
||||
+ 'other characters such as ( ) belong to the word.';
|
||||
const DEFAULT_DESCRIPTION = 'The Tag image dialog and batches start from this.';
|
||||
const INSTRUCTIONS_DESCRIPTION = 'Instructions for the language-model engines. Built-in can’t be changed; duplicate it to make your own.';
|
||||
const BATCH = 'Tag each output image from its own pixels with the model above. Prompts are not changed. Review the scope first.';
|
||||
|
||||
/** Runs in the page: Auto-tagging's surface under `mainSelector`: rows, field, actions, operation, geometry and tokens. */
|
||||
@@ -62,7 +66,7 @@ function measureTagging(mainSelector) {
|
||||
text: [...row.firstElementChild.children].map(node => [name(node), text(node), node.localName === 'label' ? node.htmlFor : null]),
|
||||
control: control(row), flexWrap: getComputedStyle(row).flexWrap,
|
||||
controlBelow: row.lastElementChild.getBoundingClientRect().top >= row.firstElementChild.getBoundingClientRect().bottom - 0.5 })),
|
||||
field: { tree: [...field.children].map(name), label: [text(field.querySelector('label')), field.querySelector('label').htmlFor],
|
||||
field: field && { tree: [...field.children].map(name), label: [text(field.querySelector('label')), field.querySelector('label').htmlFor],
|
||||
textarea: { id: field.querySelector('textarea').id, value: field.querySelector('textarea').value,
|
||||
...css(field.querySelector('textarea'), 'fontSize', 'minHeight', 'backgroundColor', 'color', 'borderTopColor') },
|
||||
...css(field, 'display', 'rowGap', 'fontSize', 'marginTop', 'marginBottom') },
|
||||
@@ -70,6 +74,31 @@ function measureTagging(mainSelector) {
|
||||
...css(actions, 'display', 'flexWrap', 'columnGap', 'justifyContent'),
|
||||
primary: css(actions.querySelector('.ca-primary'), 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft'),
|
||||
plain: css(actions.querySelector('button:not(.ca-primary)'), 'backgroundColor', 'color', 'fontSize', 'borderTopColor') },
|
||||
// D202 (#431): the Skipped tags block (the concept has none).
|
||||
skipped: (block => block && { tree: [...block.children].map(name), key: block.dataset.key,
|
||||
text: [...block.firstElementChild.children].map(node => [node.localName, text(node), node.localName === 'label' ? node.htmlFor : null]),
|
||||
add: [...block.querySelector('.skip-add').children].map(node => [name(node), node.id, node.getAttribute('placeholder'), text(node)]),
|
||||
preview: [text(block.querySelector('.skip-preview')), block.querySelector('.skip-preview').getAttribute('role')],
|
||||
...css(block, 'paddingTop', 'paddingBottom', 'borderTopWidth', 'borderTopColor'),
|
||||
label: css(block.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
description: css(block.querySelector('.setting-block-text p'), 'fontSize', 'color', 'marginTop'),
|
||||
input: css(block.querySelector('input'), 'fontSize', 'backgroundColor', 'color', 'borderTopColor') })(
|
||||
surface.querySelector(':scope > .setting-block[data-setting="tag-skipped"]')),
|
||||
// D203 (#431): Saved instructions (the concept has a template field instead).
|
||||
instructions: (block => block && { tree: [...block.children].map(name), key: block.dataset.key,
|
||||
text: [...block.firstElementChild.children].map(node => [node.localName, text(node)]),
|
||||
list: [...block.querySelectorAll('[role="option"]')].map(option => [text(option.firstElementChild),
|
||||
[...option.querySelectorAll('.badges > .chip')].map(chip => [text(chip), chip.className]), option.getAttribute('aria-selected')]),
|
||||
listRole: block.querySelector('.instruction-list').getAttribute('role'),
|
||||
actions: [...block.querySelectorAll('.instruction-actions > button')].map(item => [text(item), item.className, item.disabled]),
|
||||
editor: [...block.querySelectorAll('.instruction-editor .field')].map(item => [text(item.querySelector('label > span')),
|
||||
item.querySelector('input, textarea').value, item.querySelector('input, textarea').readOnly]),
|
||||
note: text(block.querySelector('#instruction-note')),
|
||||
layout: getComputedStyle(block.querySelector('.instructions-layout')).gridTemplateColumns.split(' ').length,
|
||||
...css(block, 'paddingTop', 'paddingBottom', 'borderTopWidth', 'borderTopColor'),
|
||||
label: css(block.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
textarea: css(block.querySelector('textarea'), 'fontSize', 'backgroundColor', 'borderTopColor') })(
|
||||
surface.querySelector(':scope > .setting-block[data-setting="tag-instructions"]')),
|
||||
operations: [...surface.querySelectorAll(':scope > .operation')].map(row => ({ tree: [...row.children].map(name),
|
||||
title: text(row.querySelector('h3')), description: text(row.querySelector('p')),
|
||||
actions: [...row.querySelectorAll('.row-actions > button')].map(item => [text(item), item.className]) })),
|
||||
@@ -103,13 +132,21 @@ function measureTagging(mainSelector) {
|
||||
* The concept's Auto-tagging with the recorded deviations applied, for the fixture (all engines on disk, none loaded).
|
||||
* `phone` is a viewport at the concept's 700px breakpoint or below.
|
||||
*/
|
||||
function mapTaggingConcept(reference, { template, phone }) {
|
||||
function mapTaggingConcept(reference, { builtin, saved, phone }) {
|
||||
const [model, existing, general, character] = reference.rows;
|
||||
const [preview] = reference.actions.buttons;
|
||||
// D203: Built-in first, then the saved entries, in each default select.
|
||||
const names = ['Built-in', ...saved.map(entry => entry.name)];
|
||||
const defaultRow = (engine, long) => ({ ...existing,
|
||||
text: [['label', `Default for ${long}`, `instruction-default-${engine}`], ['p', DEFAULT_DESCRIPTION, null]],
|
||||
control: { ...existing.control, id: `instruction-default-${engine}`, options: names.map(label => [label, false]),
|
||||
selected: names[1] } });
|
||||
return {
|
||||
...reference,
|
||||
// D133 adds a row after the thresholds; D134 adds the operation after the actions.
|
||||
tree: [...reference.tree.slice(0, 6), 'div.setting-row', ...reference.tree.slice(6), 'div.operation'],
|
||||
// D133 adds a row after the thresholds, D202 the Skipped tags block after it; D203 replaces the template field
|
||||
// with Saved instructions, the two default rows and their note; D134 adds the operation after the actions.
|
||||
tree: [...reference.tree.slice(0, 6), 'div.setting-row', 'div.setting-block', 'div.setting-block', 'div.setting-row',
|
||||
'div.setting-row', 'p.instruction-reset.small', reference.tree[7], 'div.operation'],
|
||||
rows: [
|
||||
// D132 (OQ5, OQ6): the four real engines with their download size, and the chosen one's live status.
|
||||
{ ...model, text: [model.text[0], ['p', 'Downloaded. Not loaded.', null]],
|
||||
@@ -120,16 +157,49 @@ function mapTaggingConcept(reference, { template, phone }) {
|
||||
// D133: the persisted suggestion count, on the concept's number row.
|
||||
{ ...general, text: [['label', 'Suggestions per image', 'tag-suggestions'], ['p', SUGGESTIONS, null]],
|
||||
control: { ...general.control, id: 'tag-suggestions', value: '10' } },
|
||||
// D203: the default of each language-model engine (the fixture's saved entry is both), on the concept's
|
||||
// select row.
|
||||
defaultRow('gguf', 'GGUF JoyCaption'), defaultRow('hf', 'HuggingFace JoyCaption'),
|
||||
],
|
||||
// The saved template (the server's default until one is saved) replaces the concept's sample text.
|
||||
field: { ...reference.field, textarea: { ...reference.field.textarea, value: template } },
|
||||
// D203: Saved instructions in the concept's setting-row tokens (as D202), Built-in selected and read-only.
|
||||
instructions: { tree: ['div.setting-block-text', 'div.instructions-layout'], key: 'autotag_instructions',
|
||||
text: [['label', 'Saved instructions'], ['p', `${INSTRUCTIONS_DESCRIPTION} ${saved.length} of 20 saved · Built\u2011in is not counted.`]],
|
||||
list: [['Built-in', [['Read-only', 'chip']], 'true'],
|
||||
...saved.map((entry, index) => [entry.name, index === 0
|
||||
? [['Default for GGUF', 'chip blue'], ['Default for HuggingFace', 'chip blue']] : [], 'false'])],
|
||||
listRole: 'listbox',
|
||||
actions: [['+ New', '', false], ['Duplicate', '', false], ['Delete', 'danger', true]],
|
||||
editor: [['Name', 'Built-in', true], ['Text', builtin, true]],
|
||||
note: 'Built-in instructions can’t be changed. Duplicate them to make your own.',
|
||||
layout: phone ? 1 : 2,
|
||||
paddingTop: reference.row.paddingTop, paddingBottom: reference.row.paddingBottom,
|
||||
borderTopWidth: reference.row.borderTopWidth, borderTopColor: reference.row.borderTopColor,
|
||||
label: reference.label,
|
||||
textarea: { fontSize: reference.field.textarea.fontSize, backgroundColor: reference.field.textarea.backgroundColor,
|
||||
borderTopColor: reference.field.textarea.borderTopColor } },
|
||||
// D202 (#431): the Skipped tags block in the concept's setting-row tokens: its label and description as a row's,
|
||||
// the row's padding and rule above, the concept's field input; an empty list says so.
|
||||
skipped: { tree: ['div.setting-block-text', 'div.skip-add', 'p.skip-problem.small', 'div.skip-chips.tags', 'p.skip-preview.small'],
|
||||
key: 'autotag_skipped_tags',
|
||||
text: [['label', 'Skipped tags', 'tag-skipped-input'], ['p', SKIPPED_DESCRIPTION, null]],
|
||||
add: [['input', 'tag-skipped-input', 'Add a word or words', ''], ['button', 'tag-skipped-add', null, 'Add']],
|
||||
preview: ['No tags are skipped.', 'status'],
|
||||
paddingTop: reference.row.paddingTop, paddingBottom: reference.row.paddingBottom,
|
||||
borderTopWidth: reference.row.borderTopWidth, borderTopColor: reference.row.borderTopColor,
|
||||
label: reference.label,
|
||||
description: { fontSize: reference.description.fontSize, color: reference.description.color,
|
||||
marginTop: reference.description.marginTop },
|
||||
input: { fontSize: '12px', backgroundColor: reference.field.textarea.backgroundColor, color: reference.field.textarea.color,
|
||||
borderTopColor: reference.field.textarea.borderTopColor } },
|
||||
// D203: no template field.
|
||||
field: null,
|
||||
// D132: Download (for an engine already on disk, "Download again") and Unload replace the two simulated jobs.
|
||||
actions: { ...reference.actions, buttons: [preview, ['Download again', ''], ['Unload', '']] },
|
||||
// D134: the prompt-tag batch, reviewed first.
|
||||
operations: [{ tree: ['div', 'div.row-actions'], title: 'Tag output images', description: BATCH, actions: [['Review', '']] }],
|
||||
// D132 (#398): every select shows its whole chosen option at every width (at 390 the concept's 120px cap cuts
|
||||
// them; production lets each grow past it).
|
||||
selectsFit: reference.selectsFit.map(() => true),
|
||||
selectsFit: [...reference.selectsFit, true, true].map(() => true),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -153,7 +223,9 @@ async function productionTagging(page, theme) {
|
||||
async function savedTemplate(request, page) {
|
||||
const response = await request.get('/prompt_manager/settings');
|
||||
await expect(response).toBeOK();
|
||||
return { template: (await response.json()).settings.autotag_template, phone: page.viewportSize().width <= 700 };
|
||||
const { settings } = await response.json();
|
||||
return { builtin: settings.autotag_builtin_instructions, saved: settings.autotag_instructions,
|
||||
phone: page.viewportSize().width <= 700 };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
@@ -177,14 +249,15 @@ for (const theme of THEMES) for (const width of [390, 320]) {
|
||||
await page.locator('#workspace-main #tag-existing').selectOption('replace_after_review');
|
||||
await page.locator('#workspace-main section.surface').screenshot({ path: testInfo.outputPath(`production-tagging-selects-${theme}-${width}.png`) });
|
||||
const selects = page.locator('#workspace-main section.surface > .setting-row > select');
|
||||
await expect(selects).toHaveCount(2);
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
// D203 adds the two per-engine default selects.
|
||||
await expect(selects).toHaveCount(4);
|
||||
for (let index = 0; index < 4; index += 1) {
|
||||
const select = selects.nth(index);
|
||||
const values = await select.evaluate(node => [...node.options].map(option => option.value));
|
||||
for (const value of values) {
|
||||
await select.selectOption(value);
|
||||
const measured = await page.evaluate(measureTagging, '#workspace-main');
|
||||
expect(measured.selectsFit, `${await select.getAttribute('id')}=${value}`).toEqual([true, true]);
|
||||
expect(measured.selectsFit, `${await select.getAttribute('id')}=${value}`).toEqual([true, true, true, true]);
|
||||
expect(measured.pageScroll, value).toBe(false);
|
||||
}
|
||||
}
|
||||
@@ -198,12 +271,16 @@ test('auto-tagging comparison detects mutations', async ({ page, request }) => {
|
||||
const expected = mapTaggingConcept(concept, saved);
|
||||
const production = await productionTagging(page, 'dark');
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Unmapped, the section differs only where D132, D133 and D134 are recorded (and the saved template's text).
|
||||
expect(diffInventory(concept, production).map(item => item.path.split('.').slice(0, 3).join('.'))
|
||||
// Unmapped, the section differs only where D132, D133, D134, D202 and D203 are recorded (and the saved template's text).
|
||||
// (The concept has no Skipped tags block, so every path under it is one D202 difference.)
|
||||
expect(diffInventory(concept, production).map(item => (/^(skipped|instructions|field)\b/.test(item.path) ? item.path.split('.')[0]
|
||||
: item.path.split('.').slice(0, 3).join('.')))
|
||||
.filter((path, index, all) => all.indexOf(path) === index).sort()).toEqual([
|
||||
'actions.buttons.1', 'actions.buttons.2', 'field.textarea.value', 'operations.0.actions', 'operations.0.description',
|
||||
'operations.0.title', 'operations.0.tree', 'operations.length', 'rows.0.control', 'rows.0.text', 'rows.4.control', 'rows.4.controlBelow',
|
||||
'rows.4.flexWrap', 'rows.4.text', 'rows.4.tree', 'rows.length', 'tree.6', 'tree.7', 'tree.8', 'tree.9', 'tree.length'].sort());
|
||||
'actions.buttons.1', 'actions.buttons.2', 'field', 'instructions', 'operations.0.actions', 'operations.0.description',
|
||||
'operations.0.title', 'operations.0.tree', 'operations.length', 'rows.0.control', 'rows.0.text',
|
||||
...[4, 5, 6].flatMap(row => ['control', 'controlBelow', 'flexWrap', 'text', 'tree'].map(key => `rows.${row}.${key}`)),
|
||||
'rows.length', 'selectsFit.2', 'selectsFit.3', 'selectsFit.length', 'skipped',
|
||||
...Array.from({ length: 8 }, (_, index) => `tree.${index + 6}`), 'tree.length'].sort());
|
||||
const measure = () => page.evaluate(measureTagging, '#workspace-main');
|
||||
const paths = async () => diffInventory(expected, await measure()).map(item => item.path);
|
||||
// A removed row.
|
||||
@@ -254,8 +331,8 @@ test('auto-tagging at 390 wraps only the engine row', async ({ page, request })
|
||||
const concept = await conceptTagging(page, 'dark');
|
||||
const production = await productionTagging(page, 'dark');
|
||||
expect(diffInventory(mapTaggingConcept(concept, await savedTemplate(request, page)), production)).toEqual([]);
|
||||
expect(production.rows.map(row => row.flexWrap)).toEqual(['wrap', 'nowrap', 'nowrap', 'nowrap', 'nowrap']);
|
||||
expect(production.selectsFit).toEqual([true, true]);
|
||||
expect(production.rows.map(row => row.flexWrap)).toEqual(['wrap', 'nowrap', 'nowrap', 'nowrap', 'nowrap', 'nowrap', 'nowrap']);
|
||||
expect(production.selectsFit).toEqual([true, true, true, true]);
|
||||
});
|
||||
|
||||
/** Runs in the page: the open review dialog: its title, body and the concept's tokens. */
|
||||
|
||||
@@ -92,6 +92,8 @@ export function cardActionsInventory(card) {
|
||||
};
|
||||
}
|
||||
|
||||
// #431: the fixture's saved "Describe the image" entry, the default of both engines (workspace_tagging_fixture.py).
|
||||
export const FIXTURE_INSTRUCTIONS_ID='d35c71be0a1e';
|
||||
// D84 (plan D61): the legacy engines' values, labels and sizes, GGUF JoyCaption in the concept's Ollama slot.
|
||||
const MODEL_LABELS=Object.freeze({'wd14-swinv2':['wd14-swinv2','WD14 SwinV2 · ~400 MB'],'wd14-vit':['wd14-vit','WD14 ViT · ~400 MB'],
|
||||
ollama:['gguf','GGUF JoyCaption (Recommended) · ~4.5 GB'],huggingface:['hf','HuggingFace JoyCaption · ~16 GB']});
|
||||
@@ -106,17 +108,24 @@ const CONCEPT_LABELS=Object.freeze({'WD14 SwinV2 · fast classifier':'WD14 SwinV
|
||||
* (`chip`: a concept `.chip` inventory, the heading's size chip) around the row's own text (12px) in the checked
|
||||
* row's colours (accent border, `--soft-blue`);
|
||||
* - D84 (plan D61): the legacy engine labels and values, and the server's default instructions (`instructions`);
|
||||
* - D204 (#431): a "Saved instructions" select (`picker`: its value and options) before the instructions, on the
|
||||
* concept's select field tokens;
|
||||
* - D91 (plan D68): a Suggestions count field after the model's own fields, on the concept's number field tokens
|
||||
* (`numberField`: the concept's General confidence field) with the configured count (`limit`);
|
||||
* - D29: the prototype's "Sample suggestions from …" status reads "Suggestions from …".
|
||||
* The opened state is compared unmapped: both show the concept's on-screen "Generate suggestions with the selected
|
||||
* model before approving." (its renderTagModelOptions, a1062–1076, replaces the markup's sentence at once).
|
||||
*/
|
||||
export function mapConceptTagging(reference,{instructions,limit=10,chip,numberField}) {
|
||||
export function mapConceptTagging(reference,{instructions,limit=10,chip,numberField,
|
||||
picker={value:FIXTURE_INSTRUCTIONS_ID,options:[['builtin','Built-in'],[FIXTURE_INSTRUCTIONS_ID,`${instructions} · default`]]}}) {
|
||||
const [select,...own]=reference.modelPanel.fields;
|
||||
// D204 (#431): a "Saved instructions" select on the concept's select field, before the instructions.
|
||||
const pickField={...select,label:'Saved instructions',control:{...select.control,id:'image-tag-instructions-pick',
|
||||
value:picker.value,options:picker.options}};
|
||||
const suggestionsField={...numberField,label:'Suggestions',control:{...numberField.control,id:'image-suggestion-limit',
|
||||
value:String(limit),min:'1',max:'100',step:'1'}};
|
||||
const mappedOwn=own.map(field=>field.control.id==='image-tag-instructions'?{...field,control:{...field.control,value:instructions}}:field);
|
||||
const mappedOwn=own.flatMap(field=>field.control.id==='image-tag-instructions'
|
||||
?[pickField,{...field,control:{...field.control,value:instructions}}]:[field]);
|
||||
const rows=reference.suggestions.rows.map(row=>({...row,
|
||||
control:row.control?.kind==='checkbox'&&row.control.checked?{kind:'remove',label:`Remove ${row.name}`}:row.control,
|
||||
borderTopLeftRadius:chip.borderTopLeftRadius}));
|
||||
|
||||
@@ -156,8 +156,9 @@ test('at 390px the window fits the screen, actions stay reachable and the page s
|
||||
expect(frame.bottom-frame.top).toBeLessThan(844);
|
||||
expect(frame.bottom-frame.top).toBeGreaterThanOrEqual(280);
|
||||
expect(await page.evaluate(()=>document.documentElement.scrollWidth)).toBeLessThanOrEqual(390);
|
||||
// The last row's actions can be scrolled to inside the window and fit its width.
|
||||
const last=rows(page).last().getByRole('button',{name:'Rename / merge',exact:true});
|
||||
// The first page's last row's actions can be scrolled to inside the window and fit its width.
|
||||
// Pinned to row 100: reaching it loads the next page (D183), and last() would then follow row 200.
|
||||
const last=rows(page).nth(99).getByRole('button',{name:'Rename / merge',exact:true});
|
||||
await last.scrollIntoViewIfNeeded();await expect(last).toBeInViewport();
|
||||
const [button]=await boxes(last);const [after]=await boxes(tagWindow(page));
|
||||
expect(button.right).toBeLessThanOrEqual(after.right);
|
||||
|
||||
@@ -233,7 +233,7 @@ test('a view stored under older rules still lists and opens with its usable valu
|
||||
// D153 (#447): the workspace no longer offers "Highest rated"; the API still stores such a view as it was saved.
|
||||
test('a view saved under Highest rated opens on Recently added on Images and the Library', async ({ page, request }) => {
|
||||
const images = await create(request, { page: 'images', name: 'Rated cats', state: { ...IMAGE_DEFAULTS, q: 'cat', sort: 'rating' } });
|
||||
const library = await create(request, { page: 'library', name: 'Rated dogs', state: { q: 'dog', category: '', sort: 'rating', tags: [], tag_mode: 'all', folder: '' } });
|
||||
await create(request, { page: 'library', name: 'Rated dogs', state: { q: 'dog', category: '', sort: 'rating', tags: [], tag_mode: 'all', folder: '' } });
|
||||
expect((await listed(request)).map(view => view.state.sort)).toEqual(['rating', 'rating']);
|
||||
const listings = [];
|
||||
page.on('request', item => {
|
||||
|
||||
@@ -60,7 +60,7 @@ async function openReview(page, request, items = 30) {
|
||||
const inViewport = (rect, page) => rect.top >= 0 && rect.bottom <= page.viewportSize().height;
|
||||
|
||||
test.beforeEach(async ({ page, browserName }) => { await startCoverage(page, browserName); });
|
||||
test.afterEach(async ({ page, browserName }, testInfo) => { await saveCoverage(page, browserName, testInfo); });
|
||||
test.afterEach(async ({ page, browserName }) => { await saveCoverage(page, browserName); });
|
||||
|
||||
for (const viewport of [null, PHONE]) {
|
||||
const label = viewport ? ' at 390px' : '';
|
||||
|
||||
@@ -284,6 +284,23 @@ test('a restore seen finishing elsewhere closes an open restore review and disca
|
||||
expect((await serverJson(request, '/__test__/system/backups')).stages).toBe(0);
|
||||
});
|
||||
|
||||
// #583: in WebKit, while a page.route is armed, a file part of a FormData body is read only when the request goes on;
|
||||
// a garbage collection in between dropped the form once nothing referenced it, and the server got an empty file
|
||||
// ("Uploaded file is empty"). The page keeps the form until its upload is answered. The route holds the upload
|
||||
// through a collection; without the fix WebKit lost the file every time (0/10 in the pinned image). Chromium and
|
||||
// Firefox keep the body anyway, so there it only checks that a held upload still opens the review.
|
||||
test('a review upload survives a garbage collection while it is in flight', async ({ page, request }) => {
|
||||
const bytes = await backupBytes(request);
|
||||
await page.route(`**${PREVIEW_URL}`, async route => {
|
||||
await page.requestGC();
|
||||
await route.continue();
|
||||
});
|
||||
await openBackups(page);
|
||||
expect((await chooseBackup(page, bytes)).status()).toBe(200);
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Restore preview');
|
||||
expect((await serverJson(request, '/__test__/system/backups')).stages).toBe(1);
|
||||
});
|
||||
|
||||
test('invalid file shows the validator message; nothing is staged', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
await openBackups(page);
|
||||
|
||||
@@ -11,7 +11,6 @@ const REPORT_URL = '/prompt_manager/system/diagnostics';
|
||||
const LOGS_URL = '/prompt_manager/logs';
|
||||
const CONFIG_URL = '/prompt_manager/logs/config';
|
||||
const PREVIEW_URL = '/prompt_manager/maintenance/preview';
|
||||
const APPLY_URL = '/prompt_manager/maintenance/apply';
|
||||
const TEST_LINK_URL = '/prompt_manager/diagnostics/test-link';
|
||||
const CHANGED = 'The log files changed since the review; nothing was deleted. Review again.';
|
||||
const KEY = 'sk-s12-browser-SECRET-0123456789';
|
||||
|
||||
@@ -64,7 +64,9 @@ test('the section shows the concept rows; Metadata cache follows Q5 (D102)', asy
|
||||
item.querySelector('.row-actions > button').textContent, item.querySelector('.row-actions > button').className]),
|
||||
]))).toEqual([
|
||||
['Database', [['Integrity & relationship checks', 'Check', ''], ['Duplicate prompt text', 'Review plan', ''],
|
||||
['Optimize database', 'Review plan', ''], ['Unused prompt components', 'Review plan', 'danger']]],
|
||||
['Optimize database', 'Review plan', ''], ['Unused prompt components', 'Review plan', 'danger'],
|
||||
// D192 (#550): beyond the concept.
|
||||
['Pair category and tags', 'Review plan', '']]],
|
||||
['Images & cache', [['Missing-file links', 'Review plan', 'danger'], ['Duplicate image files', 'Find duplicates', ''],
|
||||
['Thumbnail cache', 'Review options', ''], ['Metadata cache', 'Review plan', '']]],
|
||||
]);
|
||||
@@ -177,6 +179,70 @@ test("plan dialog shows the server's scope, protected, backup policy and media l
|
||||
expect(await serverJson(request, '/__test__/system/digest')).toEqual(before);
|
||||
});
|
||||
|
||||
test('D192: Pair category and tags reviews old Pair prompts, keeps an edited one and fills the other (#550)',
|
||||
async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/pair-metadata')).toBeOK();
|
||||
const answers = [];
|
||||
page.on('response', response => { if (response.url().endsWith(PREVIEW_URL)) answers.push(response.json()); });
|
||||
await openMaintenance(page);
|
||||
const pairRow = row(page, 'Pair category and tags');
|
||||
await expect(pairRow.locator('p')).toHaveText('Fill an empty category or tags on prompts older Pair node runs saved, ' +
|
||||
'from their saved pairs. Only empty fields of never-edited prompts.');
|
||||
await pairRow.getByRole('button', { name: 'Review plan', exact: true }).click();
|
||||
const plan = dialog(page);
|
||||
await expect(plan.locator('#modal-body > h3')).toHaveText('Pair category and tags');
|
||||
const answer = await answers[0];
|
||||
expect(answer.operation).toBe('pair_metadata');
|
||||
expect(await gridOf(page)).toEqual([
|
||||
['Scope', '1 prompt to fill: 1 category, 1 tag set'],
|
||||
['Protected', '1 edited after it was saved'],
|
||||
['Backup', `Create and verify before changes (${answer.backup.destination_label})`],
|
||||
['Media on disk', 'No image file or saved pair is changed.'],
|
||||
]);
|
||||
await expect(plan.locator('details > summary')).toHaveText('Listed records (3)');
|
||||
await plan.locator('details > summary').click();
|
||||
// Protected rows come first, the Record row last.
|
||||
const lines = (await plan.locator('details > pre').textContent()).split('\n');
|
||||
expect(lines).toHaveLength(3);
|
||||
expect(lines[0]).toBe('S550 harbor · Kept: edited after it was saved; its empty category or tags may be intentional');
|
||||
expect(lines[1]).toMatch(/^S550 lantern · Fills category “night” and 2 tags: s550-glow, s550-warm \(pair \d+\)$/);
|
||||
expect(lines[2]).toMatch(/^Record · The job's report lists every filled field\. Download it to keep it/);
|
||||
await plan.getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
const job = await ended(request, page.url().split('/').pop());
|
||||
expect([job.title, job.state, job.counters.changed, job.counters.protected])
|
||||
.toEqual(['Pair category and tags', 'finished', 1, 1]);
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(main.locator('h1')).toHaveText('Pair category and tags');
|
||||
await expect(main.getByRole('button', { name: 'Download report', exact: true })).toBeVisible();
|
||||
// #550 live test: the Changed group lists the path-less record rows as plain text, never an open action.
|
||||
const results = main.getByRole('region', { name: 'Results', exact: true });
|
||||
await results.locator('.outcome-list > button').filter({ hasText: 'Changed' }).click();
|
||||
const records = results.locator('.outcome-files > .file-row');
|
||||
await expect(records).toHaveCount(1);
|
||||
await expect(records.first()).toHaveText(/^Prompt \d+: category “night”; tags s550-glow, s550-warm$/);
|
||||
expect(await records.first().evaluate(node => node.localName)).toBe('div');
|
||||
await expect(results.locator('.outcome-files button')).toHaveCount(0);
|
||||
await expect(results.getByRole('searchbox', { name: 'Filter results' })).toHaveAttribute('placeholder', 'Filter records…');
|
||||
// Protected lists each kept prompt with the review's reason (#550 live re-check).
|
||||
await results.locator('.outcome-list > button').filter({ hasText: 'Protected' }).click();
|
||||
await expect(records).toHaveCount(1);
|
||||
await expect(records.first()).toHaveText(
|
||||
/^Prompt \d+: kept, edited after it was saved; its empty category or tags may be intentional$/);
|
||||
// An empty group says records, not files.
|
||||
await results.locator('.outcome-list > button').filter({ hasText: 'Errors' }).click();
|
||||
await expect(results.locator('.outcome-files > p.empty-state')).toHaveText('No records in this group.');
|
||||
const library = await serverJson(request, '/prompt_manager/library');
|
||||
const lantern = library.items.find(item => item.text === 'S550 lantern');
|
||||
const harbor = library.items.find(item => item.text === 'S550 harbor');
|
||||
const filled = (await serverJson(request, `/prompt_manager/library/${lantern.id}?archive=all`)).positive;
|
||||
expect([filled.category, [...filled.tags].sort()]).toEqual(['night', ['s550-glow', 's550-warm']]);
|
||||
expect((await serverJson(request, `/prompt_manager/library/${harbor.id}?archive=all`)).positive.category).toBeFalsy();
|
||||
await page.goto(`${workspace}#/library`);
|
||||
await expect(page.locator(`article[data-prompt-id="${lantern.id}"] .card-title > .chip`)).toHaveText('night');
|
||||
});
|
||||
|
||||
test('a plan with nothing to do shows only Close, never Apply', async ({ page, request }) => {
|
||||
const applies = [];
|
||||
page.on('request', item => { if (item.url().endsWith(APPLY_URL)) applies.push(item.url()); });
|
||||
|
||||
@@ -112,13 +112,14 @@ test('model options are the four real engines with size and the description show
|
||||
await expect(status(page)).toHaveText('Unsaved changes.');
|
||||
// The other rows keep the concept's copy and the saved values (D133 adds Suggestions per image).
|
||||
expect(await surface(page).locator(':scope > .setting-row label').allTextContents()).toEqual(['Tagging model', 'Existing tags',
|
||||
'General-tag confidence', 'Character-tag confidence', 'Suggestions per image']);
|
||||
'General-tag confidence', 'Character-tag confidence', 'Suggestions per image', 'Default for GGUF JoyCaption',
|
||||
'Default for HuggingFace JoyCaption']);
|
||||
await expect(page.locator('#tag-existing')).toHaveValue('skip');
|
||||
await expect(page.locator('#threshold-general')).toHaveValue('0.35');
|
||||
await expect(page.locator('#threshold-character')).toHaveValue('0.85');
|
||||
await expect(page.locator('#tag-suggestions')).toHaveValue('10');
|
||||
const settings = await serverJson(request, SETTINGS_URL);
|
||||
await expect(page.locator('#tag-template')).toHaveValue(settings.settings.autotag_template);
|
||||
// D203 (#431): Saved instructions replace the template field.
|
||||
await expect(page.locator('#tag-template')).toHaveCount(0);
|
||||
// Unload is offered only while a model is loaded (none is).
|
||||
await expect(actionOf(page, 'Unload')).toBeDisabled();
|
||||
});
|
||||
@@ -418,3 +419,156 @@ test('a model that is not downloaded cannot be reviewed for a batch; the button
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(page.modelCalls.filter(call => call.includes('/batch/preview'))).toEqual([]);
|
||||
});
|
||||
|
||||
// D202 (#431): Skipped tags: add (button and Enter), the preview over the library's tags and the WD14 label list, the
|
||||
// per-chip counts, remove, Save preferences and a reload; a list the server refuses highlights the block.
|
||||
const SKIPPED_PREVIEW = '/prompt_manager/autotag/skipped-tags/preview';
|
||||
const skippedBlock = page => surface(page).locator(':scope > .setting-block[data-setting="tag-skipped"]');
|
||||
const skippedInput = page => page.locator('#tag-skipped-input');
|
||||
const skippedChips = page => skippedBlock(page).locator('.skip-chips .chip.tag-suggestion');
|
||||
const skippedPreview = page => skippedBlock(page).locator('.skip-preview');
|
||||
|
||||
test('skipped tags add, preview, remove, save and reload (D202)', async ({ page, request }) => {
|
||||
await openTagging(page);
|
||||
await expect(skippedBlock(page).locator('label')).toHaveText('Skipped tags');
|
||||
await expect(skippedBlock(page)).toHaveAttribute('data-key', 'autotag_skipped_tags');
|
||||
await expect(skippedPreview(page)).toHaveText('No tags are skipped.');
|
||||
await expect(skippedPreview(page)).toHaveAttribute('role', 'status');
|
||||
// The label list on disk (blurry, blurry_background, signature, …) and the library tags (warm_light, …).
|
||||
let answer = page.waitForResponse(item => item.url().endsWith(SKIPPED_PREVIEW));
|
||||
await skippedInput(page).fill('blurry');
|
||||
await skippedInput(page).press('Enter');
|
||||
expect((await answer).status()).toBe(200);
|
||||
await expect(skippedInput(page)).toHaveValue('');
|
||||
await expect(skippedPreview(page)).toHaveText('Hides 2 tags from suggestions: blurry, blurry_background. '
|
||||
+ 'Tags already on images are not changed.');
|
||||
answer = page.waitForResponse(item => item.url().endsWith(SKIPPED_PREVIEW));
|
||||
await skippedInput(page).fill('light');
|
||||
await page.getByRole('button', { name: 'Add', exact: true }).click();
|
||||
await answer;
|
||||
await expect(skippedPreview(page)).toHaveText('Hides 3 tags from suggestions: warm_light, blurry, blurry_background. '
|
||||
+ 'Tags already on images are not changed.');
|
||||
await expect(skippedChips(page)).toHaveText(['blurry2×', 'light1×']);
|
||||
await expect(skippedChips(page).first()).toHaveAttribute('title', 'Hides 2 tags');
|
||||
// An entry that hides nothing is muted; a duplicate is not added and the line under the input says so.
|
||||
answer = page.waitForResponse(item => item.url().endsWith(SKIPPED_PREVIEW));
|
||||
await skippedInput(page).fill('zebra');
|
||||
await skippedInput(page).press('Enter');
|
||||
await answer;
|
||||
await expect(skippedChips(page).last()).toHaveClass(/\bnone\b/);
|
||||
await expect(skippedChips(page).last()).toHaveAttribute('title', 'Hides no known tag');
|
||||
await skippedInput(page).fill('Blurry');
|
||||
await skippedInput(page).press('Enter');
|
||||
await expect(page.locator('#tag-skipped-problem')).toHaveText('Already in the list.');
|
||||
await expect(skippedInput(page)).toHaveAttribute('aria-invalid', 'true');
|
||||
await skippedInput(page).fill('a, b');
|
||||
await skippedInput(page).press('Enter');
|
||||
await expect(page.locator('#tag-skipped-problem')).toHaveText('No commas or line breaks.');
|
||||
// Remove zebra, then save.
|
||||
await skippedChips(page).last().getByRole('button', { name: 'Remove zebra', exact: true }).click();
|
||||
await expect(skippedChips(page)).toHaveCount(2);
|
||||
await expect(status(page)).toHaveText('Unsaved changes.');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Saved to the server configuration.');
|
||||
expect((await serverJson(request, SETTINGS_URL)).settings.autotag_skipped_tags).toEqual(['blurry', 'light']);
|
||||
await openTagging(page);
|
||||
await expect(skippedChips(page)).toHaveText(['blurry2×', 'light1×']);
|
||||
// Removing a chip and adding it back is no change.
|
||||
await skippedChips(page).last().getByRole('button', { name: 'Remove light', exact: true }).click();
|
||||
await expect(status(page)).toHaveText('Unsaved changes.');
|
||||
await skippedInput(page).fill('light');
|
||||
await skippedInput(page).press('Enter');
|
||||
await expect(status(page)).toHaveText('No unsaved changes.');
|
||||
});
|
||||
|
||||
test('skipped tags preview checks library tags only without a label list, and a refused save highlights it', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/tagging/defaults', { data: { general: 0.35, character: 0.85, limit: 10,
|
||||
labels: false } })).toBeOK();
|
||||
await openTagging(page);
|
||||
const answer = page.waitForResponse(item => item.url().endsWith(SKIPPED_PREVIEW));
|
||||
await skippedInput(page).fill('blurry');
|
||||
await skippedInput(page).press('Enter');
|
||||
await answer;
|
||||
await expect(skippedPreview(page)).toHaveText('Hides 0 tags from suggestions. Checked against library tags only; no WD14 '
|
||||
+ 'label list is downloaded. Tags already on images are not changed.');
|
||||
// The server is the authority: a list it refuses keeps the draft, says why and highlights the block.
|
||||
await page.route(`**${SETTINGS_URL}`, route => (route.request().method() === 'POST'
|
||||
? route.fulfill({ status: 400, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'Skipped tags: entry 1 is empty.' }) })
|
||||
: route.fallback()));
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Skipped tags: entry 1 is empty.');
|
||||
await expect(skippedBlock(page)).toHaveAttribute('aria-invalid', 'true');
|
||||
await page.unrouteAll({ behavior: 'ignoreErrors' });
|
||||
});
|
||||
|
||||
// D203 (#431): Saved instructions: Built-in read-only, New, edit, per-engine defaults, Delete resets a default, Save
|
||||
// preferences and a reload; the server pairs the two keys. D205: the batch review names the engine's default.
|
||||
const instructionOption = (page, name) => page.locator('#instruction-list [role="option"]').filter({ has: page.locator(`span:text-is("${name}")`) });
|
||||
|
||||
test('saved instructions: new, edit, per-engine default, delete, save and reload (D203)', async ({ page, request }) => {
|
||||
await openTagging(page);
|
||||
const settings = (await serverJson(request, SETTINGS_URL)).settings;
|
||||
await expect(page.locator('#instructions-count')).toHaveText('1 of 20 saved · Built\u2011in is not counted.');
|
||||
await expect(page.locator('#instruction-list [role="option"] > span:first-child')).toHaveText(['Built-in', 'Describe the image']);
|
||||
await expect(page.locator('#instruction-name')).toHaveValue('Built-in');
|
||||
await expect(page.locator('#instruction-text')).toHaveValue(settings.autotag_builtin_instructions);
|
||||
await expect(page.locator('#instruction-text')).toHaveAttribute('readonly', '');
|
||||
await expect(page.locator('#instruction-delete')).toBeDisabled();
|
||||
await expect(page.locator('#instruction-default-gguf')).toHaveValue(settings.autotag_instructions[0].id);
|
||||
// + New, then name and text it, and make it HuggingFace's default.
|
||||
await page.locator('#instruction-new').click();
|
||||
await expect(page.locator('#instruction-name')).toHaveValue('New instructions');
|
||||
await expect(page.locator('#instruction-note')).toHaveText('The text must not be empty.');
|
||||
await page.locator('#instruction-name').fill('Photo realism');
|
||||
await page.locator('#instruction-text').fill('Tag camera angle and lighting.');
|
||||
await expect(page.locator('#instruction-text').locator('xpath=..').locator('label .muted')).toHaveText('30 of 4,000 characters');
|
||||
await page.locator('#instruction-default-hf').selectOption({ label: 'Photo realism' });
|
||||
await expect(instructionOption(page, 'Photo realism').locator('.chip')).toHaveText(['Default for HuggingFace']);
|
||||
await expect(status(page)).toHaveText('Unsaved changes.');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Saved to the server configuration.');
|
||||
let saved = (await serverJson(request, SETTINGS_URL)).settings;
|
||||
expect(saved.autotag_instructions.map(entry => [entry.name, entry.text])).toEqual([['Describe the image', 'Describe the image'],
|
||||
['Photo realism', 'Tag camera angle and lighting.']]);
|
||||
expect(saved.autotag_default_instructions.hf).toBe(saved.autotag_instructions[1].id);
|
||||
// A reload shows the saved list; deleting GGUF's default resets it to Built-in and says so.
|
||||
await openTagging(page);
|
||||
await instructionOption(page, 'Describe the image').click();
|
||||
await page.locator('#instruction-delete').click();
|
||||
await expect(page.locator('#instruction-reset')).toHaveText('GGUF now uses Built-in.');
|
||||
await expect(page.locator('#instruction-default-gguf')).toHaveValue('builtin');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Saved to the server configuration.');
|
||||
saved = (await serverJson(request, SETTINGS_URL)).settings;
|
||||
expect(saved.autotag_instructions.map(entry => entry.name)).toEqual(['Photo realism']);
|
||||
expect(saved.autotag_default_instructions.gguf).toBe('builtin');
|
||||
expect(saved.autotag_template).toBe(saved.autotag_builtin_instructions);
|
||||
});
|
||||
|
||||
test('a refused instructions save names the entry and highlights the block (D203)', async ({ page }) => {
|
||||
await openTagging(page);
|
||||
await page.locator('#instruction-duplicate').click();
|
||||
await page.locator('#instruction-name').fill('describe the IMAGE');
|
||||
await expect(page.locator('#instruction-note')).toHaveText('The name “describe the IMAGE” is already used.');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Saved instructions: entry 2 name “describe the IMAGE” is already used.');
|
||||
await expect(surface(page).locator('.setting-block[data-setting="tag-instructions"]')).toHaveAttribute('aria-invalid', 'true');
|
||||
});
|
||||
|
||||
test('the batch review names the language model\'s default instructions (D205)', async ({ page }) => {
|
||||
await openTagging(page);
|
||||
await page.locator('#tag-model').selectOption('gguf');
|
||||
await page.locator('#tag-existing').selectOption('merge');
|
||||
const answer = page.waitForResponse(item => item.url().endsWith(PREVIEW_URL));
|
||||
await reviewButton(page).click();
|
||||
await answer;
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review tag output images');
|
||||
const grid = await gridOf(page);
|
||||
expect(grid.slice(-3)).toEqual([['Instructions', 'Describe the image'], ['Settings', 'Up to 10 tags per image'],
|
||||
grid.at(-1)]);
|
||||
// The batch sends the default's text.
|
||||
const start = page.waitForRequest(item => item.url().endsWith('/prompt_manager/jobs/autotag'));
|
||||
await dialog(page).getByRole('button', { name: 'Start tagging', exact: true }).click();
|
||||
expect((await start).postDataJSON().options).toEqual({ instructions: 'Describe the image' });
|
||||
});
|
||||
|
||||
@@ -8,6 +8,8 @@ import { readCoverageTable, parseControl, CONTROL_EXEMPTIONS } from '../../tools
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const OVERVIEW_URL = '/prompt_manager/system/overview';
|
||||
const THUMBNAILS_URL = '/prompt_manager/images/thumbnails/status';
|
||||
// The sidebar's Images count (state.mjs NAV_COUNT_SOURCES): the Images page's own total.
|
||||
const IMAGES_COUNT_URL = '/prompt_manager/images/output?mode=workspace&limit=1';
|
||||
const number = value => value.toLocaleString('en-US');
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
@@ -55,6 +57,7 @@ test('overview shows real counts and Not checked', async ({ page, request }) =>
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) writes.push(item.url()); });
|
||||
const overview = await serverJson(request, OVERVIEW_URL);
|
||||
const thumbnails = await serverJson(request, THUMBNAILS_URL);
|
||||
const images = (await serverJson(request, IMAGES_COUNT_URL)).total;
|
||||
// The concept fixture: six positives, their saved negatives and 24 linked images, all with thumbnails.
|
||||
expect([overview.library.positive_prompts, overview.images.indexed, thumbnails.pending]).toEqual([6, 24, 0]);
|
||||
// I2: the Overview never walks the sources for thumbnails; the row reads the status route itself.
|
||||
@@ -66,7 +69,8 @@ test('overview shows real counts and Not checked', async ({ page, request }) =>
|
||||
[item.querySelector('strong').textContent, item.querySelector('span').textContent]))).toEqual([
|
||||
[number(overview.library.positive_prompts), 'Positive prompts'],
|
||||
[number(overview.library.negative_variants), 'Negative variants'],
|
||||
[number(overview.images.indexed), 'Images'],
|
||||
// #555: the Images page's own count, the same number as its sidebar badge.
|
||||
[number(images), 'Images'],
|
||||
['Not checked', 'Database integrity'],
|
||||
]);
|
||||
expect(await rowsOf(layout, 'Library health')).toEqual([
|
||||
@@ -79,11 +83,31 @@ test('overview shows real counts and Not checked', async ({ page, request }) =>
|
||||
expect(await rowsOf(layout, 'Workspace preferences')).toEqual([
|
||||
['One display configuration', 'Grid and density settings are shared across views.', 'Appearance'],
|
||||
['2 output sources', 'Monitoring and enrichment share one source list.', 'Manage sources'],
|
||||
['Statistics', 'Runs by category, with library totals.', 'Explore'],
|
||||
['Statistics', 'Runs by category, with database record counts.', 'Explore'],
|
||||
]);
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
|
||||
test('#555: the Images metric is the sidebar Images count, not the indexed image rows', async ({ page, request }) => {
|
||||
const overview = await serverJson(request, OVERVIEW_URL);
|
||||
// The Images page lists files no row links (the demo library: 122 files, 110 rows); the listing says so.
|
||||
const listed = overview.images.indexed + 12;
|
||||
await page.route(`**${IMAGES_COUNT_URL}`, async route => {
|
||||
const response = await route.fetch();
|
||||
await route.fulfill({ response, json: { ...(await response.json()), total: listed } });
|
||||
});
|
||||
const layout = await openOverview(page);
|
||||
const imagesMetric = layout.locator('.summary-metrics > .metric').filter({ has: page.locator('span', { hasText: /^Images$/ }) });
|
||||
await expect(imagesMetric.locator('strong')).toHaveText(number(listed));
|
||||
await expect(page.locator('#primary-nav a[data-route="images"] .nav-count')).toHaveText(number(listed));
|
||||
// A failed count reads as a dash, never the row count.
|
||||
await page.unroute(`**${IMAGES_COUNT_URL}`);
|
||||
await page.route(`**${IMAGES_COUNT_URL}`, route => route.fulfill({ status: 503, json: { success: false, error: 'Unavailable' } }));
|
||||
await page.goto(`${workspace}#/system/appearance`);
|
||||
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect(imagesMetric.locator('strong')).toHaveText('–');
|
||||
});
|
||||
|
||||
// D129 (#249 S6b): where the library lives. The hook makes the fixture's database read as a node-folder library; its
|
||||
// Move library is a stand-in that touches no file (workspace_system_fixture.py).
|
||||
const banner = page => page.locator('#library-banner');
|
||||
@@ -364,7 +388,7 @@ test('View backups / Appearance / Manage sources switch sections', async ({ page
|
||||
}
|
||||
});
|
||||
|
||||
test('statistics dialog lists runs by category and library totals', async ({ page, request }) => {
|
||||
test('statistics dialog lists runs by category and database record counts', async ({ page, request }) => {
|
||||
const statistics = await serverJson(request, '/prompt_manager/system/statistics');
|
||||
expect(statistics.by_category.length).toBeGreaterThan(1);
|
||||
const layout = await openOverview(page);
|
||||
@@ -373,13 +397,14 @@ test('statistics dialog lists runs by category and library totals', async ({ pag
|
||||
const dialog = page.locator('dialog#modal[open]');
|
||||
await expect(dialog.locator('#modal-title')).toHaveText('Usage statistics');
|
||||
const tables = dialog.locator('#modal-body > table');
|
||||
await expect(tables.locator('caption')).toHaveText(['Runs by category', 'Library totals']);
|
||||
await expect(tables.locator('caption')).toHaveText(['Runs by category', 'Database records']);
|
||||
// D153 (#447): no rating column and no rated-images total.
|
||||
await expect(tables.first().locator('thead th')).toHaveText(['Category', 'Runs']);
|
||||
const cells = table => table.locator('tbody tr').evaluateAll(rows => rows.map(row => [...row.cells].map(cell => cell.textContent)));
|
||||
expect(await cells(tables.first())).toEqual(statistics.by_category.map(row => [row.category ?? 'Uncategorized', number(row.runs)]));
|
||||
expect(await cells(tables.nth(1))).toEqual([['Prompts', number(statistics.totals.total_prompts)],
|
||||
['Tags', number(statistics.totals.total_tags)], ['Images', number(statistics.totals.total_images)]]);
|
||||
// #555: raw rows of the legacy /stats, labelled as rows so they never read as the Library or Images page counts.
|
||||
expect(await cells(tables.nth(1))).toEqual([['Prompt records', number(statistics.totals.total_prompts)],
|
||||
['Tags on prompts', number(statistics.totals.total_tags)], ['Image records', number(statistics.totals.total_images)]]);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(explore).toBeFocused();
|
||||
// A statistics read that fails says so; the Overview stays.
|
||||
|
||||
@@ -341,3 +341,21 @@ test('a running scan makes the start report busy', async ({ page, request }) =>
|
||||
await request.post(`/__test__/jobs/${scan.job.id}/finish`, { data: { state: 'finished' } });
|
||||
}
|
||||
});
|
||||
|
||||
// D204 (#431): Auto-tag selected has the same picker; the job sends the text the dialog shows.
|
||||
test('Auto-tag selected picks saved instructions and sends the text shown (D204)', async ({ page }) => {
|
||||
await select(page, FIRST);
|
||||
await barButton(page, 'Auto-tag selected').click();
|
||||
await expect(title(page)).toHaveText('Auto-tag selected images');
|
||||
const body = modal(page).locator('#modal-body');
|
||||
await body.getByRole('combobox', { name: 'Tagging model', exact: true }).selectOption('gguf');
|
||||
const pick = body.getByRole('combobox', { name: 'Saved instructions', exact: true });
|
||||
await expect(pick).toHaveAttribute('id', 'bulk-tag-instructions-pick');
|
||||
await expect(pick.locator('option')).toHaveText(['Built-in', 'Describe the image · default']);
|
||||
const text = body.getByRole('textbox', { name: 'Tagging instructions', exact: true });
|
||||
await text.fill('Bulk one-off.');
|
||||
await expect(pick.locator('option:checked')).toHaveText('Edited for this run');
|
||||
const started = jobStart(page);
|
||||
await actionButtons(page).filter({ hasText: 'Start auto-tagging' }).click();
|
||||
expect((await started).postDataJSON().options.instructions).toBe('Bulk one-off.');
|
||||
});
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { saveImageTags } from './concept-inventory.mjs';
|
||||
import { taggingInventory } from './tagging-canonical-comparison.mjs';
|
||||
import { FIXTURE_INSTRUCTIONS_ID, taggingInventory } from './tagging-canonical-comparison.mjs';
|
||||
|
||||
const images = '/prompt_manager/workspace#/images';
|
||||
const MODULE = '/prompt_manager/js/workspace/image-tagging.mjs';
|
||||
@@ -849,3 +849,61 @@ test('a failed listing re-read after a save says so and keeps the cards (M2)', a
|
||||
await expect(page.locator('#gallery-continuation [aria-disabled="true"]')).toHaveCount(0);
|
||||
await page.unroute(failing);
|
||||
});
|
||||
|
||||
// #431: the saved Skipped tags hide matching suggestions before the limit (WD14 fills it from lower scores), and the
|
||||
// chip hint says how many were not suggested.
|
||||
test('suggestions omit skipped tags and say how many were hidden', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/tagging/defaults', { data: { general: 0.35, character: 0.85, limit: 4,
|
||||
skipped: ['desert', 'rock'] } })).toBeOK();
|
||||
await openTag(page);
|
||||
const answer = await (await generated(page)).json();
|
||||
expect(answer.skipped).toBe(2);
|
||||
await expect(chips(page).locator(':scope > span:first-child')).toHaveText(['sandstone', 'arch', 'sky', 'dune']);
|
||||
await expect(dialog(page).locator('#tag-suggestions > p.small')).toHaveText(`${CHIP_HINT} 2 skipped tags were not suggested.`);
|
||||
});
|
||||
|
||||
// D204 (#431): the Saved instructions select above Tagging instructions: the engine default, a pick, a one-off edit
|
||||
// ("Edited for this run" and the muted note), an engine switch resets, WD14 hides both; Generate sends the text shown.
|
||||
test('the saved instructions picker fills, edits one-off and resets per engine (D204)', async ({ page, request }) => {
|
||||
const settings = await (await request.get('/prompt_manager/settings')).json();
|
||||
const builtin = settings.settings.autotag_builtin_instructions;
|
||||
await openTag(page);
|
||||
const options = dialog(page).locator('#image-tag-model-options');
|
||||
const pick = options.getByRole('combobox', { name: 'Saved instructions', exact: true });
|
||||
const text = options.getByRole('textbox', { name: 'Tagging instructions', exact: true });
|
||||
const note = options.locator('.instructions-edited');
|
||||
await expect(pick).toHaveCount(0);
|
||||
await model(page).selectOption('gguf');
|
||||
await expect(pick.locator('option')).toHaveText(['Built-in', 'Describe the image · default']);
|
||||
await expect(pick).toHaveAttribute('id', 'image-tag-instructions-pick');
|
||||
await expect(text).toHaveValue('Describe the image');
|
||||
await expect(note).toBeHidden();
|
||||
// A keyboard pick (the select focused) keeps focus on the select the fields are re-rendered with.
|
||||
await pick.focus();
|
||||
await pick.selectOption('builtin');
|
||||
await expect(text).toHaveValue(builtin);
|
||||
await expect(pick).toBeFocused();
|
||||
await text.fill('Only the sky.');
|
||||
await expect(text).toHaveAttribute('aria-describedby', 'image-tag-instructions-pick-note');
|
||||
await expect(pick.locator('option:checked')).toHaveText('Edited for this run');
|
||||
await expect(pick.locator('option[value="edited"]')).toBeDisabled();
|
||||
await expect(note).toHaveText('Edits apply to this run only. Manage saved instructions in System › Auto-tagging.');
|
||||
await expect(text).toBeFocused();
|
||||
// Generate sends the edited text; the saved entries are unchanged.
|
||||
const sent = page.waitForRequest(item => item.url().endsWith('/image-annotations/suggest'));
|
||||
await generate(page).click();
|
||||
expect((await sent).postDataJSON().options.instructions).toBe('Only the sky.');
|
||||
// Choosing an entry replaces the edit; switching engines starts from the new engine's default.
|
||||
await pick.selectOption('builtin');
|
||||
await expect(text).toHaveValue(builtin);
|
||||
await expect(note).toBeHidden();
|
||||
await text.fill('Edited again.');
|
||||
await model(page).selectOption('hf');
|
||||
await expect(text).toHaveValue('Describe the image');
|
||||
await expect(pick).toHaveValue(FIXTURE_INSTRUCTIONS_ID);
|
||||
await model(page).selectOption('wd14-vit');
|
||||
await expect(pick).toHaveCount(0);
|
||||
await expect(text).toHaveCount(0);
|
||||
expect((await (await request.get('/prompt_manager/settings')).json()).settings.autotag_instructions)
|
||||
.toEqual(settings.settings.autotag_instructions);
|
||||
});
|
||||
|
||||
@@ -145,9 +145,10 @@ test('long vocabularies append and Load more is keyboard reachable',async({page,
|
||||
await expect(loadMore(page)).toHaveCount(0);
|
||||
await expect(rows(page).nth(200).getByRole('button',{name:'Browse',exact:true})).toBeFocused();
|
||||
await expect(rows(page).locator('td:nth-child(1)')).toHaveText(manifest.tags);
|
||||
// Reload with the window at its top: Firefox restores a scrolled window, whose end then appends a page before the check.
|
||||
await main(page).locator('.two-col .table-wrap').evaluate(node=>new Promise(resolve=>{node.scrollTop=0;requestAnimationFrame(()=>requestAnimationFrame(resolve));}));
|
||||
await page.reload();
|
||||
// Open the screen as a new history entry, not a reload: Firefox restores a reloaded page's scrolled tag window, and
|
||||
// its end then appends a page before the check (#567). Scrolling the window to its top first does not always stick.
|
||||
await page.goto('about:blank');
|
||||
await page.goto(route);
|
||||
await expect(rows(page)).toHaveCount(100);
|
||||
// D183: the end of the list arrives inside the tag window.
|
||||
const scroll=top=>main(page).locator('.two-col .table-wrap').evaluate((node,y)=>new Promise(resolve=>{node.scrollTop=y ?? node.scrollHeight;requestAnimationFrame(()=>requestAnimationFrame(resolve));}),top);
|
||||
|
||||
@@ -41,6 +41,11 @@ read. These per-server hooks only change what a System test needs on top:
|
||||
- ``POST /__test__/system/duplicate-prompts`` (S7) adds two prompts whose
|
||||
text differs only in case and spacing (a duplicate prompt text group);
|
||||
``{"again": true}`` adds a third, so a reviewed plan no longer matches;
|
||||
- ``POST /__test__/system/pair-metadata`` (#550) saves two saved pairs the
|
||||
way the Pair node did before #546 (the pair holds the category and tags,
|
||||
its new positive prompt none): ``S550 lantern`` (category ``night``, tags
|
||||
``s550-glow``, ``s550-warm``) fills; ``S550 harbor`` was edited after it was
|
||||
saved, so the plan keeps it;
|
||||
- ``POST /__test__/system/duplicate-files`` (S7) writes one PNG's bytes three
|
||||
times: ``output/s7-dupes/same.png``, ``input/s7-dupes/same.png`` (the same
|
||||
relative path in the other source) and ``output/s7-dupes/copy.png``;
|
||||
@@ -344,6 +349,25 @@ def register_system_commands(app, routes, api, root: Path):
|
||||
conn.commit()
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/system/pair-metadata")
|
||||
async def pair_metadata(request):
|
||||
def seed():
|
||||
api.db.save_prompt_pair(
|
||||
"S550 lantern",
|
||||
"S550 blurry",
|
||||
category="night",
|
||||
tags=["s550-glow", "s550-warm"],
|
||||
)
|
||||
api.db.save_prompt_pair("S550 harbor", category="night")
|
||||
with api.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"UPDATE prompts SET updated_at='2099-01-01T00:00:00+00:00' "
|
||||
"WHERE text='S550 harbor'"
|
||||
)
|
||||
|
||||
await api._run_in_executor(seed)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/system/duplicate-files")
|
||||
async def duplicate_files(request):
|
||||
body = await request.json() if request.can_read_body else {}
|
||||
|
||||
@@ -226,6 +226,7 @@ def register_tagging_commands(
|
||||
return download["result"]
|
||||
|
||||
_set_defaults(model, DEFAULTS) # the server starts on the standard ones
|
||||
_write_labels(model, True)
|
||||
|
||||
@routes.post("/__test__/tagging/reset")
|
||||
async def reset(request: web.Request) -> web.Response:
|
||||
@@ -244,6 +245,7 @@ def register_tagging_commands(
|
||||
model.loaded = None
|
||||
restore_download()
|
||||
_set_defaults(model, DEFAULTS)
|
||||
await api._run_in_executor(_write_labels, model, True)
|
||||
# A test that ends with the dialog open leaves its capture behind; drop
|
||||
# every capture so the next test starts below the store's limits.
|
||||
await api._run_in_executor(api.close_annotation_captures)
|
||||
@@ -353,13 +355,22 @@ def register_tagging_commands(
|
||||
|
||||
``{"general": g, "character": c, "limit": n}``: the WD14 thresholds
|
||||
(in [0, 1]) and the persisted suggestion count (1..100), so a test can
|
||||
tell the server's values from the dialog's built-in ones. Reset
|
||||
restores the standard 0.35 / 0.85 / 10.
|
||||
tell the server's values from the dialog's built-in ones. Optional
|
||||
``skipped`` sets the saved Skipped tags (#431) and ``labels`` (true or
|
||||
false) writes or removes the fake WD14 SwinV2 label list the Skipped
|
||||
tags preview reads. Reset restores the standard 0.35 / 0.85 / 10, no
|
||||
skipped tags and the label list.
|
||||
"""
|
||||
data = await request.json() if request.can_read_body else None
|
||||
if request.query or not _valid_defaults(data):
|
||||
raise web.HTTPBadRequest()
|
||||
_set_defaults(model, (data["general"], data["character"], data["limit"]))
|
||||
if "skipped" in data:
|
||||
from py.config import AutotagConfig
|
||||
|
||||
AutotagConfig.SKIPPED_TAGS = tuple(data["skipped"])
|
||||
if "labels" in data:
|
||||
await api._run_in_executor(_write_labels, model, data["labels"])
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.get("/__test__/tagging/captures")
|
||||
@@ -380,10 +391,15 @@ DOWNLOAD_HOLD_LIMIT_S = 60
|
||||
DEFAULTS = (0.35, 0.85, 10)
|
||||
|
||||
|
||||
# The saved "Describe the image" entry's id (#431).
|
||||
FIXTURE_INSTRUCTIONS_ID = "d35c71be0a1e"
|
||||
|
||||
|
||||
def _set_defaults(model, values) -> None:
|
||||
"""Set the saved tagging defaults (``AutotagConfig``, #328); a reset also
|
||||
puts back the saved model, batch policy and template (the fake model's
|
||||
short instructions, which the concept comparisons show)."""
|
||||
puts back the saved model, batch policy, skipped tags and instructions
|
||||
(the fake model's short instructions, which the concept comparisons
|
||||
show)."""
|
||||
from py.config import AutotagConfig
|
||||
|
||||
general, character, limit = values
|
||||
@@ -393,7 +409,48 @@ def _set_defaults(model, values) -> None:
|
||||
if values == DEFAULTS:
|
||||
AutotagConfig.MODEL_TYPE = "wd14-swinv2"
|
||||
AutotagConfig.EXISTING = "skip"
|
||||
AutotagConfig.TEMPLATE = model.default_prompt
|
||||
# #431: Built-in is always the shipped text, so the fake model's short
|
||||
# instructions are a saved entry, the default of both engines (the
|
||||
# concept comparisons show it).
|
||||
AutotagConfig.INSTRUCTIONS = (
|
||||
{
|
||||
"id": FIXTURE_INSTRUCTIONS_ID,
|
||||
"name": model.default_prompt,
|
||||
"text": model.default_prompt,
|
||||
},
|
||||
)
|
||||
AutotagConfig.DEFAULT_INSTRUCTIONS = {
|
||||
"gguf": FIXTURE_INSTRUCTIONS_ID,
|
||||
"hf": FIXTURE_INSTRUCTIONS_ID,
|
||||
}
|
||||
AutotagConfig._LEGACY_TEMPLATE = None
|
||||
AutotagConfig.SKIPPED_TAGS = ()
|
||||
|
||||
|
||||
# The fake WD14 SwinV2 label list (general 0, character 4, rating 9): the
|
||||
# names the Skipped tags preview checks beside the library's tags (#431).
|
||||
FAKE_LABELS = (
|
||||
("blurry", "0"),
|
||||
("blurry_background", "0"),
|
||||
("signature", "0"),
|
||||
("watermark", "0"),
|
||||
("t-shirt", "0"),
|
||||
("half-closed_eyes", "0"),
|
||||
("saber_(fate)", "4"),
|
||||
("general", "9"),
|
||||
)
|
||||
|
||||
|
||||
def _write_labels(model, present: bool) -> None:
|
||||
"""Write (or remove) the fake label list under the fake models folder."""
|
||||
path = Path(model.models_dir) / "WD14-SwinV2" / "selected_tags.csv"
|
||||
if not present:
|
||||
path.unlink(missing_ok=True)
|
||||
return
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
lines = ["tag_id,name,category,count"]
|
||||
lines += [f"{i},{name},{cat},1" for i, (name, cat) in enumerate(FAKE_LABELS)]
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _valid_defaults(data: object) -> bool:
|
||||
@@ -402,9 +459,20 @@ def _valid_defaults(data: object) -> bool:
|
||||
and not isinstance(value, bool)
|
||||
and 0 <= value <= 1
|
||||
)
|
||||
optional = {"skipped", "labels"}
|
||||
if isinstance(data, dict) and "skipped" in data:
|
||||
try:
|
||||
from py.tagging_defaults import validate_skipped_tags
|
||||
|
||||
validate_skipped_tags(data["skipped"])
|
||||
except ValueError:
|
||||
return False
|
||||
if isinstance(data, dict) and not isinstance(data.get("labels", True), bool):
|
||||
return False
|
||||
return (
|
||||
isinstance(data, dict)
|
||||
and set(data) == {"general", "character", "limit"}
|
||||
and {"general", "character", "limit"} <= set(data)
|
||||
and set(data) <= {"general", "character", "limit"} | optional
|
||||
and unit(data["general"])
|
||||
and unit(data["character"])
|
||||
and type(data["limit"]) is int
|
||||
|
||||
@@ -131,11 +131,40 @@ test('native image inspector previews and sends a pair into real visible widgets
|
||||
const twinId=await page.evaluate(async()=>{const {app}=await import('/scripts/app.js');const node=LiteGraph.createNode('PromptManagerPair');app.graph.add(node);return node.id;});
|
||||
await find0.click();
|
||||
const targets=sender.getByRole('combobox',{name:'Destination node',exact:true});
|
||||
await expect(targets.locator('option')).toHaveCount(3);
|
||||
await expect(targets.locator('option')).toHaveCount(4);
|
||||
await expect(targets).toHaveValue('');
|
||||
await expect(apply).toBeDisabled();
|
||||
await expect(targets.locator('option').nth(2)).toHaveText(new RegExp(`^#${twinId} `));
|
||||
await page.evaluate(async twinId=>{const {app}=await import('/scripts/app.js');app.graph.remove(app.graph.getNodeById(twinId));},twinId);
|
||||
// #542 D191: "All 2 eligible nodes" writes both real nodes, one after another; a linked third node is not offered.
|
||||
const linkedId=await page.evaluate(async()=>{
|
||||
const {app}=await import('/scripts/app.js');const {convertToInput}=await import('/extensions/core/widgetInputs.js');
|
||||
const node=LiteGraph.createNode('PromptManagerPair');app.graph.add(node);
|
||||
const widget=node.widgets.find(widget=>widget.name==='text');
|
||||
if(!node.inputs.some(input=>input.name==='text'))convertToInput(node,widget);
|
||||
const source=LiteGraph.createNode('PrimitiveString');app.graph.add(source);
|
||||
source.connect(0,node,node.inputs.findIndex(input=>input.name==='text'));
|
||||
for(const each of app.graph._nodes.filter(item=>item.type==='PromptManagerPair')){const text=each.widgets.find(widget=>widget.name==='text');if(text)text.value=`Before all ${each.id}`;}
|
||||
return node.id;
|
||||
});
|
||||
await find0.click();
|
||||
await expect(targets.locator('option')).toHaveText(['Choose a destination node',new RegExp(`^#${id} `),new RegExp(`^#${twinId} `),'All 2 eligible nodes']);
|
||||
await expect(sender.getByRole('status')).toContainText('1 other Prompt Manager node cannot receive this prompt.');
|
||||
await targets.selectOption({label:'All 2 eligible nodes'});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();
|
||||
await expect(sender.locator('section.prompt-sender-node > h4')).toHaveText([new RegExp(`^#${id} `),new RegExp(`^#${twinId} `)]);
|
||||
await apply.click();
|
||||
// The tab title gains ComfyUI's unsaved mark once the workflow changed, so the window is matched loosely.
|
||||
await expect(sender.getByRole('status')).toHaveText(/^Sent to 2 of 2 nodes in .*ComfyUI\.$/);
|
||||
await expect(apply).toBeDisabled();
|
||||
const all=await page.evaluate(async ids=>{
|
||||
const {app}=await import('/scripts/app.js');const output=(await app.graphToPrompt()).output;
|
||||
return ids.map(nodeId=>{const node=app.graph.getNodeById(nodeId);const serialized=node.serialize();const index=node.widgets.findIndex(widget=>widget.name==='negative_text');
|
||||
return{prompt:output[String(nodeId)]?.inputs.negative_text,serialized:serialized.widgets_values[index],text:node.widgets.find(widget=>widget.name==='text')?.value};});
|
||||
},[id,twinId,linkedId]);
|
||||
expect(all.slice(0,2)).toEqual([{prompt:negative,serialized:negative,text:positive},{prompt:negative,serialized:negative,text:positive}]);
|
||||
expect(all[2].text).toBe(`Before all ${linkedId}`);
|
||||
expect(await page.evaluate(()=>window.senderEffects.queue-window.senderSendBaseline.queue)).toBe(0);
|
||||
await page.evaluate(async ids=>{const {app}=await import('/scripts/app.js');for(const nodeId of ids)app.graph.remove(app.graph.getNodeById(nodeId));},[twinId,linkedId]);
|
||||
// #535: the status line says why a window cannot receive.
|
||||
const find=sender.getByRole('button',{name:'Find ComfyUI windows',exact:true});
|
||||
const preview=sender.getByRole('button',{name:'Preview changes',exact:true});
|
||||
|
||||
@@ -105,6 +105,7 @@
|
||||
"POST /prompt_manager/autotag/batch/preview",
|
||||
"POST /prompt_manager/autotag/download/{model_type}",
|
||||
"POST /prompt_manager/autotag/single",
|
||||
"POST /prompt_manager/autotag/skipped-tags/preview",
|
||||
"POST /prompt_manager/autotag/start",
|
||||
"POST /prompt_manager/autotag/unload",
|
||||
"POST /prompt_manager/bulk/category",
|
||||
@@ -184,10 +185,14 @@
|
||||
"monitor.scheduled",
|
||||
"revision",
|
||||
"settings",
|
||||
"settings.autotag_builtin_instructions",
|
||||
"settings.autotag_character_threshold",
|
||||
"settings.autotag_default_instructions",
|
||||
"settings.autotag_existing",
|
||||
"settings.autotag_general_threshold",
|
||||
"settings.autotag_instructions",
|
||||
"settings.autotag_model_type",
|
||||
"settings.autotag_skipped_tags",
|
||||
"settings.autotag_suggestion_limit",
|
||||
"settings.autotag_template",
|
||||
"settings.cpu_count",
|
||||
|
||||
@@ -21,14 +21,6 @@ function jsonResponse(status, body) {
|
||||
json: async () => body };
|
||||
}
|
||||
|
||||
function sseResponse(frames) {
|
||||
const bytes = new TextEncoder().encode(frames.map(frame => `data: ${JSON.stringify(frame)}\n\n`).join(''));
|
||||
let sent = false;
|
||||
return { ok: true, status: 200, body: { getReader: () => ({
|
||||
read: async () => (sent ? { done: true } : (sent = true, { done: false, value: bytes })),
|
||||
}) } };
|
||||
}
|
||||
|
||||
function load(file, className, fetchUrl, extra = {}, dom = elements()) {
|
||||
const timers = [];
|
||||
const notifications = [];
|
||||
@@ -63,8 +55,6 @@ function load(file, className, fetchUrl, extra = {}, dom = elements()) {
|
||||
return { page, dom, timers, notifications, storage };
|
||||
}
|
||||
|
||||
const BUSY = 'Scan for metadata already running';
|
||||
|
||||
test('admin settings save shows why config.json refused it', async () => {
|
||||
const message = 'config.json cannot be read, so nothing was saved. Fix or remove the file, then save again.';
|
||||
const { page, notifications } = load('admin.js', 'PromptAdmin',
|
||||
|
||||
@@ -225,3 +225,141 @@ test('acknowledgement cannot claim an extra role beyond approved changes',async(
|
||||
const {createSendSession}=await source;const s=setup(createSendSession,{apply:async()=>({targetId:'one',values:{positive:'new',negative:'also changed'}})});
|
||||
await s.session.discover();s.session.selectHost('host');s.session.selectTarget('one');await s.session.preview();await s.session.apply();assert.equal(s.session.state.phase,'error');assert.match(s.session.state.message,/unexpected acknowledgement/);s.session.dispose();
|
||||
});
|
||||
// #542: "All N eligible nodes" sends to every eligible node of one window, previewing every node first (Q542-2 C).
|
||||
function allHost({nodes=['10','9','2'],revision,preview,apply,discover}={}) {
|
||||
const calls=[];const sent=new Set();
|
||||
const targets=nodes.map(id=>pm(id,['positive'],revision?{revision}:{}));
|
||||
const client={
|
||||
discover:discover ?? (async()=>[window1(targets)]),
|
||||
preview:async(host,id,roles)=>{calls.push(['preview',id]);if(preview)return preview(id,calls,sent);return{ticket:`t-${id}-${calls.length}`,revision,target:{id,label:`Node ${id}`},fields:[{role:'positive',before:sent.has(id)?'new':'old',after:roles.positive}]};},
|
||||
apply:async(host,ticket)=>{const id=ticket.split('-')[1];calls.push(['apply',id,ticket]);if(apply)return apply(id,calls);sent.add(id);return{targetId:id,values:{positive:'new'}};},
|
||||
dispose:()=>calls.push(['dispose']),
|
||||
};
|
||||
return {client,calls,sent};
|
||||
}
|
||||
async function allSession(createSendSession,options) {
|
||||
const host=allHost(options);const session=createSendSession({client:host.client,roles:{positive:'new'},onChange:()=>{}});
|
||||
await session.discover();return {...host,session};
|
||||
}
|
||||
const order=calls=>calls.filter(call=>call[0]!=='dispose').map(call=>`${call[0]} ${call[1]}`);
|
||||
test('All is offered only with two or more eligible nodes and is never selected for you',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;
|
||||
const one=await allSession(createSendSession,{nodes:['1']});
|
||||
one.session.selectTarget(ALL_NODES);assert.equal(one.session.state.targetId,'');assert.equal(one.calls.length,0);one.session.dispose();
|
||||
const two=await allSession(createSendSession,{nodes:['1','2']});
|
||||
assert.equal(two.session.state.targetId,'');
|
||||
two.session.selectTarget(ALL_NODES);assert.equal(two.session.state.targetId,ALL_NODES);
|
||||
assert.match(two.session.state.message,/all 2 eligible nodes in Flow - ComfyUI/);
|
||||
await two.session.discover();assert.equal(two.session.state.targetId,'');two.session.dispose();
|
||||
});
|
||||
test('Apply to All previews every node first, then sends one after another in node id order',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;const s=await allSession(createSendSession);
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.deepEqual(order(s.calls),['preview 2','preview 9','preview 10','apply 2','preview 9','apply 9','preview 10','apply 10']);
|
||||
assert.equal(s.session.state.phase,'success');assert.equal(s.session.state.message,'Sent to 3 of 3 nodes in Flow - ComfyUI.');
|
||||
await s.session.apply();assert.equal(s.calls.filter(call=>call[0]==='apply').length,3);s.session.dispose();
|
||||
});
|
||||
test('a double click on Apply to All sends to each node once',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;const s=await allSession(createSendSession,{nodes:['1','2']});
|
||||
s.session.selectTarget(ALL_NODES);await Promise.all([s.session.apply(),s.session.apply()]);
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['1','2']);s.session.dispose();
|
||||
});
|
||||
test('a node refusing its preview stops All before anything is written and names the node',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;
|
||||
const s=await allSession(createSendSession,{preview:id=>{if(id==='9')throw Error('positive is connected to an upstream input.');return{ticket:`t-${id}`,target:{id},fields:[{role:'positive',before:'old',after:'new'}]};}});
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.equal(s.calls.filter(call=>call[0]==='apply').length,0);assert.equal(s.session.state.phase,'error');
|
||||
assert.match(s.session.state.message,/^#9 Node 9: positive is connected to an upstream input\. Nothing was sent\./);
|
||||
assert.equal(s.session.state.outcomeUnknown,false);s.session.dispose();
|
||||
});
|
||||
test('a refusal while sending to All is reported by node and the other nodes are still sent',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;
|
||||
const s=await allSession(createSendSession,{apply:id=>{if(id==='9')throw Error('Workflow or prompt changed. Preview again.');return{targetId:id,values:{positive:'new'}};}});
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['2','9','10']);
|
||||
assert.equal(s.session.state.phase,'error');assert.equal(s.session.state.outcomeUnknown,false);
|
||||
assert.equal(s.session.state.message,'Sent to 2 of 3 nodes in Flow - ComfyUI. #9 Node 9: Workflow or prompt changed. Preview again.');s.session.dispose();
|
||||
});
|
||||
test('a lost reply on one node of All leaves the later nodes unsent and that outcome unknown',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;
|
||||
const s=await allSession(createSendSession,{apply:id=>{if(id==='9')throw Error('Bridge request timed out');return{targetId:id,values:{positive:'new'}};}});
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['2','9']);
|
||||
assert.equal(s.session.state.outcomeUnknown,true);
|
||||
assert.match(s.session.state.message,/^Sent to 1 of 3 nodes in Flow - ComfyUI\. No confirmation received from #9 Node 9; the update may have applied\. Not sent: #10 Node 10\. Check ComfyUI, then preview again\. Bridge request timed out$/);
|
||||
await s.session.apply();assert.equal(s.calls.filter(call=>call[0]==='apply').length,2);s.session.dispose();
|
||||
});
|
||||
test('closing the dialog during All stops after the current node',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;let release;
|
||||
const s=await allSession(createSendSession,{apply:id=>id==='2'?new Promise(done=>release=()=>done({targetId:id,values:{positive:'new'}})):{targetId:id,values:{positive:'new'}}});
|
||||
s.session.selectTarget(ALL_NODES);const pending=s.session.apply();
|
||||
for(let i=0;i<50 && !release;i++)await tick();assert.ok(release,'the first node was sent');
|
||||
s.session.dispose();release();await pending;
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['2']);
|
||||
});
|
||||
test('All accepts the revision our own write moved, but not a workflow changed before the run',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;
|
||||
// The host moves the graph version on every write: later nodes preview with the new revision.
|
||||
let version=4;
|
||||
const bumped=await allSession(createSendSession,{revision:'1:4',preview:id=>({ticket:`t-${id}`,revision:`1:${version}`,target:{id},fields:[{role:'positive',before:'old',after:'new'}]}),apply:id=>{version++;return{targetId:id,values:{positive:'new'}};}});
|
||||
bumped.session.selectTarget(ALL_NODES);await bumped.session.apply();
|
||||
assert.equal(bumped.session.state.message,'Sent to 3 of 3 nodes in Flow - ComfyUI.');bumped.session.dispose();
|
||||
const moved=await allSession(createSendSession,{revision:'1:4',preview:id=>({ticket:`t-${id}`,revision:'1:5',target:{id},fields:[{role:'positive',before:'old',after:'new'}]})});
|
||||
moved.session.selectTarget(ALL_NODES);await moved.session.apply();
|
||||
assert.match(moved.session.state.message,/workflow changed since this window was found/i);assert.equal(moved.calls.filter(call=>call[0]==='apply').length,0);moved.session.dispose();
|
||||
});
|
||||
test('a node whose text changed after the preview pass is refused, not overwritten',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;let edited=false;
|
||||
const s=await allSession(createSendSession,{preview:id=>({ticket:`t-${id}`,target:{id},fields:[{role:'positive',before:id==='10'&&edited?'typed in ComfyUI':'old',after:'new'}]}),apply:id=>{edited=true;return{targetId:id,values:{positive:'new'}};}});
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['2','9']);
|
||||
assert.match(s.session.state.message,/^Sent to 2 of 3 nodes in Flow - ComfyUI\. #10 Node 10: Its prompt changed since the preview\./);s.session.dispose();
|
||||
});
|
||||
test('Preview with All shows every node in order, and Apply sends fresh tickets that match the review',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;const s=await allSession(createSendSession);
|
||||
s.session.selectTarget(ALL_NODES);await s.session.preview();
|
||||
assert.equal(s.session.state.phase,'ready');assert.equal(s.session.state.preview,null);
|
||||
assert.deepEqual(s.session.state.previews.map(item=>item.label),['#2 Node 2','#9 Node 9','#10 Node 10']);
|
||||
assert.match(s.session.state.message,/Sending to all 3 eligible nodes in Flow - ComfyUI\. Review the complete changes, then apply\./);
|
||||
const first=s.session.state.previews[0].preview.ticket;
|
||||
await s.session.apply();
|
||||
// A held review may have outlived the host's 30 s tickets, so even the first node takes a fresh one.
|
||||
assert.deepEqual(order(s.calls),['preview 2','preview 9','preview 10','preview 2','apply 2','preview 9','apply 9','preview 10','apply 10']);
|
||||
assert.notEqual(s.calls.find(call=>call[0]==='apply')[2],first);assert.equal(s.session.state.phase,'success');
|
||||
s.session.selectTarget('2');assert.deepEqual(s.session.state.previews,[]);s.session.dispose();
|
||||
});
|
||||
test('the dialog offers All for two or more nodes and reviews each node under its name',async()=>{
|
||||
const {createPromptSender,ALL_NODES}=await source;const {createScope}=await import('../../web/js/shared/toolkit/core.js');
|
||||
const dom=fakeDOM();const scope=createScope();
|
||||
const {client}=allHost({nodes:['1','2']});
|
||||
try {
|
||||
createPromptSender({scope,clientFactory:()=>client}).open({positive:'new'});await tick();
|
||||
const dialog=dom.body.children.at(-1);const target=dom.find(dialog,node=>node.id==='prompt-sender-target');
|
||||
assert.deepEqual(target.children.map(option=>[option.value,option.textContent]),[['','Choose a destination node'],['1','#1 Node 1'],['2','#2 Node 2'],[ALL_NODES,'All 2 eligible nodes']]);
|
||||
assert.equal(target.value,'');target.value=ALL_NODES;target.dispatchEvent(new Event('change'));
|
||||
dom.find(dialog,node=>node.textContent==='Preview changes').dispatchEvent(new Event('click'));for(let i=0;i<5;i++)await tick();
|
||||
const sections=[];(function walk(node){if(node.className==='prompt-sender-node')sections.push(node);node.children.forEach(walk);})(dialog);
|
||||
assert.deepEqual(sections.map(section=>section.children[0].textContent),['#1 Node 1','#2 Node 2']);
|
||||
dom.find(dialog,node=>node.textContent==='Apply to ComfyUI').dispatchEvent(new Event('click'));for(let i=0;i<10;i++)await tick();
|
||||
assert.ok(dom.find(dialog,node=>node.textContent==='Sent to 2 of 2 nodes in Flow - ComfyUI.'));
|
||||
} finally {scope.dispose();dom.restore();}
|
||||
});
|
||||
test('an expired review still sends every node, and a held review checks the discovered revision first',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;let reviewing=true;
|
||||
// The host refuses every ticket taken during the review, as after its 30 s expiry.
|
||||
const s=await allSession(createSendSession,{preview:(id,calls)=>({ticket:`t-${id}-${reviewing?'old':calls.length}`,target:{id},fields:[{role:'positive',before:'old',after:'new'}]}),apply:id=>({targetId:id,values:{positive:'new'}})});
|
||||
const realApply=s.client.apply;s.client.apply=async(host,ticket,options)=>{if(ticket.endsWith('-old'))throw Error('Preview expired or unavailable; preview again');return realApply(host,ticket,options);};
|
||||
s.session.selectTarget(ALL_NODES);await s.session.preview();reviewing=false;await s.session.apply();
|
||||
assert.equal(s.session.state.message,'Sent to 3 of 3 nodes in Flow - ComfyUI.');s.session.dispose();
|
||||
const moved=await allSession(createSendSession,{revision:'1:4',preview:(id,calls)=>({ticket:`t-${id}`,revision:calls.length>3?'1:5':'1:4',target:{id},fields:[{role:'positive',before:'old',after:'new'}]})});
|
||||
moved.session.selectTarget(ALL_NODES);await moved.session.preview();assert.equal(moved.session.state.phase,'ready');await moved.session.apply();
|
||||
assert.match(moved.session.state.message,/^Sent to 0 of 3 nodes in Flow - ComfyUI\. #2 Node 2: The workflow changed since this window was found/);
|
||||
assert.equal(moved.calls.filter(call=>call[0]==='apply').length,0);moved.session.dispose();
|
||||
});
|
||||
test('a changed prepend or append after the review refuses that node',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;let edited=false;
|
||||
const s=await allSession(createSendSession,{preview:id=>({ticket:`t-${id}`,target:{id},fields:[{role:'positive',before:'old',after:'new'}],composition:{prepend:id==='9'&&edited?'typed':'trigger',append:'',before:'trigger old',after:'trigger new'}}),apply:id=>{edited=true;return{targetId:id,values:{positive:'new'}};}});
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['2','10']);
|
||||
assert.match(s.session.state.message,/^Sent to 2 of 3 nodes in Flow - ComfyUI\. #9 Node 9: Its prompt changed since the preview\./);s.session.dispose();
|
||||
});
|
||||
|
||||
@@ -53,8 +53,12 @@ function observed(overrides = {}, capture = {}) {
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
// #431: the saved instructions; the fixture's entry is the default of both engines.
|
||||
const SAVED_ENTRY = { id: '0123456789ab', name: 'Saved', text: DEFAULT_PROMPT };
|
||||
const models = (overrides = {}) => ({
|
||||
success: true, models: { 'wd14-swinv2': { downloaded: true } }, default_prompt: DEFAULT_PROMPT,
|
||||
success: true, models: { 'wd14-swinv2': { downloaded: true } },
|
||||
instructions: [{ id: 'builtin', name: 'Built-in', text: 'Built-in text.' }, SAVED_ENTRY],
|
||||
default_instructions: { gguf: SAVED_ENTRY.id, hf: SAVED_ENTRY.id },
|
||||
wd14_general_threshold: 0.35, wd14_character_threshold: 0.85, suggestion_limit: 10, ...overrides,
|
||||
});
|
||||
const SANDSTONE = [
|
||||
@@ -244,10 +248,13 @@ test('dialog starts on the saved model unless a preset or the user chose one', a
|
||||
});
|
||||
|
||||
test('an unreadable or malformed models read falls back to the documented defaults', async () => {
|
||||
for (const route of [() => fail(500, null), () => response(200, models({ default_prompt: 4, suggestion_limit: 0, wd14_general_threshold: 2, wd14_character_threshold: 'x' }))]) {
|
||||
for (const route of [() => fail(500, null), () => response(200, models({ instructions: 4, suggestion_limit: 0, wd14_general_threshold: 2, wd14_character_threshold: 'x' }))]) {
|
||||
const { c } = await opened({ [`GET ${MODELS}`]: route });
|
||||
assert.equal(c.state.phase, 'ready');
|
||||
assert.deepEqual({ ...c.state.options }, { general_threshold: 0.35, character_threshold: 0.85, instructions: '', suggestion_limit: 10 });
|
||||
// #431: the client's Built-in copy (D204's read failure).
|
||||
const { BUILTIN_INSTRUCTIONS } = await load();
|
||||
assert.deepEqual({ ...c.state.options }, { general_threshold: 0.35, character_threshold: 0.85, instructions: BUILTIN_INSTRUCTIONS,
|
||||
suggestion_limit: 10 });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -936,3 +943,99 @@ test('#515: text typed before Retry after a failed first observation follows the
|
||||
assert.equal(c.state.manual, 'Minimal, warm_light, dunes');
|
||||
assert.equal(c.state.statusText, '');
|
||||
});
|
||||
|
||||
test('the chip hint says how many skipped tags were not suggested (#431)', async () => {
|
||||
const { skippedNote } = await load();
|
||||
assert.equal(skippedNote(0), '');
|
||||
assert.equal(skippedNote(undefined), '');
|
||||
assert.equal(skippedNote(1), ' 1 skipped tag was not suggested.');
|
||||
assert.equal(skippedNote(2), ' 2 skipped tags were not suggested.');
|
||||
for (const [skipped, hint] of [[2, 'Remove anything you do not want to add. 2 skipped tags were not suggested.'],
|
||||
[0, 'Remove anything you do not want to add.'], ['3', 'Remove anything you do not want to add.']]) {
|
||||
const { c } = await opened({ [`POST ${SUGGEST}`]: () => response(200, { ...suggested(), skipped }) });
|
||||
await c.generate();
|
||||
assert.equal(c.state.chipHint, hint, String(skipped));
|
||||
}
|
||||
});
|
||||
|
||||
// D204 (#431): the Saved instructions picker of both dialogs.
|
||||
const PHOTO_ENTRY = { id: '0123456789ac', name: 'Photo', text: 'Photo tags.' };
|
||||
const picker = (overrides = {}) => ({ [`GET ${MODELS}`]: () => response(200, models({ model_type: 'gguf',
|
||||
instructions: [{ id: 'builtin', name: 'Built-in', text: 'Built-in text.' }, SAVED_ENTRY, PHOTO_ENTRY],
|
||||
default_instructions: { gguf: SAVED_ENTRY.id, hf: PHOTO_ENTRY.id }, ...overrides })) });
|
||||
|
||||
test('the picker starts at the engine default; a pick replaces the text and an edit is one-off (D204)', async () => {
|
||||
const { c } = await opened(picker());
|
||||
assert.deepEqual([c.state.instructionId, c.state.options.instructions, c.state.instructionsEdited], [SAVED_ENTRY.id, DEFAULT_PROMPT, false]);
|
||||
const revision = c.state.optionsRevision;
|
||||
c.pickInstructions(PHOTO_ENTRY.id);
|
||||
assert.deepEqual([c.state.instructionId, c.state.options.instructions, c.state.instructionsEdited], [PHOTO_ENTRY.id, 'Photo tags.', false]);
|
||||
assert.equal(c.state.optionsRevision, revision + 1, 'a pick re-renders the textarea');
|
||||
c.setOption('instructions', 'Photo tags, please.');
|
||||
assert.equal(c.state.instructionsEdited, true);
|
||||
assert.equal(c.state.optionsRevision, revision + 1, 'typing never re-renders');
|
||||
// Choosing an entry again replaces the edit without asking; an unknown id changes nothing.
|
||||
c.pickInstructions(PHOTO_ENTRY.id);
|
||||
assert.deepEqual([c.state.options.instructions, c.state.instructionsEdited], ['Photo tags.', false]);
|
||||
c.pickInstructions('nope');
|
||||
assert.equal(c.state.instructionId, PHOTO_ENTRY.id);
|
||||
// Switching engines resets to the new engine's default and drops a one-off edit; WD14 is never "edited".
|
||||
c.setOption('instructions', 'edited');
|
||||
c.setModel('hf');
|
||||
assert.deepEqual([c.state.instructionId, c.state.options.instructions, c.state.instructionsEdited], [PHOTO_ENTRY.id, 'Photo tags.', false]);
|
||||
c.setModel('wd14-vit');
|
||||
assert.equal(c.state.instructionsEdited, false);
|
||||
c.setModel('gguf');
|
||||
assert.deepEqual([c.state.instructionId, c.state.options.instructions], [SAVED_ENTRY.id, DEFAULT_PROMPT]);
|
||||
c.dispose();
|
||||
});
|
||||
|
||||
test('a preset shows the entry with its text, else "Edited for this run" (D204)', async () => {
|
||||
const { c } = await opened(picker(), { preset: { model: 'hf', options: { instructions: DEFAULT_PROMPT } } });
|
||||
assert.deepEqual([c.state.instructionId, c.state.instructionsEdited], [SAVED_ENTRY.id, false]);
|
||||
const { c: other } = await opened(picker(), { preset: { model: 'hf', options: { instructions: 'Something else' } } });
|
||||
assert.deepEqual([other.state.instructionId, other.state.options.instructions, other.state.instructionsEdited],
|
||||
[PHOTO_ENTRY.id, 'Something else', true]);
|
||||
for (const controller of [c, other]) controller.dispose();
|
||||
});
|
||||
|
||||
test('a models read without instructions offers Built-in only with the client copy (D204)', async () => {
|
||||
const { BUILTIN_INSTRUCTIONS, readChoices } = await load();
|
||||
assert.equal(BUILTIN_INSTRUCTIONS.startsWith('Generate only comma-separated Danbooru tags'), true);
|
||||
for (const value of [null, {}, { instructions: [] }, { instructions: [{ id: 'x', name: 'X', text: 'y' }] }]) {
|
||||
assert.deepEqual(readChoices(value).entries.map(entry => entry.id), ['builtin'], JSON.stringify(value));
|
||||
}
|
||||
// A default naming no entry is Built-in.
|
||||
assert.deepEqual({ ...readChoices({ instructions: [{ id: 'builtin', name: 'Built-in', text: 't' }], default_instructions: { gguf: 'gone' } }).defaults },
|
||||
{ gguf: 'builtin', hf: 'builtin' });
|
||||
const { c } = await opened(picker({ instructions: 'broken' }));
|
||||
assert.deepEqual([c.state.instructionId, c.state.options.instructions], ['builtin', BUILTIN_INSTRUCTIONS]);
|
||||
c.dispose();
|
||||
});
|
||||
|
||||
test('the bulk options pick and reset like the Tag image dialog (D204)', async () => {
|
||||
const { createTaggingOptionsController } = await load();
|
||||
const options = createTaggingOptionsController({ api: harness(defaultRoutes(picker())).api });
|
||||
await options.open();
|
||||
assert.deepEqual([options.state.model, options.state.instructionId, options.state.options.instructions], ['gguf', SAVED_ENTRY.id, DEFAULT_PROMPT]);
|
||||
options.pickInstructions(PHOTO_ENTRY.id);
|
||||
assert.equal(options.request().options.instructions, 'Photo tags.');
|
||||
options.setOption('instructions', 'One-off.');
|
||||
assert.equal(options.state.instructionsEdited, true);
|
||||
assert.equal(options.request().options.instructions, 'One-off.');
|
||||
options.setModel('hf');
|
||||
assert.deepEqual([options.state.instructionId, options.state.instructionsEdited], [PHOTO_ENTRY.id, false]);
|
||||
options.pickInstructions('nope');
|
||||
options.dispose();
|
||||
options.pickInstructions(SAVED_ENTRY.id);
|
||||
assert.equal(options.state.instructionId, PHOTO_ENTRY.id);
|
||||
});
|
||||
|
||||
test('the client Built-in copy equals the server DEFAULT_TAG_TEMPLATE (#431)', async () => {
|
||||
const { BUILTIN_INSTRUCTIONS } = await load();
|
||||
const source = require('node:fs').readFileSync(require('node:path').join(__dirname, '../../py/tagging_defaults.py'), 'utf8');
|
||||
const block = source.slice(source.indexOf('DEFAULT_TAG_TEMPLATE = ('), source.indexOf('\n)\n', source.indexOf('DEFAULT_TAG_TEMPLATE = (')));
|
||||
const text = [...block.matchAll(/"((?:[^"\\]|\\.)*)"/g)].map(([, part]) => JSON.parse(`"${part}"`)).join('');
|
||||
assert.ok(text.length > 100);
|
||||
assert.equal(BUILTIN_INSTRUCTIONS, text);
|
||||
});
|
||||
|
||||
@@ -180,7 +180,9 @@ test('the bulk options start from the server defaults and wait for them before S
|
||||
assert.equal(options.state.canStart, false);
|
||||
assert.equal(options.state.canGenerate, false);
|
||||
const opening = options.open();
|
||||
answer({ wd14_general_threshold: 0.5, wd14_character_threshold: 0.9, default_prompt: 'Say it', suggestion_limit: 7,
|
||||
answer({ wd14_general_threshold: 0.5, wd14_character_threshold: 0.9, suggestion_limit: 7,
|
||||
instructions: [{ id: 'builtin', name: 'Built-in', text: 'Built-in text.' }, { id: '0123456789ab', name: 'Say', text: 'Say it' }],
|
||||
default_instructions: { gguf: '0123456789ab', hf: 'builtin' },
|
||||
models: { 'wd14-swinv2': { downloaded: true }, 'wd14-vit': { downloaded: false }, gguf: { downloaded: true } } });
|
||||
await opening;
|
||||
assert.equal(options.state.canStart, true);
|
||||
|
||||