Compare commits
20
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
34df48c031 | ||
|
|
a33b0f4648 | ||
|
|
c14e1af87f | ||
|
|
981639740b | ||
|
|
57ab85b002 | ||
|
|
38f5491da1 | ||
|
|
c0328e993a | ||
|
|
8a81114dfc | ||
|
|
7a2a04b752 | ||
|
|
cb94b12c1b | ||
|
|
3cb90b0c1e | ||
|
|
9a43b67008 | ||
|
|
c3c75c3369 | ||
|
|
93bf24f47d | ||
|
|
e29c23de8d | ||
|
|
b82e68a9d0 | ||
|
|
fb02066cda | ||
|
|
ab17818792 | ||
|
|
ef97d01dbd | ||
|
|
b7a43755ae |
@@ -1,9 +1,9 @@
|
||||
name: Windows and macOS
|
||||
name: Windows
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Non-blocking portability runs (#420): never on pull requests or in the merge queue, and no context
|
||||
# Non-blocking Windows runs (#420; macOS dropped from CI in #571): never on pull requests or in the merge queue, and no context
|
||||
# here is a required check. A red run is fixed at its cause before the workspace cutover (#334).
|
||||
on:
|
||||
push:
|
||||
@@ -25,10 +25,7 @@ env:
|
||||
# run counts (tests/vendor/ci-workflow.test.mjs recomputes it, #493).
|
||||
# Measured 2026-10-29 on main runs 40935337099 (dd247e3), 40938407252 (dd247e3), 40941240624 (dd247e3):
|
||||
# slowest leg: 7m24s (py3.12 windows) / 7m31s (py3.13 windows) / 7m32s (py3.14 windows) -> 15 min
|
||||
# (windows 4m56s-7m32s, macos 4m39s-6m14s)
|
||||
# native-trash (#499) has no green main runs yet: its probe, run 40949370943 (2026-10-29), took
|
||||
# about 1 minute per macOS leg, so ceil(1.5 x 2) + 3 gives it 6 minutes (step 4). Recalibrate
|
||||
# after three green runs.
|
||||
# (windows 4m56s-7m32s)
|
||||
# native-trash-linux (#543) has no green main runs yet: it installs the dev requirements and runs
|
||||
# one test module, about 1 minute by the same estimate, so ceil(1.5 x 2) + 3 gives it 6 minutes.
|
||||
# Recalibrate after three green runs.
|
||||
@@ -40,7 +37,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [windows-latest, macos-latest]
|
||||
os: [windows-latest]
|
||||
python-version: ["3.12", "3.13", "3.14"]
|
||||
|
||||
steps:
|
||||
@@ -60,7 +57,7 @@ jobs:
|
||||
|
||||
# The runner's TEMP is on C:, the OS disk, where a flush took about 6 ms
|
||||
# and a test database's create-to-delete about 111 ms; on the D: work
|
||||
# disk the same took 0.5 ms and 23 ms (macOS: 19 ms). Tests write their
|
||||
# disk the same took 0.5 ms and 23 ms. Tests write their
|
||||
# databases under TEMP, so it moves to a folder on D: (measured in #487,
|
||||
# run 40915866444). The runner's TEMP was an 8.3 short name
|
||||
# (C:\Users\RUNNER~1\...), so the new one is spelled by its short name too
|
||||
@@ -83,31 +80,6 @@ jobs:
|
||||
PM_TEST_SLOWEST: "40"
|
||||
run: python -m tests.run_suite -v
|
||||
|
||||
# The real Trash on both macOS CPUs (#499 design §10.2). It runs apart from the suite, so a red
|
||||
# suite never skips the native proof.
|
||||
native-trash:
|
||||
name: native trash on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 6
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [macos-latest, macos-15-intel]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Native trash proof
|
||||
timeout-minutes: 4
|
||||
env:
|
||||
PM_NATIVE_TRASH: macos
|
||||
run: python -m unittest tests.test_trash_macos_native tests.test_trash_macos -v
|
||||
|
||||
# The same tests as the fake mount table, on a real ext4 mount used as the output root (#543). A
|
||||
# runner can mount with sudo; the tests run as the normal user, so owner and mode checks are real.
|
||||
native-trash-linux:
|
||||
|
||||
@@ -28,7 +28,7 @@ env:
|
||||
# py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
# 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs.
|
||||
# Windows and macOS moved to portability.yml (#420), which has its own measured limit (#493);
|
||||
# Windows moved to portability.yml (#420; macOS left CI in #571), which has its own measured limit (#493);
|
||||
# this Linux-only matrix keeps 27 minutes until it is re-measured.
|
||||
# real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min
|
||||
# coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min
|
||||
@@ -39,7 +39,7 @@ env:
|
||||
# Pull requests run a fast subset (#420): Python 3.12 on ubuntu-latest, coverage, lint and frontend
|
||||
# modules. The merge queue (merge_group), pushes to main and manual runs add Python 3.13 and 3.14 on
|
||||
# Linux and the real ComfyUI job. "all tests" is the required gate that holds the queue to that set.
|
||||
# Windows and macOS run in portability.yml (push to main, nightly, on demand) and never gate a merge.
|
||||
# Windows runs in portability.yml (push to main, nightly, on demand) and never gate a merge.
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
|
||||
@@ -41,9 +41,13 @@ concurrency:
|
||||
# 7.3m-12.2m), because --shard splits by test count and the slow concept specs sort into shard 1. The page
|
||||
# workflows now run in three shards per browser with the limits unchanged; the per-shard times are
|
||||
# re-measured after three green main runs.
|
||||
# #563: coverage shards reached their 18-minute step limit three times on 2026-10-31 with no failing
|
||||
# test (PR #560 shard 2/3: 814 passed in 17.9m; queue run 41206495451 shard 1/3: 827 passed; PR #564
|
||||
# shard 2/3 twice), against recent green shards of 11m-17m. Coverage now runs in four shards with the
|
||||
# limits unchanged; the per-shard times are re-measured after three green main runs.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/4)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -59,7 +63,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3]
|
||||
shard: [1, 2, 3, 4]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -88,13 +92,13 @@ jobs:
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
# Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests
|
||||
# run on the lightest shard instead.
|
||||
- if: matrix.shard == 3
|
||||
- if: matrix.shard == 4
|
||||
run: npm run --prefix tools/vendor check
|
||||
- if: matrix.shard == 3
|
||||
- if: matrix.shard == 4
|
||||
run: npm run --prefix tools/vendor test
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/4
|
||||
timeout-minutes: 18
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/4
|
||||
- name: Upload raw coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
@@ -136,13 +140,13 @@ jobs:
|
||||
with:
|
||||
pattern: coverage-raw-*
|
||||
path: ${{ runner.temp }}/coverage-raw
|
||||
# coverage:merge only checks that raw files exist; merging two of three shards would report
|
||||
# coverage:merge only checks that raw files exist; merging three of four shards would report
|
||||
# partial coverage, so a missing or empty shard artifact fails here.
|
||||
- name: Verify every shard's raw coverage arrived
|
||||
shell: bash
|
||||
env:
|
||||
STAGED_RAW: ${{ runner.temp }}/coverage-raw
|
||||
EXPECTED_SHARDS: 3
|
||||
EXPECTED_SHARDS: 4
|
||||
run: |
|
||||
found=0
|
||||
for shard in $(seq 1 "$EXPECTED_SHARDS"); do
|
||||
|
||||
@@ -167,6 +167,8 @@ Searches your library by text, category, tags and minimum rating, then outputs t
|
||||
|
||||

|
||||
|
||||
To get the images of a Search List batch linked to their prompts, wire the Search List output into a Prompt Manager Text node, not straight into CLIP Text Encode. Wired straight in, the images stay unlinked: the search text is never saved as a prompt.
|
||||
|
||||
## Install
|
||||
|
||||
**Requirements:** ComfyUI and Python 3.12 or newer. `watchdog` and `Pillow` are the only dependencies of their own; the rest comes with ComfyUI.
|
||||
@@ -236,12 +238,12 @@ The documentation for those pages is in [README-legacy.md](README-legacy.md).
|
||||
|
||||
| Layer | What runs |
|
||||
| --- | --- |
|
||||
| Python | Over 200 test files on Linux, Windows and macOS with Python 3.12 to 3.14, with an 85% coverage gate |
|
||||
| Python | Over 200 test files on Linux and Windows with Python 3.12 to 3.14, with an 85% coverage gate |
|
||||
| Frontend | About 100 unit test files, with coverage floors per file |
|
||||
| Browser | Over 60 Playwright spec files in Chromium, Firefox, WebKit and touch emulation |
|
||||
| Real ComfyUI | A pinned CPU ComfyUI, started in Classic and Nodes 2.0, running real workflows |
|
||||
|
||||
Linux runs on every pull request and in the merge queue. Windows and macOS run on every push to main and nightly. To run the tests yourself, see [TESTING.md](TESTING.md).
|
||||
Linux runs on every pull request and in the merge queue. Windows runs on every push to main and nightly. macOS is not tested; it should work, but support is best-effort. To run the tests yourself, see [TESTING.md](TESTING.md).
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+169
-31
@@ -74,14 +74,14 @@ V8 files it writes under `coverage/js/raw`:
|
||||
|
||||
```sh
|
||||
PM_TEST_WORKERS=8 npm run --prefix tools/vendor test:coverage # whole run, one report
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 1/3 # node units, viewer suite, workflows 1/3
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 2/3 # workflows 2/3 (likewise 3/3)
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 1/4 # node units, viewer suite, workflows 1/4
|
||||
npm run --prefix tools/vendor test:coverage:shard -- 2/4 # workflows 2/4 (likewise 3/4, 4/4)
|
||||
npm run --prefix tools/vendor coverage:merge # report and baseline check of every raw file
|
||||
npm run --prefix tools/vendor test:workflows -- --project webkit --shard 2/2
|
||||
```
|
||||
|
||||
`coverage:merge` refuses an empty raw directory; it does not count shards, so run
|
||||
every shard first (CI's merge job checks all three arrived). Measured on a 32-CPU
|
||||
every shard first (CI's merge job checks all four arrived). Measured on a 32-CPU
|
||||
machine in the pinned image (2026-10-21): `test:coverage` takes about 5 minutes
|
||||
with 8 workers and 24 minutes with 1; all four projects, 8 workers, each test 3
|
||||
times (6132 tests), 24 minutes. A CI-sized run (`docker run --cpus=4`,
|
||||
@@ -189,14 +189,14 @@ check, so run it before pushing a new or deleted frontend file instead of
|
||||
waiting for the `frontend coverage` jobs.
|
||||
|
||||
The `viewerjs` check runs on every PR, including backend-only changes. It is an
|
||||
aggregate job. It requires the three `frontend coverage (shard N/3)` jobs and
|
||||
aggregate job. It requires the four `frontend coverage (shard N/4)` jobs and
|
||||
`frontend coverage (merge)` to succeed on every event. The four
|
||||
`page workflows (firefox|webkit, shard N/2)` jobs run only in the merge queue,
|
||||
on `main` and on `workflow_dispatch` (see "Pull requests and the merge queue").
|
||||
`viewerjs` requires them to be skipped on a pull request and to succeed
|
||||
everywhere else. Any other failed, cancelled or skipped job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
|
||||
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3`
|
||||
(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins
|
||||
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/4`
|
||||
(shard 1 also runs the node units and the viewer suite, shard 4 the vendor pins
|
||||
and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3
|
||||
days). The merge job downloads them, fails unless all three arrived with raw
|
||||
files and without clashing names, then runs `coverage:merge` (report plus
|
||||
@@ -261,7 +261,7 @@ the process exits non-zero.
|
||||
job fails anyway and the dump is what matters.
|
||||
- `PM_TEST_SLOWEST=N` (off by default) lists the N slowest tests after the
|
||||
summary, each timed from its `start:` line to the next one like the hang
|
||||
limit. The Windows and macOS jobs (`portability.yml`) set it to 40, so every
|
||||
limit. The Windows jobs (`portability.yml`) set it to 40, so every
|
||||
run shows where its time went (#487). The list prints only when the run
|
||||
ends: a job GitHub cancels or a watchdog exit prints none, so read the
|
||||
timestamps between `start:` lines instead.
|
||||
@@ -327,7 +327,7 @@ Python 3.13/3.14, real ComfyUI and Firefox/WebKit failures first show up in the
|
||||
merge queue, on the merge-group commit rather than the
|
||||
pushed one. Find that run with `gh run list --event merge_group --workflow test.yml`
|
||||
(or `viewerjs-check.yml`) and match its head branch
|
||||
(`gh-readonly-queue/main/pr-<N>-...`). Windows and macOS failures show up only
|
||||
(`gh-readonly-queue/main/pr-<N>-...`). Windows failures show up only
|
||||
after the merge, in `gh run list --workflow portability.yml`.
|
||||
|
||||
Start fixing as soon as a failed log shows the cause, but push the fix only after
|
||||
@@ -345,11 +345,16 @@ and the time left. The steps, in order:
|
||||
monitor's callbacks (Needs review recorder, preview queue, Images listing
|
||||
listener).
|
||||
2. `ImageMonitor.stop_monitoring` works against one 5 s deadline. Closing the
|
||||
image handler joins the image worker; watchdog's `observer.stop()` (not
|
||||
bounded here) and the observer join get what is left, which can be 0 s.
|
||||
3. `close_auto_thumbnails(final=True)` stops the preview worker (30 s) and
|
||||
kills ffmpeg.
|
||||
4. `PromptTracker.shutdown` stops the prompt cleanup thread (2 s).
|
||||
image handler joins the image worker; watchdog's `observer.stop()` and the
|
||||
observer join get what is left, which can be 0 s. `stop()` joins watchdog's
|
||||
emitters without a limit, so it runs on a daemon thread and is waited for
|
||||
only until the deadline; a later stop waits for that same call (#561).
|
||||
3. `close_auto_thumbnails(final=True)` stops the preview worker (30 s),
|
||||
kills ffmpeg and shuts the interactive image pool down.
|
||||
4. `shutdown_job_pool` (`utils/parallel.py`) cancels the long-lived job pool's
|
||||
queued items and joins its threads (5 s; part `workers`, only when a job
|
||||
used the pool, #523).
|
||||
5. `PromptTracker.shutdown` stops the prompt cleanup thread (2 s).
|
||||
|
||||
A failing step, or a second Ctrl+C, is logged with its traceback and does not
|
||||
skip the later steps. A clean run logs one INFO line to PromptManager's log:
|
||||
@@ -361,6 +366,15 @@ stored, with the parts not reached False, and the `atexit` callback is
|
||||
removed: the later steps are skipped, no summary line is logged, and a later
|
||||
call returns that result and runs nothing.
|
||||
|
||||
The observer is started from a short-lived daemon thread, so every thread it
|
||||
creates is a daemon (a thread inherits that flag from the thread that creates
|
||||
it). watchdog 7 on Windows starts a plain `threading.Thread` per watch for its
|
||||
`ReadDirectoryChangesW` reader; created from ComfyUI's main thread it would not
|
||||
be a daemon, and the interpreter waits for every such thread before it runs any
|
||||
`atexit` callback, so the teardown would never start (#561).
|
||||
`tests/test_package_exit.py` runs a child whose observer does both (a plain
|
||||
reader thread, a `stop()` that never returns) and requires its exit.
|
||||
|
||||
ComfyUI never runs aiohttp's `on_cleanup` hooks, because it never calls
|
||||
`runner.cleanup()`. So `atexit` is the live host's path. It runs on Ctrl+C
|
||||
(`main.py` catches `KeyboardInterrupt` and returns) and on a normal exit.
|
||||
@@ -445,12 +459,44 @@ A hot SQLite read that returns many rows aggregates them into one step
|
||||
(`json_group_array`) instead of stepping once per row, so it does not wait for
|
||||
the GIL once per row while ComfyUI generates (#354, `get_image_link_matches`).
|
||||
|
||||
A metadata read never decodes pixels (#523). Pillow's `PngImageFile.text` loads
|
||||
every pixel to reach text chunks after IDAT; the scan (`_extract_comfyui_metadata`)
|
||||
and the image monitor (`ComfyUIMetadataExtractor.extract_metadata`) read PNG text
|
||||
through `utils/png_text.py`, which runs Pillow's own chunk handlers after the
|
||||
first IDAT and seeks over pixel data. The legacy `.text` read is the floor:
|
||||
`tests/test_png_text.py` compares both readers on every image under
|
||||
`tests/fixtures`, a curated corpus (text after IDAT, zTXt, iTXt, split IDAT, bad
|
||||
checksums, truncations, oversized text, APNG) and a seeded fuzz, and asserts that
|
||||
neither reader decodes pixels by patching `ImageFile.load`. The only reads that
|
||||
differ are files whose pixel data is undecodable or cut short: legacy found
|
||||
nothing, the walk keeps the text before the pixels. Animated PNGs still use
|
||||
`.text`.
|
||||
|
||||
Image work runs on PromptManager's own threads (#523). glibc keeps each thread's
|
||||
malloc arena at its high-water mark, so image buffers spread over the default
|
||||
executor's threads stay resident. Interactive image work (view previews behind their
|
||||
gate, workspace card facts, both inspector routes) runs on the API's fixed
|
||||
`pm-image` pool (`IMAGE_POOL_SIZE` 3, `_run_image`, `_map_image`; card facts already
|
||||
cached answer without a trip; while the gate's 3 preview writes hold every thread, a
|
||||
card or inspector read waits for one of them, by design of the merged pool; after the
|
||||
package teardown the pool is not recreated and late calls use the default executor). Jobs (scan metadata and file facts, thumbnail and
|
||||
convert jobs, the legacy gallery generation, capture previews) run through
|
||||
`map_parallel` on one long-lived `pm-worker` pool of Worker threads, reused between
|
||||
jobs; a new Worker threads value replaces it (the teardown also joins a replaced pool
|
||||
still finishing), and a call made from a job thread runs inline. After a scan, thumbnail or convert job, or the legacy gallery generation,
|
||||
`release_free_memory` (`utils/native_memory.py`) calls glibc's `malloc_trim(0)` once;
|
||||
it does nothing on Windows, macOS and musl and never raises. No allocator setting
|
||||
changes. `tests/test_image_pools.py` asserts the thread of each read and write and the
|
||||
trim; `tests/test_utils_parallel.py` the pool's reuse, resize, nesting and shutdown.
|
||||
`tools/perf/memory_drive.py` (manual, Linux, never in CI) measures RSS, heap, threads
|
||||
and arenas under the #523 load against the disposable browser test server.
|
||||
|
||||
The `perf budget` job in `.github/workflows/test.yml` is the reference
|
||||
measurement: `ubuntu-latest`, one Python, no coverage, only the wall-clock tests.
|
||||
It runs on every push to `main` and on `workflow_dispatch`, never on pull
|
||||
requests, because shared-runner timing must not block a merge. Dispatch it on a
|
||||
branch to measure before merging.
|
||||
Shared Windows and macOS runners do not run them; the structural tests cover
|
||||
Shared Windows runners do not run them; the structural tests cover
|
||||
portability. Run it locally with
|
||||
`PROMPTMANAGER_PERF_BUDGET=1 python -m unittest tests.test_api_workspace_images.TestWorkspaceImages.test_filter_budget`
|
||||
and quote the printed p95 in the PR body.
|
||||
@@ -646,10 +692,10 @@ Before changing paths, SQLite, streams, restore or cleanup:
|
||||
ordinary busy timeout; cross-thread handoffs wait on events with generous
|
||||
ceilings, never a short wall clock (#242).
|
||||
- Run native Windows CI early for these changes. Pull requests and the merge
|
||||
queue run only Linux, so start Windows and macOS on the branch with
|
||||
queue run only Linux, so start Windows on the branch with
|
||||
`gh workflow run portability.yml --ref <branch>`.
|
||||
Include a Windows impact and evidence note in the PR. All six Windows/macOS
|
||||
combinations in that run, every Linux leg and every applicable
|
||||
Include a Windows impact and evidence note in the PR. All three Windows
|
||||
legs in that run, every Linux leg and every applicable
|
||||
exact-head check must pass before merge. Portable `ntpath`/`PureWindowsPath`
|
||||
controls do not establish native filesystem, locking or cleanup success.
|
||||
|
||||
@@ -664,11 +710,11 @@ what runs:
|
||||
| `py3.13 on ubuntu-latest`, `py3.14 on ubuntu-latest` | not created | run |
|
||||
| `real ComfyUI compatibility` | skipped | runs |
|
||||
| `coverage gate (85%)`, `black and flake8`, `frontend modules`, `bandit (high severity)` | run | run |
|
||||
| `frontend coverage (shard N/3)`, `frontend coverage (merge)` (vendor pins and tooling tests on shard 3) | run | run |
|
||||
| `frontend coverage (shard N/4)`, `frontend coverage (merge)` (vendor pins and tooling tests on shard 4) | run | run |
|
||||
| `page workflows (firefox\|webkit, shard N/2)` | skipped | run |
|
||||
| `all tests`, `viewerjs` (gates) | run | run |
|
||||
| `perf budget` | skipped | push and dispatch only (not in the queue) |
|
||||
| `py3.12-3.14 on windows-latest/macos-latest` (`portability.yml`) | not run | push to `main`, nightly, `workflow_dispatch` only; never required |
|
||||
| `py3.12-3.14 on windows-latest` (`portability.yml`) | not run | push to `main`, nightly, `workflow_dispatch` only; never required |
|
||||
|
||||
GitHub has one required set for pull requests and the queue. A job skipped by
|
||||
its `if:` reports Success. A dropped matrix leg never reports, so it would stay
|
||||
@@ -681,8 +727,9 @@ guarantee:
|
||||
So a merge group cannot turn green unless all three Linux Python legs, real
|
||||
ComfyUI and the four browser shards passed.
|
||||
|
||||
Windows and macOS (Python 3.12-3.14) are out of the blocking path for now
|
||||
(maintainer, 2026-10-28). They run in `portability.yml` on every push to `main`,
|
||||
Windows (Python 3.12-3.14) is out of the blocking path for now
|
||||
(maintainer, 2026-10-28). macOS is not tested in CI (maintainer, 2026-10-31, #571);
|
||||
its code stays and support is best-effort. They run in `portability.yml` on every push to `main`,
|
||||
nightly and on demand, and no context there is required. A red portability run
|
||||
is still a failure: fix it at its cause, not by rerunning, and have every one
|
||||
green before the workspace cutover (#334). `tests/vendor/ci-workflow.test.mjs` pins the
|
||||
@@ -1407,6 +1454,8 @@ is shorter than the row it reserves.
|
||||
Each Overview card is the count of the page it names: the shell's Library, Images and Tags sidebar counts, one source,
|
||||
so a card and its badge cannot differ. `/prompt_manager/stats` is for the legacy page and counts raw rows. A visit re-reads
|
||||
the three unless a read is in flight. An unread or failed card shows a dash; Retry re-reads only the failed counts.
|
||||
System › Overview's Images metric reads the same shell count (#555), so it, the Overview card and the Images badge show
|
||||
one number; its Statistics dialog labels the legacy row counts as "Database records".
|
||||
Tests: `tests/js/workspace-state.test.js`, `tests/browser/workspace.spec.mjs`, `tests/comfyui/workspace.spec.mjs`.
|
||||
|
||||
### Images page controls and recorded deviations (#249 PR C)
|
||||
@@ -2549,8 +2598,8 @@ files to the system trash. **No code path unlinks or overwrites a media file.**
|
||||
("Moving files to the Recycle Bin isn't available on Windows: files larger than the bin's
|
||||
capacity would be deleted permanently."). Library-only removal works everywhere. Tests use `tests/fake_trash.py`; the
|
||||
browser test server installs it, so no test touches a real trash. The native proof,
|
||||
`tests/test_trash_macos_native.py`, runs only with `PM_NATIVE_TRASH=macos` in
|
||||
portability.yml's `native-trash` job (`macos-latest` and `macos-15-intel`): the startup
|
||||
`tests/test_trash_macos_native.py`, runs only with `PM_NATIVE_TRASH=macos` on a Mac, by hand
|
||||
(CI dropped its `native-trash` macOS job in #571; it last passed on `macos-latest` and `macos-15-intel`): the startup
|
||||
volume, a name clash, fresh APFS and FAT32 disk images, an image used as an output root,
|
||||
a read-only image and an unwritable folder (design and probe:
|
||||
`docs/egm/499-macos-trash/`, local).
|
||||
@@ -3844,8 +3893,8 @@ workspace never asks; previews are a cache the server keeps:
|
||||
- `GET /images/serve/thumbnails/<rel>_thumb<ext>` for a missing preview is refused as
|
||||
text first (traversal, a backslash, a drive or stream colon, empty or dot segments, so
|
||||
`D:/x`, `\\server\share` and a leading `/` never reach a joined path), then admitted
|
||||
on the event loop (`_PreviewGate`): 3 previews are written at once on the API's own
|
||||
`pm-preview` pool (never the default executor), one flight runs per file (the history
|
||||
on the event loop (`_PreviewGate`): 3 previews are written at once on the API's
|
||||
interactive `pm-image` pool (never the default executor; #523), one flight runs per file (the history
|
||||
route shares the flight key and the admission), and at most 64 requests wait; one
|
||||
more gets its fallback at once without listing a folder. Inside the admission the
|
||||
preview maps back to its media file under the same root (exact names, then one folder
|
||||
@@ -3878,7 +3927,7 @@ workspace never asks; previews are a cache the server keeps:
|
||||
`exclusive` job runs (`_auto_thumbnail_blocker`), checked when a write starts (after
|
||||
its slot) and again just before its rename.
|
||||
- Shutdown is bounded and logged: the worker stops at a file boundary, ffmpeg is
|
||||
killed, an idle worker leaves (the next output starts a fresh one), the view pool
|
||||
killed, an idle worker leaves (the next output starts a fresh one), the image pool
|
||||
shuts down, and after the app's cleanup a late monitor event
|
||||
starts nothing. A package reload stops the workers the previous import left.
|
||||
- Cards and strips load previews only: Images in every layout and size, Library cards,
|
||||
@@ -4006,7 +4055,11 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
"Save preferences" is `aria-disabled` (opacity .55, `not-allowed`) while clean or
|
||||
saving; a setting the last save could not save gets a red outline. CSS under D92.
|
||||
- **D93** (Overview): real values. The metrics are the Library's eligible positives
|
||||
and active saved negatives and the indexed `generated_images` rows (Q2). "Database
|
||||
and active saved negatives and the Images count (#555, amending Q2's indexed
|
||||
`generated_images` rows): the shell's Images sidebar count, the Images page's own
|
||||
total, which also lists files no row links, as the concept's `images.length` is
|
||||
both. A visit re-reads it unless a read is in flight; "–" until read or after a
|
||||
failed read. `images.indexed` stays in the API and is not shown. "Database
|
||||
integrity" is "Not checked" until a check ran this session, then "N issues" (checks
|
||||
that found issues, even beside a check that could not run), "Passed" or "Incomplete".
|
||||
The review row is "N ambiguous image links" (the Needs review queue's pending count,
|
||||
@@ -4044,10 +4097,11 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
- **D96** (Statistics; amended by D153): "Usage statistics" without the sample note: the
|
||||
concept's "Runs by category" table without its Avg. rating column (Category / Runs)
|
||||
with the real runs of eligible parents per category ("Uncategorized"), then a second
|
||||
table "Library totals" (Total / Count: Prompts, Tags, Images from the legacy
|
||||
statistics; no Rated images). The API still reports `average_rating` and
|
||||
table "Database records" (Total / Count: Prompt records, Tags on prompts, Image
|
||||
records: the raw row counts of the legacy statistics, labelled as rows so they never
|
||||
read as the Library or Images page counts, #555; no Rated images). The API still reports `average_rating` and
|
||||
`rated_images`; the workspace does not show them.
|
||||
The Overview row says "Runs by category, with library totals." because
|
||||
The Overview row says "Runs by category, with database record counts." because
|
||||
that is what the dialog shows (the concept's own dialog shows only "Runs by
|
||||
category"); per-prompt run counts, prompt tags and dated runs are recorded, but no
|
||||
table of them exists yet. A failed read toasts "Statistics could not
|
||||
@@ -4350,6 +4404,23 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
choice. The host's checks still refuse a linked field or a changed workflow, and the
|
||||
sender also refuses if the destination's `revision` (node object plus graph version)
|
||||
moved since discovery; it is undefined on an older tab, which skips the comparison. A host error reads as a refusal, not "may have applied".
|
||||
- **D191** (#542, beyond the concept; maintainer 2026-10-31, Q542-1 A to 5 A): a window with two or more
|
||||
eligible nodes adds a last Destination node option "All N eligible nodes" (`ALL_NODES` in
|
||||
`prompt-sender.mjs`), never selected for you (D173). Eligible is D173's filter; the nodes it leaves out
|
||||
stay counted in the discovery sentence. One window only. Preview stays optional (D175): Preview changes
|
||||
shows one `section.prompt-sender-node` per node, headed "#<id> <title>", in send order (node id,
|
||||
numeric). Apply previews every node first and writes nothing if any refuses ("#<id> <title>: <reason>
|
||||
Nothing was sent."). Then it sends one node after another. Only the first node of a pass taken in that
|
||||
click keeps its ticket; every other node, and every node of a held Preview (host tickets expire after
|
||||
30 s), takes a fresh ticket that is refused unless its Before texts and composition equal the reviewed
|
||||
ones. That fresh preview checks the discovered revision until something has been posted, and not after,
|
||||
since our own write may move the graph version. Refusals are best effort: "Sent to 2 of 3 nodes in
|
||||
<window>. #12 <title>: <host's refusal>."; Apply stays available and a retry sends to every node again
|
||||
(rewriting the same text). A lost reply stops the run, names that node as unknown and lists the nodes
|
||||
not sent. Closing the dialog stops after the current node. The review sections are labelled by their
|
||||
`h4` node heading. Unit tests in `tests/js/prompt-sender.test.js`, a fake three-node host in
|
||||
`tests/browser/prompt-inspector.spec.mjs`, real Classic and Nodes 2.0 widgets in
|
||||
`tests/comfyui/prompt-sender.spec.mjs`.
|
||||
- **D106** (Integrations, S10, Q8): the send bridge always asks for the ComfyUI window
|
||||
and node, and the nodes always follow ComfyUI's own renderer, so "Default send
|
||||
destination" offers only "Ask each time" and "Host compatibility" only "Follow host
|
||||
@@ -5413,7 +5484,7 @@ list stay unscoped: any row's control proves them.
|
||||
| Worker threads | `admin.html:437–445` | S4 Sources | covered | `sources` · slider · "Worker threads" | worker range max is cpu_count (workspace-system-sources.spec.mjs) |
|
||||
| LoraManager badge, enable, path, trigger words, CivitAI key, clear key | `admin.html:448–490`; `admin.js:1052–1156` | S10 Integrations | covered | `integrations` · switch · "LoraManager integration" | LoRA rows show detected / enabled / cache-loaded separately (workspace-system-integrations.spec.mjs) |
|
||||
| LoRA data import with progress | `admin.html:492–498,517–541`; `admin.js:1157` | S10 Integrations | covered | `integrations` · button · "Review import" | Review import shows replacement, blocked items and network effects with the backup box; Start opens the job (D107) (workspace-system-integrations.spec.mjs) |
|
||||
| Statistics | `admin.html:159`; `admin.js:1627` | S2 Overview | covered | `overview` · button · "Explore" | statistics dialog lists runs by category and library totals (workspace-system.spec.mjs) |
|
||||
| Statistics | `admin.html:159`; `admin.js:1627` | S2 Overview | covered | `overview` · button · "Explore" | statistics dialog lists runs by category and database record counts (workspace-system.spec.mjs) |
|
||||
| Export components JSON `/export` | `admin.html:163`; `admin.js:1607`; `py/api/prompts.py:238` | S9 Backup & export | covered | `backups` · button · "Export all prompts JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Saved pairs export | `py/api/pairs.py:84` | S9 Backup & export | covered | `backups` · button · "Export saved pairs JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Library export | `py/api/library.py:223` | S9 Backup & export | covered | `backups` · button · "Export JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
@@ -5652,6 +5723,73 @@ are closed before `os.replace`; renames stay inside one folder; folder identity
|
||||
folder are refused. The browser test server closes its own log handlers before its temp
|
||||
root (which now holds the logs) is removed.
|
||||
|
||||
### Thumbnail location contract (#320)
|
||||
|
||||
Previews live in each source's `thumbnails/` (the default) or in one folder, `<folder>/<source key>/…`. Every reader,
|
||||
writer and remover asks `py/api/preview_store.py`; `tests/test_preview_store.py` fails on a `thumbnails` path join
|
||||
anywhere else. The setting is `thumbnails.location` in config.json, and only the Move thumbnails job writes it: POST
|
||||
`/settings` answers 409 `location-needs-move`.
|
||||
|
||||
- **Keys and marker.** A key is a slug plus a hash of the source's identity. The folder's
|
||||
`.promptmanager-thumbnails.json` stores each key with its identity and install. Only a single `<slug>-<hex>` name
|
||||
is ever joined onto the folder, and anything else in a marker is dropped. Every write and delete passes
|
||||
`preview_store.may_write`: below a key folder, with no link, junction or reparse point on the way
|
||||
(`tests/test_preview_store_folder.py`, including tampered markers).
|
||||
- **Unavailable.** A folder is unavailable when it is missing, is a link, holds no marker or holds another
|
||||
folder's marker. A probe checks it every 5 s, and a store call that fails or stalls marks it at once. While it is
|
||||
unavailable:
|
||||
- originals are served and nothing is written anywhere;
|
||||
- generate, convert, clear and prune answer 409 `location-unavailable`;
|
||||
- the folder is never made again except by Recreate folder.
|
||||
|
||||
The configured location is in force before any request. A bad value is `invalid`, never a fall-back to the
|
||||
sources.
|
||||
- **Move thumbnails.** The job is a `thumbnails` job with `scope.mode` `move` (`tests/test_thumbnail_move.py`). It
|
||||
runs in this order:
|
||||
1. The review answers `POST /images/thumbnails/plan`; not enough space answers 507.
|
||||
2. A token starts the job.
|
||||
3. Automatic previews are held.
|
||||
4. Copies run on the job pool in chunks of 2 or 4 files, each with its size checked and its mtime rounded up to
|
||||
2 s.
|
||||
5. One config commit makes the switch and records the old stores under `cleanup`. The job cannot be cancelled
|
||||
from here.
|
||||
6. The old copies are removed.
|
||||
|
||||
If it fails or is cancelled before the switch, nothing changes. A crash leaves
|
||||
`<data>/data/thumbnail-move.json`, which the next start undoes. Old copies still held open are removed by the
|
||||
next start's "Move thumbnails (finishing)" job. Leaving a folder also removes the previews of this install's
|
||||
sources that are no longer listed (Q320-8).
|
||||
- **Remove thumbnails of removed sources** (`scope.mode` `prune`). It is offered in Thumbnail maintenance only in
|
||||
folder mode, and it touches only this install's keys.
|
||||
- **Page tests.** `tests/browser/workspace-thumbnail-location.spec.mjs` covers the row, both moves, the in-place
|
||||
refusal, the confirmation, the notice with Recreate and with Use this folder. The legacy gallery runs once in
|
||||
folder mode (`workflows.spec.mjs`). The hooks are `tests/browser/thumbnail_location_fixture.py`.
|
||||
|
||||
Beyond the concept, against the approved static mock `docs/egm/post-3.4.1/320-mock`:
|
||||
|
||||
- D196: the Thumbnail location row in System › Sources & scans, after Thumbnail generation.
|
||||
- The status line is "Now: inside each source." or "Now: one folder, <path>.". It does not show the mock's
|
||||
"(3 sources, 8,512 thumbnails)" counts, because counting means walking every preview on each view. It says
|
||||
"Using the default folder next to your library." and names old copies that wait to be removed.
|
||||
- D197: the Move thumbnails review uses the shared review.
|
||||
- A grid value may be a lead sentence with a list (`[lead, items]`).
|
||||
- Sources no longer listed appear under "Removed after the move", where the mock has a "Not moved" row
|
||||
(maintainer answer to Q320-8: remove).
|
||||
- The Free space line is "<size> free", without naming the disk.
|
||||
- D198: Activity's view of `mode: move` and `mode: prune`.
|
||||
- Tiles for a move: Moved, Already there, Left in place, Errors. Tiles for a prune: Removed, Left in place,
|
||||
Errors.
|
||||
- The step line under the bar is `#job-phase`.
|
||||
- Cancel shows only until the switch.
|
||||
- D199: the unavailable notice on Overview and Sources & scans.
|
||||
- Recreate folder appears for a missing folder, after a confirmation.
|
||||
- Use this folder appears for another folder's marker.
|
||||
- Change location focuses the row; on Overview it opens Sources & scans.
|
||||
- D200: Thumbnail maintenance's "Remove thumbnails of removed sources" scope, with the plan line and a grid of
|
||||
key folders.
|
||||
- D201: Move library's review adds "Thumbnails: Stay in <path>. Move them in System › Sources & scans." in folder
|
||||
mode.
|
||||
|
||||
### Backup and restore contract (#249 System S8)
|
||||
|
||||
A restore is staged, then published (`database/restore_stage.py`). The upload is
|
||||
|
||||
+39
-16
@@ -92,10 +92,21 @@ try:
|
||||
routes = config.routes
|
||||
api = PromptManagerAPI()
|
||||
api.add_routes(routes)
|
||||
# Temp files of preview writes a crash interrupted (#317), off the
|
||||
# import path: the walk may cover a large thumbnails folder.
|
||||
|
||||
# The thumbnail location's startup check (#320), then the temp files of
|
||||
# preview writes a crash interrupted (#317), off the import path: the
|
||||
# walk may cover a large thumbnails folder.
|
||||
# #320: the configured thumbnail location is in force before any request
|
||||
# (no I/O); the thread below checks it.
|
||||
api.use_preview_location()
|
||||
|
||||
def _preview_startup():
|
||||
api.start_preview_location()
|
||||
api.recover_thumbnail_move() # an interrupted move, before the sweep
|
||||
api.sweep_thumbnail_temps()
|
||||
|
||||
threading.Thread(
|
||||
target=api.sweep_thumbnail_temps,
|
||||
target=_preview_startup,
|
||||
name="PromptManagerPreviewTempSweep",
|
||||
daemon=True,
|
||||
).start()
|
||||
@@ -108,6 +119,8 @@ try:
|
||||
# Build each output root's Images listing once the server's loop runs,
|
||||
# so the first open does not wait for the walk (#348).
|
||||
host_app.on_startup.append(api.warm_gallery_listings)
|
||||
# #320: old thumbnails a move left to remove, once it is checked.
|
||||
host_app.on_startup.append(api.finish_thumbnail_move)
|
||||
if host_app is not None and hasattr(host_app, "on_shutdown"):
|
||||
# Open Needs review change streams end before the server waits on them.
|
||||
host_app.on_shutdown.append(api.close_review_events)
|
||||
@@ -280,11 +293,11 @@ def _teardown_logger():
|
||||
|
||||
|
||||
def _stop_background_work():
|
||||
"""Stop the image monitor, the preview worker and the prompt cleanup
|
||||
thread against one TEARDOWN_SECONDS deadline, once.
|
||||
"""Stop the image monitor, the preview worker, the job worker threads
|
||||
and the prompt cleanup thread against one TEARDOWN_SECONDS deadline, once.
|
||||
|
||||
Returns ``{"monitor", "thumbnails", "tracker"}`` booleans (True: it
|
||||
stopped), leaving out parts that never started. Later calls return a
|
||||
Returns ``{"monitor", "thumbnails", "workers", "tracker"}`` booleans
|
||||
(True: it stopped), leaving out parts that never started. Later calls return a
|
||||
copy of the first result. A Ctrl+C inside a step is logged and the
|
||||
step is False; one between steps is re-raised after the result (parts
|
||||
not reached are False) is stored and the ``atexit`` entry removed.
|
||||
@@ -298,11 +311,14 @@ def _stop_background_work():
|
||||
api,
|
||||
_global_prompt_tracker,
|
||||
)
|
||||
from .utils import parallel
|
||||
|
||||
# A part whose global is set has started; False until its step says
|
||||
# otherwise, so an interrupt between steps leaves an honest result.
|
||||
started = (
|
||||
("monitor", monitor),
|
||||
("thumbnails", thumbnails),
|
||||
("workers", parallel.job_pool_started() or None),
|
||||
("tracker", tracker),
|
||||
)
|
||||
result = {name: False for name, part in started if part is not None}
|
||||
@@ -345,6 +361,16 @@ def _stop_background_work():
|
||||
result["thumbnails"] = _teardown_step(
|
||||
"the preview worker", close_previews, logger
|
||||
)
|
||||
if "workers" in result:
|
||||
|
||||
def stop_workers():
|
||||
from .utils.parallel import JOB_POOL_JOIN_SECONDS
|
||||
|
||||
return parallel.shutdown_job_pool(left(JOB_POOL_JOIN_SECONDS))
|
||||
|
||||
result["workers"] = _teardown_step(
|
||||
"the job worker threads", stop_workers, logger
|
||||
)
|
||||
if tracker is not None:
|
||||
|
||||
def stop_tracker():
|
||||
@@ -435,12 +461,9 @@ except Exception as e:
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
||||
|
||||
# Print startup message with loaded tools
|
||||
print()
|
||||
print(f"\033[94m[ComfyUI-PromptManager] Version:\033[0m {get_version()}")
|
||||
for node_key, display_name in NODE_DISPLAY_NAME_MAPPINGS.items():
|
||||
# ASCII only: a Windows console using a legacy code page cannot print emoji,
|
||||
# and a UnicodeEncodeError here would abort loading the whole node.
|
||||
print(f" \033[94mLoaded:\033[0m {display_name}")
|
||||
print(f"\033[94mTotal: {len(NODE_CLASS_MAPPINGS)} tools loaded\033[0m")
|
||||
print()
|
||||
# One startup line. ASCII only: a Windows console using a legacy code page
|
||||
# cannot print emoji, and a UnicodeEncodeError here would abort loading the node.
|
||||
print(
|
||||
f"\033[94m[ComfyUI-PromptManager] Version:\033[0m {get_version()}"
|
||||
f" - {len(NODE_CLASS_MAPPINGS)} tools loaded"
|
||||
)
|
||||
|
||||
@@ -78,6 +78,10 @@ MESSAGES = {
|
||||
),
|
||||
"same-folder": "The new folder is the node folder itself.",
|
||||
"target-invalid": "The new folder cannot hold the library.",
|
||||
"data-dir-served": (
|
||||
"The new folder is inside Prompt Manager's thumbnail folder. Nothing was "
|
||||
"moved."
|
||||
),
|
||||
"target-has-library": (
|
||||
"The new folder already holds a Prompt Manager library. Nothing was moved."
|
||||
),
|
||||
|
||||
+31
-14
@@ -11,6 +11,7 @@ Split into domain-specific mixins for maintainability:
|
||||
- JobRoutesMixin: the session job registry behind Activity (py/api/jobs.py)
|
||||
- ImageTagJobsMixin: bulk Add tags / Auto-tag as image-tags jobs
|
||||
- ThumbnailJobsMixin: thumbnail plan, thumbnail job and the gallery stream on it
|
||||
- ThumbnailLocationMixin: the thumbnail location check and payloads (#320)
|
||||
- ThumbnailConvertMixin: Convert outdated thumbnails (#317), a thumbnail job mode
|
||||
- AutoThumbnailMixin: previews made on first view, on capture and after a scan
|
||||
- ReviewRoutesMixin: the Needs review queue, item detail and decisions
|
||||
@@ -60,6 +61,11 @@ from .settings_routes import SettingsRoutesMixin
|
||||
from .job_routes import JobRoutesMixin
|
||||
from .image_tag_jobs import ImageTagJobsMixin
|
||||
from .thumbnail_jobs import ThumbnailJobsMixin
|
||||
from .thumbnail_location import ThumbnailLocationMixin
|
||||
from .thumbnail_move import ThumbnailMoveMixin
|
||||
from .thumbnail_move_plan import ThumbnailMovePlanMixin
|
||||
from .thumbnail_move_recovery import ThumbnailMoveRecoveryMixin
|
||||
from .thumbnail_prune import ThumbnailPruneMixin
|
||||
from .thumbnail_convert import ThumbnailConvertMixin
|
||||
from .thumbnail_auto import AutoThumbnailMixin
|
||||
from .system import SystemRoutesMixin
|
||||
@@ -85,6 +91,8 @@ try:
|
||||
from ...database.operations import PromptDatabase
|
||||
from ...utils.file_prompt import parse_comfyui_prompt
|
||||
from ...utils.logging_config import get_logger
|
||||
from ...utils.native_memory import release_free_memory
|
||||
from ...utils.png_text import png_text
|
||||
from ...utils.video_metadata import is_video_path, read_video_metadata
|
||||
except ImportError:
|
||||
import sys
|
||||
@@ -95,6 +103,8 @@ except ImportError:
|
||||
from database.operations import PromptDatabase
|
||||
from utils.file_prompt import parse_comfyui_prompt
|
||||
from utils.logging_config import get_logger
|
||||
from utils.native_memory import release_free_memory
|
||||
from utils.png_text import png_text
|
||||
from utils.video_metadata import is_video_path, read_video_metadata
|
||||
|
||||
|
||||
@@ -334,6 +344,11 @@ class PromptManagerAPI(
|
||||
JobRoutesMixin,
|
||||
ImageTagJobsMixin,
|
||||
ThumbnailJobsMixin,
|
||||
ThumbnailLocationMixin,
|
||||
ThumbnailMovePlanMixin,
|
||||
ThumbnailMoveMixin,
|
||||
ThumbnailMoveRecoveryMixin,
|
||||
ThumbnailPruneMixin,
|
||||
ThumbnailConvertMixin,
|
||||
AutoThumbnailMixin,
|
||||
SystemRoutesMixin,
|
||||
@@ -526,6 +541,14 @@ class PromptManagerAPI(
|
||||
return await loop.run_in_executor(None, call)
|
||||
return await loop.run_in_executor(None, func, *args)
|
||||
|
||||
async def _release_job_memory(self):
|
||||
"""After a scan or thumbnail job: glibc's free heap pages go back to
|
||||
the OS once (``malloc_trim(0)``; nothing elsewhere, #523)."""
|
||||
try:
|
||||
await self._run_in_executor(release_free_memory)
|
||||
except Exception: # best effort: the job's result already stands
|
||||
self.logger.debug("Releasing free memory failed", exc_info=True)
|
||||
|
||||
def _root_listing(self, root) -> RootListing:
|
||||
"""The shared listing record of one output root (created empty)."""
|
||||
return self._root_listings.setdefault(str(root), RootListing())
|
||||
@@ -704,6 +727,7 @@ class PromptManagerAPI(
|
||||
# ── Register domain-specific routes from mixins ───────────────
|
||||
|
||||
self._register_pair_routes(routes)
|
||||
self._register_thumbnail_move_routes(routes)
|
||||
self._register_library_routes(routes)
|
||||
self._register_library_delete_routes(routes)
|
||||
self._register_vocabulary_routes(routes)
|
||||
@@ -840,20 +864,11 @@ class PromptManagerAPI(
|
||||
f"{url_quote(rel_path.as_posix(), safe='/')}"
|
||||
)
|
||||
|
||||
# Any preview the file owns, the current format first (#317)
|
||||
from . import thumbnail_settings
|
||||
from .prompts import thumbnail_url_for
|
||||
# Any preview the file owns, the current format first
|
||||
# (#317); no store I/O here in folder mode (#320).
|
||||
from .images import listing_thumbnail_url
|
||||
|
||||
existing = thumbnail_settings.existing_thumbnail(
|
||||
output_path, rel_path
|
||||
)
|
||||
thumbnail_url = None
|
||||
if existing is not None:
|
||||
thumb_rel = (
|
||||
thumbnail_settings.thumbnail_stem_rel(rel_path)
|
||||
+ existing.suffix
|
||||
)
|
||||
thumbnail_url = thumbnail_url_for(thumb_rel, existing)
|
||||
thumbnail_url = listing_thumbnail_url(output_path, rel_path)
|
||||
if thumbnail_url:
|
||||
image["thumbnail_url"] = thumbnail_url
|
||||
break # Found matching root, stop searching
|
||||
@@ -1048,6 +1063,8 @@ class PromptManagerAPI(
|
||||
def _extract_comfyui_metadata(self, image_path):
|
||||
"""Embedded ComfyUI metadata of an image (PNG text chunks) or a video.
|
||||
|
||||
PNG text is read without decoding the pixels (``utils.png_text``, #523).
|
||||
|
||||
Videos go through ffprobe (see utils/video_metadata). Unreadable files
|
||||
are a warning, not an error: a scan over a large output tree meets
|
||||
truncated and foreign files, and it carries on past them.
|
||||
@@ -1056,7 +1073,7 @@ class PromptManagerAPI(
|
||||
return read_video_metadata(image_path)
|
||||
try:
|
||||
with Image.open(image_path) as img:
|
||||
return dict(getattr(img, "text", None) or {})
|
||||
return png_text(img)
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
f"Unreadable image {os.path.basename(str(image_path))}: {e}"
|
||||
|
||||
+5
-3
@@ -1454,6 +1454,7 @@ class AdminRoutesMixin:
|
||||
verb = "cancelled" if state == "cancelled" else "finished"
|
||||
job.log_line(f"Scan {verb} · {_plural(job.processed, 'file')} processed")
|
||||
job.finish(state)
|
||||
await self._release_job_memory()
|
||||
if state == "finished":
|
||||
# After the scan, so the guarded thumbnail job is admitted (#249).
|
||||
await self._hand_off_scan_thumbnails(job, output_dirs)
|
||||
@@ -1668,9 +1669,10 @@ class AdminRoutesMixin:
|
||||
def _extract_batch_metadata_sync(self, file_batch):
|
||||
"""Extract metadata from a batch of files in one executor call.
|
||||
|
||||
Files are read on ``PromptManagerConfig.WORKER_THREADS`` threads:
|
||||
Pillow decodes the whole image to reach trailing text chunks and
|
||||
releases the GIL while doing so.
|
||||
Files are read on the long-lived job pool of
|
||||
``PromptManagerConfig.WORKER_THREADS`` threads (#523).
|
||||
PNG text is read chunk by chunk without decoding pixels (#523); file
|
||||
reads and ffprobe release the GIL.
|
||||
"""
|
||||
from ..config import PromptManagerConfig
|
||||
|
||||
|
||||
+11
-13
@@ -19,8 +19,8 @@ from .jobs import (
|
||||
)
|
||||
|
||||
from .maintenance import disk_plan
|
||||
from .prompts import thumbnail_url_for
|
||||
from . import thumbnail_settings
|
||||
from .images import listing_thumbnail_url
|
||||
from . import preview_store
|
||||
from .. import model_gate
|
||||
|
||||
try:
|
||||
@@ -1371,23 +1371,21 @@ class AutotagRoutesMixin:
|
||||
files = await self._run_in_executor(
|
||||
self._without_removed, output_path, files
|
||||
)
|
||||
has_thumbnails = (output_path / "thumbnails").is_dir()
|
||||
location = preview_store.current()
|
||||
if location.mode == preview_store.FOLDER_MODE:
|
||||
has_thumbnails = location.available # no store I/O here (#320)
|
||||
else:
|
||||
store = preview_store.store_dir(output_path)
|
||||
has_thumbnails = store is not None and store.is_dir()
|
||||
|
||||
images = []
|
||||
for image_path in files:
|
||||
rel_path = image_path.relative_to(output_path)
|
||||
thumbnail_url = None
|
||||
if has_thumbnails:
|
||||
# Any preview the file owns, the current format first (#317)
|
||||
existing = thumbnail_settings.existing_thumbnail(
|
||||
output_path, rel_path
|
||||
)
|
||||
if existing is not None:
|
||||
thumb_rel = (
|
||||
thumbnail_settings.thumbnail_stem_rel(rel_path)
|
||||
+ existing.suffix
|
||||
)
|
||||
thumbnail_url = thumbnail_url_for(thumb_rel, existing)
|
||||
# Any preview the file owns, the current format first
|
||||
# (#317); in folder mode without store I/O (#320).
|
||||
thumbnail_url = listing_thumbnail_url(output_path, rel_path)
|
||||
images.append(
|
||||
{
|
||||
"filename": image_path.name,
|
||||
|
||||
+4
-1
@@ -843,7 +843,10 @@ class HistoryRoutesMixin:
|
||||
raise
|
||||
|
||||
async def execute():
|
||||
pending = asyncio.create_task(self._run_in_executor(project))
|
||||
# An inspection reads the file's header and text: the
|
||||
# interactive image pool (#523); media stays on the default.
|
||||
run = self._run_image if kind == "inspection" else self._run_in_executor
|
||||
pending = asyncio.create_task(run(project))
|
||||
try:
|
||||
return await asyncio.shield(pending)
|
||||
except asyncio.CancelledError:
|
||||
|
||||
+40
-15
@@ -15,10 +15,10 @@ from typing import BinaryIO
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import thumbnail_settings
|
||||
from . import preview_store, thumbnail_settings
|
||||
from .images import _canonical, _public_path, SERVABLE_EXTENSIONS, VIDEO_EXTENSIONS
|
||||
from .images import in_trash_below_roots
|
||||
from .images import existing_media_thumbnail, media_thumbnail_rel
|
||||
from .images import existing_media_thumbnail, media_preview_path
|
||||
|
||||
try:
|
||||
from ...utils.execution_outputs import (
|
||||
@@ -244,19 +244,36 @@ def resolve_current_candidate(
|
||||
return _unavailable("temporary_unavailable", error=True)
|
||||
|
||||
|
||||
def _target_authorized(target, namespace, scope, store_root=None) -> bool:
|
||||
"""A preview in the thumbnail folder (#320, folder mode) is authorized by
|
||||
the store of its source (``preview_store.owns``: inside the key folder,
|
||||
no link); everything else by the output namespace and the roots."""
|
||||
if (
|
||||
store_root is not None
|
||||
and preview_store.current().mode == preview_store.FOLDER_MODE
|
||||
):
|
||||
return preview_store.owns(store_root, target)
|
||||
return _authorized(target, namespace, scope)
|
||||
|
||||
|
||||
def _open_current_target(
|
||||
target: Path, namespace: Path, scope: CurrentOutputScope, preview: bool = False
|
||||
target: Path,
|
||||
namespace: Path,
|
||||
scope: CurrentOutputScope,
|
||||
preview: bool = False,
|
||||
store_root: Path | None = None,
|
||||
) -> BinaryIO:
|
||||
"""Open only the canonical target and verify authority against the actual handle.
|
||||
|
||||
A *preview* (a file under ``thumbnails/``) may also be JPEG XL (#317)."""
|
||||
A *preview* (a file under ``thumbnails/``) may also be JPEG XL (#317); in
|
||||
folder mode it is authorized by the store of *store_root* (#320)."""
|
||||
if _canonical(target) != target:
|
||||
raise CurrentLocationUnavailable("not_authorized", stale=True)
|
||||
if in_trash_below_roots(target, [namespace]):
|
||||
# Also at the open: the cover chooses its preview after the candidate
|
||||
# check, and a preview can be a link into a trash folder (#543).
|
||||
raise CurrentLocationUnavailable("missing")
|
||||
if not _authorized(target, namespace, scope):
|
||||
if not _target_authorized(target, namespace, scope, store_root):
|
||||
raise CurrentLocationUnavailable("not_authorized")
|
||||
suffix = target.suffix.lower()
|
||||
if suffix not in SERVABLE_EXTENSIONS and not (
|
||||
@@ -289,7 +306,7 @@ def _open_current_target(
|
||||
if (
|
||||
_canonical(target) != target
|
||||
or (observed.st_dev, observed.st_ino) != (current.st_dev, current.st_ino)
|
||||
or not _authorized(target, namespace, scope)
|
||||
or not _target_authorized(target, namespace, scope, store_root)
|
||||
):
|
||||
raise CurrentLocationUnavailable("not_authorized", stale=True)
|
||||
return os.fdopen(fd, "rb")
|
||||
@@ -326,12 +343,10 @@ class _OpenedCurrentMedia:
|
||||
|
||||
|
||||
def _preview_writable(root: Path, preview: Path) -> bool:
|
||||
"""A preview may be written: ``thumbnails`` and every link segment of the
|
||||
preview's folder resolve inside *root* (the serve route's rule, #249)."""
|
||||
folder = root / "thumbnails"
|
||||
return _canonical(folder).is_relative_to(_canonical(root)) and _canonical(
|
||||
preview
|
||||
).is_relative_to(_canonical(folder))
|
||||
"""A preview may be written: the store of *root* and every link segment
|
||||
of the preview's folder resolve inside it (the serve route's rule, #249;
|
||||
``preview_store.owns``)."""
|
||||
return preview_store.owns(root, preview)
|
||||
|
||||
|
||||
def _is_jxl(path) -> bool:
|
||||
@@ -381,8 +396,12 @@ def _cover_target(resolved, ensure_preview, placeholder, accepts_jxl):
|
||||
return _canonical(preview)
|
||||
if preview is not None:
|
||||
return _canonical(preview)
|
||||
preview = resolved.gallery_root / media_thumbnail_rel(relative)
|
||||
if ensure_preview is not None and _preview_writable(resolved.gallery_root, preview):
|
||||
preview = media_preview_path(resolved.gallery_root, relative, create=True)
|
||||
if (
|
||||
ensure_preview is not None
|
||||
and preview is not None
|
||||
and _preview_writable(resolved.gallery_root, preview)
|
||||
):
|
||||
ensure_preview(target, preview, is_video)
|
||||
if is_video and placeholder is not None:
|
||||
return _placeholder_media(placeholder)
|
||||
@@ -418,7 +437,13 @@ def _prepare_current_media(
|
||||
return chosen
|
||||
if chosen is not None:
|
||||
target, is_preview = chosen, True
|
||||
handle = _open_current_target(target, resolved.namespace, scope, is_preview)
|
||||
handle = _open_current_target(
|
||||
target,
|
||||
resolved.namespace,
|
||||
scope,
|
||||
is_preview,
|
||||
resolved.gallery_root if is_preview else None,
|
||||
)
|
||||
try:
|
||||
return _OpenedCurrentMedia(
|
||||
handle,
|
||||
|
||||
+204
-31
@@ -34,7 +34,8 @@ from .gallery_listing import (
|
||||
WalkOutcome,
|
||||
walk_root,
|
||||
)
|
||||
from . import thumbnail_settings
|
||||
from . import preview_store, thumbnail_settings
|
||||
from .thumbnail_location import location_refusal
|
||||
from .thumbnail_encode import (
|
||||
PREVIEW_HELD,
|
||||
EncoderClosed,
|
||||
@@ -1068,13 +1069,24 @@ def _iter_media_files(output_path, max_files):
|
||||
|
||||
|
||||
def media_thumbnail_rel(rel_path, settings=None):
|
||||
"""``thumbnails/<rel>_thumb<ext>`` (POSIX) for the media file *rel_path*:
|
||||
the name the preview gets when it is written now (#317), images and
|
||||
videos alike (``writer_settings``' extension)."""
|
||||
"""``thumbnails/<rel>_thumb<ext>`` (POSIX URL text) for the media file
|
||||
*rel_path*: the name the preview gets when it is written now (#317),
|
||||
images and videos alike (``writer_settings``' extension). Where it is on
|
||||
disk is ``media_preview_path``."""
|
||||
settings = settings or writer_settings()
|
||||
return thumbnail_settings.thumbnail_rel(rel_path, settings.extension)
|
||||
|
||||
|
||||
def media_preview_path(output_path, rel_path, settings=None, *, create=False):
|
||||
"""Where the preview of *rel_path* is written now (``preview_store``);
|
||||
*create* for a writer (a new source gets its key). None when nothing
|
||||
may be written (the thumbnail folder is unavailable)."""
|
||||
settings = settings or writer_settings()
|
||||
return preview_store.preview_path(
|
||||
output_path, rel_path, settings.extension, create=create
|
||||
)
|
||||
|
||||
|
||||
def existing_media_thumbnail(output_path, rel_path):
|
||||
"""The existing preview *rel_path* owns that a listing or cover shows
|
||||
(the format written now first, then older ones), or None. The Images
|
||||
@@ -1084,6 +1096,27 @@ def existing_media_thumbnail(output_path, rel_path):
|
||||
)
|
||||
|
||||
|
||||
def listing_thumbnail_url(output_path, rel_path):
|
||||
"""The thumbnail URL of an older listing (``_enrich_images``, the
|
||||
auto-tag scan), or None. These run on the event loop: in folder mode
|
||||
(#320) the store is never touched there, so the URL names the preview
|
||||
as it is written now and the serve route finds or makes it off the loop
|
||||
(None while the thumbnail folder is unavailable)."""
|
||||
from .prompts import thumbnail_url_for
|
||||
|
||||
location = preview_store.current()
|
||||
if location.mode == preview_store.FOLDER_MODE:
|
||||
if not location.available:
|
||||
return None
|
||||
quoted = urllib.parse.quote(media_thumbnail_rel(rel_path), safe="/")
|
||||
return f"/prompt_manager/images/serve/{quoted}"
|
||||
existing = thumbnail_settings.existing_thumbnail(output_path, rel_path)
|
||||
if existing is None:
|
||||
return None
|
||||
thumb_rel = thumbnail_settings.thumbnail_stem_rel(rel_path) + existing.suffix
|
||||
return thumbnail_url_for(thumb_rel, existing)
|
||||
|
||||
|
||||
def media_thumbnail(output_path, rel_path, root_index, *, root_scope=None):
|
||||
"""``(thumbnail_url, ready)`` for *rel_path* under *output_path* (a few
|
||||
blocking stats). The Images listing, the Needs review queue cards and the
|
||||
@@ -2798,13 +2831,56 @@ class ImageRoutesMixin:
|
||||
output_dirs[0], filepath, int(root_param), len(outputs)
|
||||
)
|
||||
|
||||
# Preview text is mapped by the resolver (#320); any other text,
|
||||
# ``Thumbnails/...`` media on a case-sensitive disk included,
|
||||
# keeps the plain join.
|
||||
preview_parts = (
|
||||
tuple(filepath.split("/"))
|
||||
if preview_store.is_preview_text(filepath)
|
||||
else None
|
||||
)
|
||||
location = preview_store.current()
|
||||
if location.mode == preview_store.FOLDER_MODE:
|
||||
if preview_parts is None and await self._run_image(
|
||||
self._in_source_thumbnails, output_dirs, filepath
|
||||
):
|
||||
# Previews live in the thumbnail folder: anything else
|
||||
# under a source's thumbnails/ is a leftover (#320), in
|
||||
# any case on a case-insensitive disk.
|
||||
return _forbidden()
|
||||
if preview_parts is not None:
|
||||
# Store I/O never runs on the event loop (a share may
|
||||
# stall): the lookup runs on the image pool.
|
||||
found = await self._run_image(
|
||||
self._find_preview, output_dirs, preview_parts, location
|
||||
)
|
||||
if found == "forbidden":
|
||||
return _forbidden()
|
||||
if found is not None:
|
||||
return self._file_response(found)
|
||||
for output_path in output_dirs:
|
||||
image_path = output_path / filepath
|
||||
if (
|
||||
preview_parts is not None
|
||||
and location.mode == preview_store.FOLDER_MODE
|
||||
):
|
||||
break # looked up above
|
||||
if preview_parts is None:
|
||||
image_path = output_path / filepath
|
||||
else:
|
||||
image_path = preview_store.request_path(output_path, preview_parts)
|
||||
if image_path is None:
|
||||
continue
|
||||
if not image_path.exists():
|
||||
continue
|
||||
# A symlink that points outside the root is a hard deny, not
|
||||
# a fall-through to the next root.
|
||||
if not _is_within(image_path, output_path):
|
||||
# A symlink that points outside the root (a preview: outside
|
||||
# the folder holding its store) is a hard deny, not a
|
||||
# fall-through to the next root.
|
||||
container = (
|
||||
output_path
|
||||
if preview_parts is None
|
||||
else preview_store.store_container(output_path)
|
||||
)
|
||||
if not _is_within(image_path, container):
|
||||
return _forbidden()
|
||||
if in_trash_below_roots(image_path, output_dirs):
|
||||
return _file_not_found()
|
||||
@@ -2830,6 +2906,48 @@ class ImageRoutesMixin:
|
||||
{"success": False, "error": _safe_error(e)}, status=500
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _in_source_thumbnails(output_dirs, filepath):
|
||||
"""Whether *filepath* lies in a source's ``thumbnails`` folder: its
|
||||
first part is ``thumbnails`` exactly, or another case of it that is
|
||||
the same folder (a case-insensitive disk). ``Thumbnails/`` media on
|
||||
a case-sensitive disk is another folder and stays served."""
|
||||
first = filepath.split("/", 1)[0]
|
||||
if first == "thumbnails":
|
||||
return True
|
||||
if first.casefold() != "thumbnails":
|
||||
return False
|
||||
for output_path in output_dirs:
|
||||
spelled = output_path / first
|
||||
cache = preview_store.source_folder(output_path)
|
||||
try:
|
||||
if spelled.is_dir() and cache.is_dir() and spelled.samefile(cache):
|
||||
return True
|
||||
except OSError:
|
||||
continue
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _find_preview(output_dirs, parts, location):
|
||||
"""The preview file *parts* names in folder mode, ``"forbidden"``
|
||||
(it resolves out of its store or through a link) or None (blocking;
|
||||
the first root whose store holds it, in ``_select_root`` order)."""
|
||||
for output_path in output_dirs:
|
||||
image_path = preview_store.request_path(output_path, parts, location)
|
||||
if image_path is None:
|
||||
continue
|
||||
try:
|
||||
with preview_store.store_io():
|
||||
if not os.path.lexists(image_path):
|
||||
continue
|
||||
if not preview_store.owns(output_path, image_path, location):
|
||||
return "forbidden"
|
||||
if image_path.is_file():
|
||||
return image_path
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
def _is_recorded_media(self, path) -> bool:
|
||||
"""Whether the database records ``path`` as a linked image.
|
||||
|
||||
@@ -2956,6 +3074,9 @@ class ImageRoutesMixin:
|
||||
data, error = await _json_body_or_error(request)
|
||||
if error is not None:
|
||||
return error
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
return await self._run_held(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
THUMBNAIL_GENERATION_TITLE,
|
||||
@@ -2975,6 +3096,7 @@ class ImageRoutesMixin:
|
||||
result = await self._run_in_executor(
|
||||
self._generate_gallery_thumbnails_sync, output_dirs
|
||||
)
|
||||
await self._release_job_memory()
|
||||
return web.json_response(result)
|
||||
|
||||
except Exception as e:
|
||||
@@ -2987,6 +3109,7 @@ class ImageRoutesMixin:
|
||||
"""Blocking thumbnail generation loop (run in executor)."""
|
||||
settings = writer_settings(settings)
|
||||
start = _time.monotonic()
|
||||
thumbnails_dir = preview_store.given_store(output_path, thumbnails_dir)
|
||||
plan = self._plan_thumbnails(output_path, thumbnails_dir, settings=settings)
|
||||
payload = self._execute_thumbnail_plan(plan, settings, start)
|
||||
payload["thumbnails_path"] = _public_path(thumbnails_dir)
|
||||
@@ -2999,7 +3122,8 @@ class ImageRoutesMixin:
|
||||
start = _time.monotonic()
|
||||
plan = self._plan_gallery_thumbnails(output_dirs, settings=settings)
|
||||
payload = self._execute_thumbnail_plan(plan, settings, start)
|
||||
payload["thumbnails_path"] = _public_path(output_dirs[0] / "thumbnails")
|
||||
store = preview_store.store_dir(output_dirs[0])
|
||||
payload["thumbnails_path"] = None if store is None else _public_path(store)
|
||||
return payload
|
||||
|
||||
def _execute_thumbnail_plan(self, plan: dict, settings, start: float) -> dict:
|
||||
@@ -3042,9 +3166,12 @@ class ImageRoutesMixin:
|
||||
if stop is not None and stop():
|
||||
break
|
||||
root = Path(root)
|
||||
store = preview_store.ensure_store(root)
|
||||
if store is None:
|
||||
continue # the thumbnail folder is unavailable
|
||||
plan = self._plan_thumbnails(
|
||||
root,
|
||||
root / "thumbnails",
|
||||
store,
|
||||
mode=mode,
|
||||
stop=stop,
|
||||
settings=settings,
|
||||
@@ -3083,6 +3210,8 @@ class ImageRoutesMixin:
|
||||
"format": writer_settings().format,
|
||||
# Files whose automatic preview failed this session (#249).
|
||||
"failed": self._auto_thumbnail_failures(),
|
||||
# #320: where previews live (no path here, D6A).
|
||||
"location": self._thumbnail_location_payload(with_path=False),
|
||||
"collection_id": hashlib.sha256(collection.encode()).hexdigest()[
|
||||
:16
|
||||
],
|
||||
@@ -3128,9 +3257,11 @@ class ImageRoutesMixin:
|
||||
"""
|
||||
settings = writer_settings(settings)
|
||||
output_path = Path(output_path)
|
||||
thumbnails_dir = Path(thumbnails_dir)
|
||||
thumbnails_dir = preview_store.given_store(output_path, thumbnails_dir)
|
||||
plan = {key: 0 for key in _PLAN_COUNTS}
|
||||
plan.update(targets=[], fresh_files=[])
|
||||
if thumbnails_dir is None:
|
||||
return plan # the thumbnail folder is unavailable
|
||||
folders = {}
|
||||
removed = self._removal_check_under(output_path)
|
||||
for src in _iter_media_files(output_path, MAX_GALLERY_FILES):
|
||||
@@ -3220,6 +3351,9 @@ class ImageRoutesMixin:
|
||||
"dir": src.parent.name,
|
||||
"type": "video" if is_video else "image",
|
||||
}
|
||||
# Taken when the write starts: a thumbnail location switched since
|
||||
# then drops the write at its rename (#320).
|
||||
guard = preview_store.write_guard(guard)
|
||||
dst = Path(dst)
|
||||
if dst.suffix != settings.extension:
|
||||
dst = dst.with_suffix(settings.extension)
|
||||
@@ -3230,7 +3364,7 @@ class ImageRoutesMixin:
|
||||
return {**result, "action": "error", "error": "File no longer exists"}
|
||||
if not rebuild and _thumbnail_is_fresh(src, dst):
|
||||
return {**result, "action": "skipped", "path": str(dst)}
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
preview_store.make_parent(dst)
|
||||
if is_video:
|
||||
made = self._generate_video_thumbnail(
|
||||
src,
|
||||
@@ -3435,8 +3569,12 @@ class ImageRoutesMixin:
|
||||
|
||||
Invalid thumbnail directories return 400; unavailable gallery roots
|
||||
return 404, and unexpected filesystem failures return 500. Holds the
|
||||
registry as guarded work, like generation.
|
||||
registry as guarded work, like generation. Refused (409) while the
|
||||
thumbnail folder is unavailable (#320).
|
||||
"""
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
return await self._run_held(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
THUMBNAIL_CLEAR_TITLE,
|
||||
@@ -3500,8 +3638,12 @@ class ImageRoutesMixin:
|
||||
self.logger.debug(f"No roots to sweep for preview temp files: {exc}")
|
||||
return 0
|
||||
for output_path in output_dirs:
|
||||
folder = Path(output_path) / "thumbnails"
|
||||
if folder.is_symlink() or not folder.is_dir():
|
||||
folder = preview_store.store_dir(output_path)
|
||||
if (
|
||||
folder is None
|
||||
or preview_store.store_is_link(folder)
|
||||
or not folder.is_dir()
|
||||
):
|
||||
continue
|
||||
removed += sweep_stray_temps(folder)[0]
|
||||
if removed:
|
||||
@@ -3519,13 +3661,16 @@ class ImageRoutesMixin:
|
||||
directories = []
|
||||
for output_path in output_dirs:
|
||||
output_path = Path(output_path)
|
||||
thumbnails_dir = output_path / "thumbnails"
|
||||
thumbnails_dir = preview_store.store_dir(output_path)
|
||||
if thumbnails_dir is None:
|
||||
continue
|
||||
if not thumbnails_dir.exists() and not thumbnails_dir.is_symlink():
|
||||
continue
|
||||
container = preview_store.store_container(output_path)
|
||||
if (
|
||||
thumbnails_dir.is_symlink()
|
||||
or not _is_within(thumbnails_dir, output_path)
|
||||
or _canonical(thumbnails_dir) == _canonical(output_path)
|
||||
preview_store.store_is_link(thumbnails_dir)
|
||||
or not _is_within(thumbnails_dir, container)
|
||||
or _canonical(thumbnails_dir) == _canonical(container)
|
||||
):
|
||||
raise ValueError(
|
||||
"Safety check failed: invalid thumbnails directory path"
|
||||
@@ -3540,7 +3685,15 @@ class ImageRoutesMixin:
|
||||
temps, temp_bytes = sweep_stray_temps(thumbnails_dir, min_age=0)
|
||||
cleared_count += temps
|
||||
cleared_size += temp_bytes
|
||||
for root, _dirs, files in os.walk(thumbnails_dir):
|
||||
folder_mode = preview_store.current().mode == preview_store.FOLDER_MODE
|
||||
for root, dirs, files in os.walk(thumbnails_dir):
|
||||
if folder_mode:
|
||||
# Never into a junction or reparse point (#320).
|
||||
dirs[:] = [
|
||||
d
|
||||
for d in dirs
|
||||
if not preview_store.store_is_link(Path(root) / d)
|
||||
]
|
||||
for name in files:
|
||||
path = Path(root) / name
|
||||
if (
|
||||
@@ -3548,6 +3701,7 @@ class ImageRoutesMixin:
|
||||
or path.suffix.lower()
|
||||
not in thumbnail_settings.THUMBNAIL_EXTENSIONS
|
||||
or path.is_symlink()
|
||||
or not preview_store.may_write(path)
|
||||
):
|
||||
continue
|
||||
try:
|
||||
@@ -3997,7 +4151,8 @@ class ImageRoutesMixin:
|
||||
**observed,
|
||||
)
|
||||
|
||||
body = await self._run_in_executor(inspect)
|
||||
# Header and text reads: the interactive image pool (#523).
|
||||
body = await self._run_image(inspect)
|
||||
if body is None:
|
||||
return web.json_response(
|
||||
{
|
||||
@@ -4210,6 +4365,22 @@ class ImageRoutesMixin:
|
||||
# (which Windows refuses on open files) is not held up by it.
|
||||
return _CardFacts(width, height, *self._card_parse(path, records))
|
||||
|
||||
def _cached_card_facts(self, entry: tuple) -> _CardFacts | None:
|
||||
path, mtime, root, index = entry[:4]
|
||||
return self._workspace_card_cache.get((index, str(root)), {}).get((path, mtime))
|
||||
|
||||
def _card_facts_of(self, entries: list, *, parse: bool) -> list[_CardFacts]:
|
||||
"""Card facts of *entries*, in order; files not cached yet are read on
|
||||
the interactive image pool (#523), cached ones answer here."""
|
||||
found = [self._cached_card_facts(entry) for entry in entries]
|
||||
missing = [i for i, facts in enumerate(found) if facts is None]
|
||||
read = self._map_image(
|
||||
lambda i: self._card_facts(entries[i], parse=parse), missing
|
||||
)
|
||||
for i, facts in zip(missing, read):
|
||||
found[i] = facts
|
||||
return found
|
||||
|
||||
def _card_facts(self, entry: tuple, *, parse: bool) -> _CardFacts:
|
||||
"""One listed image's card facts, read once per ``(path, mtime)``.
|
||||
|
||||
@@ -4218,11 +4389,11 @@ class ImageRoutesMixin:
|
||||
and drops the rest, and a roots change drops them all. Only files of
|
||||
the current scan are stored, so the cache is bounded by the listing.
|
||||
"""
|
||||
path, mtime, root, index = entry[:4]
|
||||
root_key, key = (index, str(root)), (path, mtime)
|
||||
cached = self._workspace_card_cache.get(root_key, {}).get(key)
|
||||
cached = self._cached_card_facts(entry)
|
||||
if cached is not None:
|
||||
return cached
|
||||
path, mtime, root, index = entry[:4]
|
||||
root_key, key = (index, str(root)), (path, mtime)
|
||||
facts = self._read_card_facts(path, parse=parse)
|
||||
if facts is None:
|
||||
# Not stored: a locked or half-written file may read next time.
|
||||
@@ -4248,15 +4419,17 @@ class ImageRoutesMixin:
|
||||
if linked
|
||||
else {}
|
||||
)
|
||||
# Videos: the card reads no video metadata. Library previews show
|
||||
# no card facts, so they skip the parse (``parse=linked``).
|
||||
stills = [image for image in images if not image["is_video"]]
|
||||
read = self._card_facts_of(
|
||||
[entries[(image["root_index"], image["path"])] for image in stills],
|
||||
parse=linked,
|
||||
)
|
||||
facts_of = {id(image): facts for image, facts in zip(stills, read)}
|
||||
for image in images:
|
||||
entry = entries[(image["root_index"], image["path"])]
|
||||
# Videos: the card reads no video metadata. Library previews show
|
||||
# no card facts, so they skip the parse (``parse=linked``).
|
||||
facts = (
|
||||
_NO_CARD_FACTS
|
||||
if image["is_video"]
|
||||
else self._card_facts(entry, parse=linked)
|
||||
)
|
||||
facts = facts_of.get(id(image), _NO_CARD_FACTS)
|
||||
image["width"], image["height"] = facts.width, facts.height
|
||||
if not linked:
|
||||
continue
|
||||
|
||||
@@ -50,6 +50,7 @@ _REFUSAL_STATUS = {
|
||||
"not-movable": 400,
|
||||
"same-folder": 400,
|
||||
"target-invalid": 400,
|
||||
"data-dir-served": 400,
|
||||
"target-has-library": 409,
|
||||
"target-conflict": 409,
|
||||
"insufficient-space": 507,
|
||||
@@ -74,6 +75,24 @@ def _failure(message, status, code):
|
||||
)
|
||||
|
||||
|
||||
def _thumbnail_folder():
|
||||
"""The thumbnail folder in use, or None (previews inside each source)."""
|
||||
from . import preview_store
|
||||
|
||||
setting = preview_store.setting()
|
||||
return setting.path if setting.mode == preview_store.FOLDER_MODE else None
|
||||
|
||||
|
||||
def _inside_thumbnail_folder(path):
|
||||
"""Whether *path* lies in the thumbnail folder in use (#320)."""
|
||||
from . import preview_store
|
||||
|
||||
setting = preview_store.setting()
|
||||
if setting.mode != preview_store.FOLDER_MODE or not setting.path:
|
||||
return False
|
||||
return ll._inside_any(path, [setting.path])
|
||||
|
||||
|
||||
def _plural(count, one, many=None):
|
||||
return f"{count:,} {one if count == 1 else (many or one + 's')}"
|
||||
|
||||
@@ -225,6 +244,8 @@ class LibraryLocationRoutesMixin:
|
||||
),
|
||||
"problem": location.problem or self._node_folder_risk(location),
|
||||
"size_bytes": size,
|
||||
# #320 D201: thumbnails kept in one folder stay there (D8A).
|
||||
"thumbnails_folder": _thumbnail_folder(),
|
||||
}
|
||||
|
||||
# The node folder (tests point it at a temp tree).
|
||||
@@ -272,6 +293,8 @@ class LibraryLocationRoutesMixin:
|
||||
def _plan_library_move(self):
|
||||
"""The checked plan (blocking); raises MoveFailed."""
|
||||
plan = lm.plan_move(self._live_location())
|
||||
if _inside_thumbnail_folder(plan.target_dir):
|
||||
raise lm.MoveFailed("data-dir-served")
|
||||
if lm.space_problem(plan) is not None:
|
||||
raise lm.MoveFailed("insufficient-space")
|
||||
return plan
|
||||
|
||||
@@ -0,0 +1,648 @@
|
||||
"""The thumbnail location setting (#320): parsing, validation, source keys
|
||||
and the thumbnail folder's marker.
|
||||
|
||||
``preview_store`` holds the runtime state and maps paths; this module holds
|
||||
the rules it applies. Nothing here keeps state.
|
||||
|
||||
Schema (config.json ``thumbnails.location``)::
|
||||
|
||||
{"mode": "folder", "path": "D:\\\\Thumbs", "id": "<uuid4>",
|
||||
"install": "<uuid4>", "cleanup": []}
|
||||
|
||||
The default (and a missing key) is ``{"mode": "source"}``: previews inside
|
||||
each source. An invalid value never falls back to the sources: it parses as
|
||||
a folder location with problem ``invalid``, so nothing is written anywhere.
|
||||
|
||||
Layout in folder mode: ``<folder>/<source key>/<rel dir>/<stem>_thumb<ext>``.
|
||||
A source key is a readable slug plus a short hash of the source's identity
|
||||
(``output-3f9a2c1b``), assigned once and stored in the marker
|
||||
``.promptmanager-thumbnails.json`` with the identity it belongs to.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import ntpath
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import stat as stat_module
|
||||
import threading
|
||||
import unicodedata
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
try:
|
||||
from ...utils import library_location
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils import library_location
|
||||
|
||||
logger = logging.getLogger("prompt_manager.thumbnails")
|
||||
|
||||
SOURCE_MODE = "source"
|
||||
FOLDER_MODE = "folder"
|
||||
MARKER_NAME = ".promptmanager-thumbnails.json"
|
||||
MARKER_VERSION = 1
|
||||
# A source key: one folder name, a slug and a 8-16 hex hash (``source_key``).
|
||||
KEY_PATTERN = re.compile(r"^[a-z0-9_-]{1,24}-[0-9a-f]{8,16}$")
|
||||
# A marker is a few keys; anything larger is not ours.
|
||||
MAX_MARKER_BYTES = 4 * 2**20
|
||||
# OS metadata files a folder may hold and still count as empty.
|
||||
OS_METADATA = frozenset({"thumbs.db", "desktop.ini", ".ds_store"})
|
||||
KEY_HASH_LENGTHS = (8, 12, 16)
|
||||
SLUG_LENGTH = 24
|
||||
# Folder + the longest key (24 + 1 + 16) + this margin must fit (Windows).
|
||||
WINDOWS_FOLDER_LIMIT = 247
|
||||
LONGEST_KEY = SLUG_LENGTH + 1 + KEY_HASH_LENGTHS[-1]
|
||||
PATH_MARGIN = 40
|
||||
# A validation stat on a share that does not answer in time is unavailable.
|
||||
VALIDATION_TIMEOUT = 5.0
|
||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||
|
||||
_RESERVED = re.compile(r"^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\..*)?$", re.IGNORECASE)
|
||||
_BAD_CHARS = re.compile(r'[<>:"|?*\x00-\x1f]')
|
||||
|
||||
# Problem codes, their HTTP status and the text the UI shows (no path).
|
||||
MESSAGES = {
|
||||
"location-not-absolute": (
|
||||
400,
|
||||
"Enter a full folder path, such as D:\\Thumbnails or /mnt/fast/thumbnails.",
|
||||
),
|
||||
"location-device-path": (
|
||||
400,
|
||||
"Enter the folder as a drive letter or \\\\server\\share path, "
|
||||
"without \\\\?\\.",
|
||||
),
|
||||
"location-invalid-name": (
|
||||
400,
|
||||
"This folder name can't be used on Windows. Choose another name.",
|
||||
),
|
||||
"location-too-broad": (
|
||||
400,
|
||||
"Choose a folder inside a drive or share, not the drive itself.",
|
||||
),
|
||||
"location-served": (
|
||||
400,
|
||||
"Choose a folder outside your image sources and ComfyUI's output, input "
|
||||
"and temp folders.",
|
||||
),
|
||||
"location-node-folder": (
|
||||
400,
|
||||
"Choose a folder outside the Prompt Manager node folder.",
|
||||
),
|
||||
"location-contains-source": (
|
||||
400,
|
||||
"This folder contains an image source. Choose a folder beside it instead.",
|
||||
),
|
||||
"location-same": (400, "Thumbnails are already stored here."),
|
||||
"location-parent-missing": (
|
||||
400,
|
||||
"The folder that would hold it doesn't exist. Check the path and that its "
|
||||
"drive is connected.",
|
||||
),
|
||||
"location-unavailable": (
|
||||
409,
|
||||
"This folder can't be reached right now. Check that its drive or share is "
|
||||
"connected.",
|
||||
),
|
||||
"location-not-empty": (
|
||||
409,
|
||||
"This folder already holds other files. Choose an empty folder or one "
|
||||
"Prompt Manager used before.",
|
||||
),
|
||||
"location-not-writable": (
|
||||
409,
|
||||
"Prompt Manager can't write to this folder. Check its permissions.",
|
||||
),
|
||||
"location-path-too-long": (
|
||||
400,
|
||||
"This path is too long for Windows. Choose a shorter folder path.",
|
||||
),
|
||||
"insufficient-space": (
|
||||
507,
|
||||
"There is not enough space for the thumbnails. Free some space or choose "
|
||||
"another folder.",
|
||||
),
|
||||
"location-needs-move": (
|
||||
409,
|
||||
"Use Move thumbnails in System › Sources & scans.",
|
||||
),
|
||||
}
|
||||
HOLDS_THUMBNAILS_REASON = "This folder holds Prompt Manager's thumbnails."
|
||||
|
||||
|
||||
def message(code):
|
||||
"""``(status, text)`` of a problem code."""
|
||||
return MESSAGES[code]
|
||||
|
||||
|
||||
# ── parsing ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedLocation:
|
||||
"""A ``thumbnails.location`` value, checked as data (no filesystem)."""
|
||||
|
||||
mode: str = SOURCE_MODE
|
||||
path: str | None = None
|
||||
id: str | None = None
|
||||
install: str | None = None
|
||||
cleanup: tuple = field(default_factory=tuple)
|
||||
problem: str | None = None
|
||||
|
||||
def to_config(self):
|
||||
"""The config.json form (``invalid`` values are never written)."""
|
||||
if self.mode == SOURCE_MODE:
|
||||
value = {"mode": SOURCE_MODE}
|
||||
else:
|
||||
value = {"mode": FOLDER_MODE, "path": self.path, "id": self.id}
|
||||
if self.install:
|
||||
value["install"] = self.install
|
||||
value["cleanup"] = list(self.cleanup)
|
||||
return value
|
||||
|
||||
|
||||
def _cleanup(value):
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
return tuple(item for item in value if isinstance(item, dict))
|
||||
|
||||
|
||||
def parse_location(value, *, flavor=os.path):
|
||||
"""The location a ``thumbnails.location`` value names.
|
||||
|
||||
Missing: inside each source. Anything else that is not a well-formed
|
||||
location (an unknown mode, a relative path, a missing id) becomes a
|
||||
folder location with problem ``invalid``: never a silent fall-back.
|
||||
"""
|
||||
if value is None:
|
||||
return ParsedLocation()
|
||||
if not isinstance(value, dict):
|
||||
return ParsedLocation(mode=FOLDER_MODE, problem="invalid")
|
||||
mode = value.get("mode", SOURCE_MODE)
|
||||
install = value.get("install") if isinstance(value.get("install"), str) else None
|
||||
cleanup = _cleanup(value.get("cleanup"))
|
||||
if mode == SOURCE_MODE:
|
||||
return ParsedLocation(install=install, cleanup=cleanup)
|
||||
if mode != FOLDER_MODE:
|
||||
return ParsedLocation(mode=FOLDER_MODE, problem="invalid", cleanup=cleanup)
|
||||
path, marker_id = value.get("path"), value.get("id")
|
||||
if (
|
||||
not isinstance(path, str)
|
||||
or form_problem(path, flavor=flavor) is not None
|
||||
or not isinstance(marker_id, str)
|
||||
or not marker_id.strip()
|
||||
):
|
||||
return ParsedLocation(
|
||||
mode=FOLDER_MODE,
|
||||
path=path if isinstance(path, str) else None,
|
||||
problem="invalid",
|
||||
cleanup=cleanup,
|
||||
)
|
||||
return ParsedLocation(
|
||||
mode=FOLDER_MODE,
|
||||
path=flavor.abspath(path),
|
||||
id=marker_id,
|
||||
install=install,
|
||||
cleanup=cleanup,
|
||||
)
|
||||
|
||||
|
||||
# ── form (text only) ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _is_windows(flavor):
|
||||
return flavor is ntpath or getattr(flavor, "__name__", "") == "ntpath"
|
||||
|
||||
|
||||
def form_problem(path, *, flavor=os.path):
|
||||
"""The text-only problem of *path*, or None (no filesystem access).
|
||||
|
||||
``\\\\?\\`` and ``\\\\.\\`` prefixes, relative, drive-relative (``C:x``)
|
||||
and root-relative (``\\x``) spellings, and names Windows cannot hold
|
||||
(checked on every OS: the folder may be moved to Windows).
|
||||
"""
|
||||
if not isinstance(path, str) or not path.strip() or path != path.lstrip():
|
||||
return "location-not-absolute"
|
||||
if path.replace("/", "\\").startswith(("\\\\?\\", "\\\\.\\")):
|
||||
return "location-device-path"
|
||||
if _is_windows(flavor):
|
||||
drive, rest = ntpath.splitdrive(path)
|
||||
if not drive or not rest.startswith(("\\", "/")):
|
||||
return "location-not-absolute"
|
||||
elif not posixpath.isabs(path):
|
||||
return "location-not-absolute"
|
||||
drive, rest = flavor.splitdrive(path)
|
||||
for part in re.split(r"[\\/]+", rest):
|
||||
if not part:
|
||||
continue
|
||||
if (
|
||||
_RESERVED.match(part)
|
||||
or part.endswith((".", " "))
|
||||
or _BAD_CHARS.search(part)
|
||||
):
|
||||
return "location-invalid-name"
|
||||
return None
|
||||
|
||||
|
||||
def is_too_broad(path, *, home=None, flavor=os.path):
|
||||
"""A filesystem or share root, or the home folder itself."""
|
||||
normalized = flavor.normpath(path)
|
||||
drive, rest = flavor.splitdrive(normalized)
|
||||
if rest in ("", "\\", "/"):
|
||||
return True
|
||||
home = home if home is not None else os.path.expanduser("~")
|
||||
return bool(home) and flavor.normcase(normalized) == flavor.normcase(
|
||||
flavor.normpath(home)
|
||||
)
|
||||
|
||||
|
||||
def too_long_for_windows(path, long_paths=None):
|
||||
"""Folder + the longest key + a margin over the folder limit (Windows
|
||||
without long paths; *long_paths* injects the registry answer)."""
|
||||
if long_paths is None:
|
||||
long_paths = long_paths_enabled()
|
||||
if long_paths:
|
||||
return False
|
||||
return len(path) + 1 + LONGEST_KEY + PATH_MARGIN > WINDOWS_FOLDER_LIMIT
|
||||
|
||||
|
||||
def long_paths_enabled():
|
||||
"""``LongPathsEnabled`` on Windows (True on other systems)."""
|
||||
if os.name != "nt":
|
||||
return True
|
||||
try:
|
||||
import winreg
|
||||
|
||||
with winreg.OpenKey(
|
||||
winreg.HKEY_LOCAL_MACHINE,
|
||||
r"SYSTEM\CurrentControlSet\Control\FileSystem",
|
||||
) as key:
|
||||
return winreg.QueryValueEx(key, "LongPathsEnabled")[0] == 1
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
# ── links ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def is_link_like(path):
|
||||
"""A symlink, a junction or any reparse point (one lstat), or missing.
|
||||
|
||||
Raises OSError other than "not found" (an unreachable share).
|
||||
"""
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
if stat_module.S_ISLNK(info.st_mode):
|
||||
return True
|
||||
attributes = getattr(info, "st_file_attributes", 0)
|
||||
if attributes & FILE_ATTRIBUTE_REPARSE_POINT:
|
||||
return True
|
||||
is_junction = getattr(os.path, "isjunction", None)
|
||||
return bool(is_junction and is_junction(path))
|
||||
|
||||
|
||||
# ── source keys ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def slug(name):
|
||||
"""``NFKD`` to ASCII, lower case, runs outside ``[a-z0-9_]`` as ``-``,
|
||||
trimmed and cut to 24 characters; empty becomes ``source``."""
|
||||
text = unicodedata.normalize("NFKD", str(name))
|
||||
text = text.encode("ascii", "ignore").decode("ascii").lower()
|
||||
text = re.sub(r"[^a-z0-9_]+", "-", text).strip("-")
|
||||
text = text[:SLUG_LENGTH].strip("-")
|
||||
return text or "source"
|
||||
|
||||
|
||||
def source_key(spelling, identity, taken):
|
||||
"""The key of a source: ``slug(basename) + "-" + sha256(identity)[:n]``.
|
||||
|
||||
*taken* maps existing keys to their identity; a key taken by another
|
||||
identity grows its hash to 12, then 16 characters.
|
||||
"""
|
||||
base = slug(ntpath.basename(str(spelling).rstrip("\\/")))
|
||||
digest = hashlib.sha256(str(identity).encode("utf-8")).hexdigest()
|
||||
for length in KEY_HASH_LENGTHS:
|
||||
key = f"{base}-{digest[:length]}"
|
||||
owner = taken.get(key)
|
||||
if owner is None or owner == identity:
|
||||
return key
|
||||
raise ValueError("no free source key")
|
||||
|
||||
|
||||
def file_identity(path):
|
||||
"""``[st_dev, st_ino]`` of an existing folder, or None (no file ids)."""
|
||||
try:
|
||||
info = os.stat(path)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not info.st_ino:
|
||||
return None
|
||||
return [info.st_dev, info.st_ino]
|
||||
|
||||
|
||||
def find_key(keys, spelling, identity, dev_ino):
|
||||
"""The marker key of a source: the same configured spelling, then the
|
||||
same identity text, then the same file id; or None."""
|
||||
keys = {k: v for k, v in keys.items() if valid_key(k) and isinstance(v, dict)}
|
||||
for key, entry in keys.items():
|
||||
if entry.get("path") == spelling:
|
||||
return key
|
||||
for key, entry in keys.items():
|
||||
if entry.get("identity") == identity:
|
||||
return key
|
||||
if dev_ino:
|
||||
for key, entry in keys.items():
|
||||
if entry.get("dev_ino") == list(dev_ino):
|
||||
return key
|
||||
return None
|
||||
|
||||
|
||||
# ── the marker ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def marker_path(folder):
|
||||
return os.path.join(folder, MARKER_NAME)
|
||||
|
||||
|
||||
def read_marker(folder):
|
||||
"""The marker in *folder* as a dict, or None (absent or unreadable as
|
||||
ours). OSError other than "not found" propagates (an unreachable share)."""
|
||||
try:
|
||||
with open(marker_path(folder), "r", encoding="utf-8") as handle:
|
||||
text = handle.read(MAX_MARKER_BYTES + 1)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return None
|
||||
if len(text) > MAX_MARKER_BYTES:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except ValueError:
|
||||
return None
|
||||
if not isinstance(data, dict) or not isinstance(data.get("id"), str):
|
||||
return None
|
||||
data["keys"] = valid_keys(data.get("keys"))
|
||||
return data
|
||||
|
||||
|
||||
def valid_key(key):
|
||||
"""Whether *key* can name a source's folder: one folder name of the
|
||||
``source_key`` form, never a separator, ``..`` or a drive."""
|
||||
return isinstance(key, str) and KEY_PATTERN.fullmatch(key) is not None
|
||||
|
||||
|
||||
def valid_keys(keys):
|
||||
"""The marker's keys that are safe to join onto the folder; a hand-edited
|
||||
or foreign entry that is not is dropped (and logged), never followed."""
|
||||
if not isinstance(keys, dict):
|
||||
return {}
|
||||
kept = {}
|
||||
for key, entry in keys.items():
|
||||
if valid_key(key) and isinstance(entry, dict):
|
||||
kept[key] = entry
|
||||
else:
|
||||
logger.warning(
|
||||
"Ignoring a thumbnail folder marker entry that is not a source " "key"
|
||||
)
|
||||
return kept
|
||||
|
||||
|
||||
def write_marker(folder, mutate):
|
||||
"""Re-read the marker, apply ``mutate(marker)`` and write it atomically
|
||||
(a temp sibling renamed over it). Callers hold ``preview_store.write_lock``.
|
||||
Returns the marker written."""
|
||||
current = read_marker(folder) or {}
|
||||
marker = {
|
||||
"version": MARKER_VERSION,
|
||||
"id": current.get("id"),
|
||||
"created_by_pm": bool(current.get("created_by_pm", False)),
|
||||
"keys": dict(current.get("keys") or {}),
|
||||
}
|
||||
mutate(marker)
|
||||
if not marker.get("id"):
|
||||
raise ValueError("a marker needs an id")
|
||||
target = marker_path(folder)
|
||||
temp = os.path.join(
|
||||
folder, f".{MARKER_NAME}.{os.getpid()}-{threading.get_ident()}.tmp"
|
||||
)
|
||||
try:
|
||||
with open(temp, "w", encoding="utf-8") as handle:
|
||||
json.dump(marker, handle, indent=2, sort_keys=True)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temp, target)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temp)
|
||||
except OSError:
|
||||
pass # gone with the rename; a leftover is swept with the temps
|
||||
return marker
|
||||
|
||||
|
||||
def new_id():
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def init_folder(folder, *, created_by_pm=False):
|
||||
"""The marker id of *folder*: an existing marker is adopted; a folder
|
||||
without one gets a new marker (new id, no keys)."""
|
||||
existing = read_marker(folder)
|
||||
if existing is not None:
|
||||
return existing["id"]
|
||||
|
||||
def create(marker):
|
||||
if not marker.get("id"):
|
||||
marker["id"] = new_id()
|
||||
marker["created_by_pm"] = created_by_pm
|
||||
|
||||
return write_marker(folder, create)["id"]
|
||||
|
||||
|
||||
def folder_state(path, expected_id):
|
||||
"""``(available, problem, marker)`` of a configured thumbnail folder.
|
||||
|
||||
Available: the folder exists, is a directory and not a link, junction or
|
||||
reparse point, and holds our marker with *expected_id*. Raises OSError
|
||||
when the folder cannot be reached (an I/O error other than not found).
|
||||
"""
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
return False, "missing", None
|
||||
except OSError as error:
|
||||
if error.errno in (errno.ENOENT, errno.ENOTDIR):
|
||||
return False, "missing", None
|
||||
raise
|
||||
if is_link_like(path):
|
||||
return False, "invalid", None
|
||||
if not stat_module.S_ISDIR(info.st_mode):
|
||||
return False, "invalid", None
|
||||
marker = read_marker(path)
|
||||
if marker is None:
|
||||
return False, "missing", None
|
||||
if marker["id"] != expected_id:
|
||||
return False, "marker-mismatch", marker
|
||||
return True, None, marker
|
||||
|
||||
|
||||
def holds_only_metadata(path):
|
||||
"""Whether an existing folder is empty but for OS metadata files."""
|
||||
with os.scandir(path) as entries:
|
||||
for entry in entries:
|
||||
name = entry.name.lower()
|
||||
if name in OS_METADATA or name.startswith("._"):
|
||||
continue
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ── validation ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Context:
|
||||
"""What a location is checked against (gathered by the caller)."""
|
||||
|
||||
roots: tuple = () # every image source of every root list
|
||||
contained: tuple = () # ComfyUI output/input/temp and the library folder
|
||||
node_folder: str | None = None
|
||||
default_path: str | None = None
|
||||
current_path: str | None = None # the folder in use (folder mode)
|
||||
home: str | None = None
|
||||
flavor: object = os.path
|
||||
long_paths: bool | None = None
|
||||
environ: object = None
|
||||
# ComfyUI's folder_paths module (None: none; unset: imported).
|
||||
folder_paths: object = library_location._UNSET
|
||||
|
||||
|
||||
def _timed(func, timeout):
|
||||
"""``func()`` on a daemon thread; raises TimeoutError past *timeout*.
|
||||
A stuck call is left to finish on its own thread."""
|
||||
result = {}
|
||||
done = threading.Event()
|
||||
|
||||
def run():
|
||||
try:
|
||||
result["value"] = func()
|
||||
except BaseException as error: # handed to the caller
|
||||
result["error"] = error
|
||||
finally:
|
||||
done.set()
|
||||
|
||||
threading.Thread(target=run, name="pm-preview-stat", daemon=True).start()
|
||||
if not done.wait(timeout):
|
||||
raise TimeoutError("the folder did not answer in time")
|
||||
if "error" in result:
|
||||
raise result["error"]
|
||||
return result.get("value")
|
||||
|
||||
|
||||
def _inside(path, folders):
|
||||
return library_location._inside_any(path, folders)
|
||||
|
||||
|
||||
def _same(first, second):
|
||||
return library_location.same_folder(first, second) or os.path.normcase(
|
||||
os.path.realpath(first)
|
||||
) == os.path.normcase(os.path.realpath(second))
|
||||
|
||||
|
||||
def _served(path, context):
|
||||
environ = os.environ if context.environ is None else context.environ
|
||||
return (
|
||||
library_location.data_dir_problem(
|
||||
path,
|
||||
folder_paths=context.folder_paths,
|
||||
environ=environ,
|
||||
gallery_roots=list(context.roots),
|
||||
)
|
||||
== "data-dir-served"
|
||||
)
|
||||
|
||||
|
||||
def _probe_write(path):
|
||||
"""Create and delete a probe file in *path*; False when refused."""
|
||||
probe = os.path.join(path, f".pm-probe-{uuid.uuid4().hex[:8]}.tmp")
|
||||
try:
|
||||
with open(probe, "wb") as handle:
|
||||
handle.write(b"")
|
||||
except OSError:
|
||||
return False
|
||||
try:
|
||||
os.unlink(probe)
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def location_problem(path, *, purpose, context, timeout=VALIDATION_TIMEOUT):
|
||||
"""Why *path* cannot be the thumbnail folder, as a code, or None.
|
||||
|
||||
*purpose*: ``move`` (every rule), ``startup`` (no ``same``, ``empty``,
|
||||
``writable`` or parent rules: availability covers them) or ``recreate``
|
||||
(the folder is missing and is made again).
|
||||
"""
|
||||
flavor = context.flavor
|
||||
code = form_problem(path, flavor=flavor)
|
||||
if code is not None:
|
||||
return code
|
||||
path = flavor.abspath(path)
|
||||
default = bool(context.default_path) and flavor.normcase(path) == flavor.normcase(
|
||||
flavor.abspath(context.default_path)
|
||||
)
|
||||
if is_too_broad(path, home=context.home, flavor=flavor):
|
||||
return "location-too-broad"
|
||||
try:
|
||||
return _timed(
|
||||
lambda: _filesystem_problem(path, purpose, context, default), timeout
|
||||
)
|
||||
except (TimeoutError, OSError):
|
||||
return "location-unavailable"
|
||||
|
||||
|
||||
def _filesystem_problem(path, purpose, context, default):
|
||||
if _served(path, context):
|
||||
return "location-served"
|
||||
if context.node_folder and not default and _inside(path, [context.node_folder]):
|
||||
return "location-node-folder"
|
||||
inner = list(context.roots) + list(context.contained)
|
||||
if any(folder and _inside(folder, [path]) for folder in inner):
|
||||
if not default:
|
||||
return "location-contains-source"
|
||||
if (
|
||||
purpose == "move"
|
||||
and context.current_path
|
||||
and (
|
||||
os.path.normcase(path) == os.path.normcase(context.current_path)
|
||||
or (os.path.isdir(path) and _same(path, context.current_path))
|
||||
)
|
||||
):
|
||||
return "location-same"
|
||||
# Windows, or a test injecting its limit (``long_paths=False``).
|
||||
if os.name == "nt" or context.long_paths is False:
|
||||
if too_long_for_windows(path, context.long_paths):
|
||||
return "location-path-too-long"
|
||||
if purpose == "startup":
|
||||
return None
|
||||
if is_link_like(path):
|
||||
# Never through a link: the folder itself must be a real folder.
|
||||
return "location-not-empty"
|
||||
exists = os.path.isdir(path)
|
||||
if not exists:
|
||||
if os.path.lexists(path):
|
||||
return "location-not-empty"
|
||||
if not default and not os.path.isdir(os.path.dirname(path)):
|
||||
return "location-parent-missing"
|
||||
return None
|
||||
if read_marker(path) is None and not holds_only_metadata(path):
|
||||
return "location-not-empty"
|
||||
if not _probe_write(path):
|
||||
return "location-not-writable"
|
||||
return None
|
||||
@@ -0,0 +1,678 @@
|
||||
"""Where previews live on disk (#320): the one resolver.
|
||||
|
||||
Every mapping between an image source (a "root"), one of its media files and
|
||||
that file's preview on disk goes through this module. URL text stays in
|
||||
``thumbnail_settings`` (``thumbnail_rel``: ``thumbnails/<rel>_thumb<ext>``,
|
||||
the shape the serve route, ``inspection-state.mjs`` and ``admin.js`` share);
|
||||
this module answers where such a preview is on disk.
|
||||
|
||||
The store of a source is where its previews live. With the default location
|
||||
(mode ``source``, "Inside each source") it is ``<source>/thumbnails``, exactly
|
||||
the folder every caller joined by hand before, so nothing moves.
|
||||
|
||||
Writes and the switch: every preview writer takes ``write_guard`` when it
|
||||
starts (it records the location's ``generation``); the final rename runs
|
||||
under ``write_lock`` and only when the generation is unchanged and no hold
|
||||
applies (``thumbnail_encode.replace_unless_held``). Changing the location
|
||||
bumps the generation under the same lock, so no write that began before can
|
||||
land in the old store afterwards. Nothing reads or commits config while
|
||||
``write_lock`` is held. The hold check runs under it too, so hold checks take
|
||||
no lock (the job registry's ``running()`` is a lock-free snapshot) and
|
||||
``bump_generation`` is never called with a registry or config lock held.
|
||||
|
||||
All functions are blocking: call them off the event loop.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
|
||||
from . import preview_location, thumbnail_settings
|
||||
|
||||
try:
|
||||
from ...utils.media_folders import PREVIEW_FOLDER
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils.media_folders import PREVIEW_FOLDER
|
||||
|
||||
logger = logging.getLogger("prompt_manager.thumbnails")
|
||||
|
||||
SOURCE_MODE = preview_location.SOURCE_MODE
|
||||
FOLDER_MODE = preview_location.FOLDER_MODE
|
||||
# The probe checks the folder this often, and a check (or a store call)
|
||||
# taking longer than the budget marks it unreachable.
|
||||
PROBE_INTERVAL = 5.0
|
||||
IO_BUDGET = 2.0
|
||||
PROBE_THREAD = "pm-preview-probe"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Location:
|
||||
"""One snapshot of the thumbnail location (take one per request or job)."""
|
||||
|
||||
mode: str = SOURCE_MODE
|
||||
path: str | None = None
|
||||
available: bool = True
|
||||
problem: str | None = None
|
||||
generation: int = 0
|
||||
|
||||
|
||||
write_lock = threading.Lock()
|
||||
_generation = 0
|
||||
# The runtime location: the parsed setting, availability and the marker keys
|
||||
# last read. Swapped as a whole; readers take one snapshot.
|
||||
_state_lock = threading.Lock()
|
||||
_parsed = preview_location.ParsedLocation()
|
||||
_available = True
|
||||
_problem = None
|
||||
_keys = {}
|
||||
_key_cache = {}
|
||||
_clock = time.monotonic
|
||||
|
||||
|
||||
def current():
|
||||
"""The thumbnail location now."""
|
||||
with _state_lock:
|
||||
parsed, available, problem, generation_now = (
|
||||
_parsed,
|
||||
_available,
|
||||
_problem,
|
||||
_generation,
|
||||
)
|
||||
if parsed.mode == SOURCE_MODE:
|
||||
return Location(generation=generation_now)
|
||||
return Location(FOLDER_MODE, parsed.path, available, problem, generation_now)
|
||||
|
||||
|
||||
def setting():
|
||||
"""The parsed setting in use (``preview_location.ParsedLocation``)."""
|
||||
return _parsed
|
||||
|
||||
|
||||
def generation():
|
||||
return _generation
|
||||
|
||||
|
||||
def bump_generation():
|
||||
"""Invalidate every write that started before (under ``write_lock``)."""
|
||||
global _generation
|
||||
with write_lock:
|
||||
_generation += 1
|
||||
return _generation
|
||||
|
||||
|
||||
def _canonical(path):
|
||||
"""Resolved and case-folded, so paths compare on every OS."""
|
||||
return Path(os.path.normcase(os.path.realpath(str(path))))
|
||||
|
||||
|
||||
def _within(path, folder):
|
||||
return _canonical(path).is_relative_to(_canonical(folder))
|
||||
|
||||
|
||||
# ── the setting and availability ─────────────────────────────────────
|
||||
|
||||
|
||||
def configure(value, *, context=None, check=True):
|
||||
"""Use the ``thumbnails.location`` *value* (config.json) from now on.
|
||||
|
||||
A folder location is validated against *context* (``startup`` rules: a
|
||||
problem other than an unreachable folder makes it ``invalid``), then
|
||||
checked once; the probe keeps checking it. Swapped under ``write_lock``
|
||||
with a generation bump, so no write that started before lands.
|
||||
"""
|
||||
global _parsed, _available, _problem, _generation
|
||||
parsed = preview_location.parse_location(value)
|
||||
problem = parsed.problem
|
||||
if parsed.mode == FOLDER_MODE and problem is None and context is not None:
|
||||
code = preview_location.location_problem(
|
||||
parsed.path, purpose="startup", context=context
|
||||
)
|
||||
if code == "location-unavailable":
|
||||
problem = "unreachable"
|
||||
elif code is not None:
|
||||
problem = "invalid"
|
||||
parsed = preview_location.ParsedLocation(
|
||||
mode=FOLDER_MODE,
|
||||
path=parsed.path,
|
||||
id=parsed.id,
|
||||
install=parsed.install,
|
||||
cleanup=parsed.cleanup,
|
||||
problem="invalid",
|
||||
)
|
||||
with write_lock:
|
||||
with _state_lock:
|
||||
_parsed = parsed
|
||||
_available = parsed.mode == SOURCE_MODE
|
||||
_problem = problem if parsed.mode == FOLDER_MODE else None
|
||||
_keys.clear()
|
||||
_key_cache.clear()
|
||||
_generation += 1
|
||||
if parsed.mode == FOLDER_MODE:
|
||||
if check and parsed.problem is None:
|
||||
refresh()
|
||||
_probe.start()
|
||||
else:
|
||||
_probe.stop()
|
||||
return current()
|
||||
|
||||
|
||||
def refresh():
|
||||
"""Check the folder now (bounded by IO_BUDGET); returns availability."""
|
||||
parsed = _parsed
|
||||
if parsed.mode != FOLDER_MODE or parsed.problem is not None:
|
||||
return parsed.mode == SOURCE_MODE
|
||||
try:
|
||||
available, problem, marker = _probe.check(parsed.path, parsed.id)
|
||||
except (OSError, TimeoutError, RuntimeError):
|
||||
available, problem, marker = False, "unreachable", None
|
||||
_set_state(parsed, available, problem, marker)
|
||||
return available
|
||||
|
||||
|
||||
def _set_keys(parsed, marker):
|
||||
"""Take the keys of a marker just written (availability unchanged)."""
|
||||
with _state_lock:
|
||||
if _parsed is not parsed:
|
||||
return
|
||||
keys = dict((marker or {}).get("keys") or {})
|
||||
if keys != _keys:
|
||||
_keys.clear()
|
||||
_keys.update(keys)
|
||||
_key_cache.clear()
|
||||
|
||||
|
||||
def set_cleanup(entries):
|
||||
"""The setting's ``cleanup`` after a removal was committed."""
|
||||
global _parsed
|
||||
with _state_lock:
|
||||
_parsed = replace(_parsed, cleanup=tuple(entries))
|
||||
|
||||
|
||||
def mark_invalid():
|
||||
"""The setting cannot be used (a startup check that failed): folder
|
||||
mode stays in force, unavailable, so nothing is written anywhere."""
|
||||
global _parsed, _available, _problem
|
||||
with _state_lock:
|
||||
if _parsed.mode != FOLDER_MODE:
|
||||
_parsed = preview_location.ParsedLocation(
|
||||
mode=FOLDER_MODE, problem="invalid"
|
||||
)
|
||||
else:
|
||||
_parsed = replace(_parsed, problem="invalid")
|
||||
_available, _problem = False, "invalid"
|
||||
|
||||
|
||||
def _set_state(parsed, available, problem, marker):
|
||||
global _available, _problem
|
||||
with _state_lock:
|
||||
if _parsed is not parsed:
|
||||
return # the setting changed meanwhile
|
||||
_available, _problem = available, problem
|
||||
keys = dict((marker or {}).get("keys") or {})
|
||||
if keys != _keys:
|
||||
_keys.clear()
|
||||
_keys.update(keys)
|
||||
_key_cache.clear()
|
||||
|
||||
|
||||
def note_failure():
|
||||
"""A store call failed or stalled: unreachable until the probe finds the
|
||||
folder again (requests do no store I/O meanwhile)."""
|
||||
global _available, _problem
|
||||
if _parsed.mode != FOLDER_MODE:
|
||||
return
|
||||
with _state_lock:
|
||||
_available, _problem = False, "unreachable"
|
||||
_probe.wake()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def store_io():
|
||||
"""Around a store call in folder mode: an I/O error (other than a file
|
||||
that is not there) or a call over IO_BUDGET marks the folder
|
||||
unreachable."""
|
||||
if _parsed.mode != FOLDER_MODE:
|
||||
yield
|
||||
return
|
||||
started = _clock()
|
||||
try:
|
||||
yield
|
||||
except OSError as error:
|
||||
if not isinstance(error, (FileNotFoundError, NotADirectoryError)):
|
||||
note_failure()
|
||||
raise
|
||||
finally:
|
||||
if _clock() - started > IO_BUDGET:
|
||||
note_failure()
|
||||
|
||||
|
||||
class _Probe:
|
||||
"""The background check of the thumbnail folder (one thread, one check
|
||||
at a time; a stuck check is left to finish on its own thread)."""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._wake = threading.Event()
|
||||
self._stop = threading.Event()
|
||||
self._thread = None
|
||||
self._checking = None
|
||||
|
||||
def check(self, path, expected_id):
|
||||
"""``folder_state`` within IO_BUDGET; raises TimeoutError. A check
|
||||
already running (the probe's, or a stuck one) is joined, never run
|
||||
twice at once."""
|
||||
with self._lock:
|
||||
pending = self._checking
|
||||
if pending is None or pending[0] != (path, expected_id):
|
||||
pending = self._start_check(path, expected_id)
|
||||
key, done, result = pending
|
||||
if not done.wait(IO_BUDGET):
|
||||
raise TimeoutError("the thumbnail folder did not answer in time")
|
||||
with self._lock:
|
||||
if self._checking is pending:
|
||||
self._checking = None
|
||||
if "error" in result:
|
||||
raise result["error"]
|
||||
return result["value"]
|
||||
|
||||
def _start_check(self, path, expected_id):
|
||||
"""Start one check on its own thread (lock held)."""
|
||||
result = {}
|
||||
done = threading.Event()
|
||||
|
||||
def run():
|
||||
try:
|
||||
result["value"] = preview_location.folder_state(path, expected_id)
|
||||
except BaseException as error: # handed to the caller
|
||||
result["error"] = error
|
||||
finally:
|
||||
done.set()
|
||||
|
||||
pending = ((path, expected_id), done, result)
|
||||
self._checking = pending
|
||||
threading.Thread(target=run, name=PROBE_THREAD + "-check", daemon=True).start()
|
||||
return pending
|
||||
|
||||
def start(self):
|
||||
with self._lock:
|
||||
if self._thread is not None and self._thread.is_alive():
|
||||
return
|
||||
# Each thread has its own stop: a stopped one never resumes.
|
||||
stop = self._stop = threading.Event()
|
||||
thread = threading.Thread(
|
||||
target=self._run, args=(stop,), name=PROBE_THREAD, daemon=True
|
||||
)
|
||||
try:
|
||||
thread.start()
|
||||
except RuntimeError:
|
||||
return # refused at interpreter exit: no probe
|
||||
self._thread = thread
|
||||
|
||||
def _run(self, stop):
|
||||
while not stop.is_set():
|
||||
self._wake.wait(PROBE_INTERVAL)
|
||||
self._wake.clear()
|
||||
if stop.is_set():
|
||||
return
|
||||
try:
|
||||
refresh()
|
||||
except Exception: # one failed check never stops the probe
|
||||
logger.debug("Thumbnail folder check failed", exc_info=True)
|
||||
|
||||
def wake(self):
|
||||
self._wake.set()
|
||||
|
||||
def stop(self, timeout=2.0):
|
||||
"""Stop the probe; True when its thread ended."""
|
||||
with self._lock:
|
||||
thread, self._thread = self._thread, None
|
||||
self._stop.set()
|
||||
self._wake.set()
|
||||
if thread is None or thread is threading.current_thread():
|
||||
return True
|
||||
thread.join(timeout)
|
||||
return not thread.is_alive()
|
||||
|
||||
|
||||
_probe = _Probe()
|
||||
|
||||
|
||||
def stop_probe(timeout=2.0):
|
||||
"""Stop the folder probe (the package teardown, tests)."""
|
||||
return _probe.stop(timeout)
|
||||
|
||||
|
||||
# ── stores ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def source_folder(root):
|
||||
"""``<source>/thumbnails``: the store in source mode, a leftover in
|
||||
folder mode."""
|
||||
return Path(root) / PREVIEW_FOLDER
|
||||
|
||||
|
||||
def store_container(root, location=None):
|
||||
"""The folder a store of *root* must stay inside: the source itself, or
|
||||
the thumbnail folder in folder mode."""
|
||||
location = location or current()
|
||||
if location.mode == FOLDER_MODE:
|
||||
return Path(location.path) if location.path else None
|
||||
return Path(root)
|
||||
|
||||
|
||||
def _identity(root):
|
||||
return os.path.normcase(os.path.realpath(str(root)))
|
||||
|
||||
|
||||
def _lookup_key(root):
|
||||
spelling = str(root)
|
||||
with _state_lock:
|
||||
if spelling in _key_cache:
|
||||
return _key_cache[spelling]
|
||||
keys = dict(_keys)
|
||||
key = preview_location.find_key(
|
||||
keys,
|
||||
spelling,
|
||||
_identity(root),
|
||||
preview_location.file_identity(root),
|
||||
)
|
||||
if key is not None:
|
||||
with _state_lock:
|
||||
_key_cache[spelling] = key
|
||||
return key
|
||||
|
||||
|
||||
def store_dir(root, location=None):
|
||||
"""The store of *root*: ``<source>/thumbnails``, or in folder mode
|
||||
``<folder>/<key>`` (None while the folder is unavailable or before the
|
||||
source has a key: a reader then finds no preview)."""
|
||||
location = location or current()
|
||||
if location.mode == SOURCE_MODE:
|
||||
return Path(root) / PREVIEW_FOLDER
|
||||
if not location.available or not location.path:
|
||||
return None
|
||||
key = _lookup_key(root)
|
||||
return None if key is None else _key_folder(location.path, key)
|
||||
|
||||
|
||||
def _key_folder(folder, key):
|
||||
"""``<folder>/<key>``, only for a key that is one folder name directly
|
||||
below the thumbnail folder (a marker is data, never a path)."""
|
||||
if not preview_location.valid_key(key):
|
||||
return None
|
||||
store = Path(folder) / key
|
||||
if store.parent != Path(folder) or store.name != key:
|
||||
return None
|
||||
return store
|
||||
|
||||
|
||||
def ensure_store(root, location=None, *, create=False):
|
||||
"""``store_dir`` for a writer: in folder mode a source without a key gets
|
||||
one (recorded in the marker); with *create* its key folder is made (never
|
||||
the thumbnail folder itself). None while the folder is unavailable."""
|
||||
location = location or current()
|
||||
store = store_dir(root, location)
|
||||
if location.mode == SOURCE_MODE or (store is None and not location.available):
|
||||
return store
|
||||
if store is None:
|
||||
store = _assign_key(root, location)
|
||||
if store is None:
|
||||
return None
|
||||
if create:
|
||||
try:
|
||||
with store_io():
|
||||
os.mkdir(store)
|
||||
except FileExistsError:
|
||||
pass
|
||||
except OSError:
|
||||
return None
|
||||
return store
|
||||
|
||||
|
||||
def _assign_key(root, location):
|
||||
parsed = _parsed
|
||||
if location.path != parsed.path:
|
||||
return None
|
||||
spelling = str(root)
|
||||
identity = _identity(root)
|
||||
dev_ino = preview_location.file_identity(root)
|
||||
assigned = {}
|
||||
|
||||
def add(marker):
|
||||
if marker.get("id") != parsed.id:
|
||||
raise ValueError("another folder's marker") # marker-mismatch
|
||||
keys = marker["keys"]
|
||||
key = preview_location.find_key(keys, spelling, identity, dev_ino)
|
||||
if key is None:
|
||||
taken = {k: v.get("identity") for k, v in keys.items()}
|
||||
key = preview_location.source_key(spelling, identity, taken)
|
||||
keys[key] = {
|
||||
"path": spelling,
|
||||
"identity": identity,
|
||||
"dev_ino": dev_ino,
|
||||
"install": parsed.install,
|
||||
}
|
||||
assigned["key"] = key
|
||||
|
||||
try:
|
||||
with write_lock, store_io():
|
||||
marker = preview_location.write_marker(location.path, add)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
_set_keys(parsed, marker)
|
||||
return _key_folder(location.path, assigned["key"])
|
||||
|
||||
|
||||
def may_write(path, location=None):
|
||||
"""The one ownership check every preview write and delete passes.
|
||||
|
||||
Source mode: True (the callers' own root rules apply, as before). Folder
|
||||
mode: *path* lies below a source key folder of the thumbnail folder,
|
||||
lexically and resolved, and no link, junction or reparse point sits
|
||||
anywhere from the key folder down to it (the file itself included).
|
||||
"""
|
||||
location = location or current()
|
||||
if location.mode == SOURCE_MODE:
|
||||
return True
|
||||
if not location.path:
|
||||
return False
|
||||
folder = Path(os.path.abspath(location.path))
|
||||
path = Path(os.path.abspath(path))
|
||||
try:
|
||||
relative = path.relative_to(folder)
|
||||
except ValueError:
|
||||
return False
|
||||
if len(relative.parts) < 2 or ".." in relative.parts:
|
||||
return False
|
||||
store = _key_folder(folder, relative.parts[0])
|
||||
if store is None or not _within(path, folder):
|
||||
return False
|
||||
try:
|
||||
return _no_link_below(store, path)
|
||||
except OSError:
|
||||
note_failure()
|
||||
return False
|
||||
|
||||
|
||||
def make_parent(dst, location=None):
|
||||
"""Make the folders a preview *dst* is written into. In folder mode only
|
||||
folders below the thumbnail folder are made, never the folder itself,
|
||||
and only while it holds this install's marker (an unplugged drive or an
|
||||
unmounted mount point raises FileNotFoundError); a *dst* that does not
|
||||
pass ``may_write`` raises PermissionError."""
|
||||
location = location or current()
|
||||
dst = Path(dst)
|
||||
if location.mode == SOURCE_MODE or not location.path:
|
||||
if location.mode != SOURCE_MODE:
|
||||
raise FileNotFoundError(errno.ENOENT, "thumbnail folder unavailable")
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
return
|
||||
folder = Path(os.path.abspath(location.path))
|
||||
with store_io():
|
||||
if not location.available or not folder.is_dir():
|
||||
raise FileNotFoundError(errno.ENOENT, "thumbnail folder unavailable")
|
||||
marker = preview_location.read_marker(str(folder))
|
||||
if marker is None or marker.get("id") != _parsed.id:
|
||||
note_failure()
|
||||
raise FileNotFoundError(errno.ENOENT, "thumbnail folder unavailable")
|
||||
if not may_write(dst, location):
|
||||
raise PermissionError(errno.EACCES, "outside the thumbnail folder")
|
||||
current_folder = folder
|
||||
for part in Path(os.path.abspath(dst.parent)).relative_to(folder).parts:
|
||||
current_folder = current_folder / part
|
||||
try:
|
||||
os.mkdir(current_folder)
|
||||
except FileExistsError:
|
||||
pass
|
||||
if preview_location.is_link_like(current_folder):
|
||||
raise PermissionError(errno.EACCES, "a link in the thumbnail folder")
|
||||
|
||||
|
||||
def store_is_link(store, location=None):
|
||||
"""Whether a store folder is a link (a junction or reparse point too in
|
||||
folder mode): it is skipped by sweeps and refused by clears."""
|
||||
location = location or current()
|
||||
if location.mode == SOURCE_MODE:
|
||||
return Path(store).is_symlink()
|
||||
return preview_location.is_link_like(store)
|
||||
|
||||
|
||||
def given_store(root, store):
|
||||
"""A caller's explicit store argument: the default spelling
|
||||
(``<root>/thumbnails``) means the resolver's store; another folder (only
|
||||
tests pass one) is used as given."""
|
||||
store = Path(store)
|
||||
if store == Path(root) / PREVIEW_FOLDER:
|
||||
return store_dir(root)
|
||||
return store
|
||||
|
||||
|
||||
def preview_path(root, media_rel, ext=None, location=None, *, create=False):
|
||||
"""Where the preview of *media_rel* (relative to *root*) is, or is to be
|
||||
written (*create*: a writer, which gives a new source its key), in *ext*
|
||||
(the extension written now when None)."""
|
||||
if create:
|
||||
store = ensure_store(root, location)
|
||||
else:
|
||||
store = store_dir(root, location)
|
||||
if store is None:
|
||||
return None
|
||||
if ext is None:
|
||||
from .thumbnail_encode import writer_settings
|
||||
|
||||
ext = writer_settings().extension
|
||||
return store / (thumbnail_settings.preview_stem(media_rel) + ext)
|
||||
|
||||
|
||||
def request_path(root, parts, location=None):
|
||||
"""The disk path of the URL parts ``("thumbnails", ...)``, or None when
|
||||
*parts* is not under an exact ``thumbnails`` first part or the store has
|
||||
no place for it. The text is checked before (``is_preview_text``)."""
|
||||
parts = tuple(parts)
|
||||
if len(parts) < 2 or parts[0] != PREVIEW_FOLDER:
|
||||
return None
|
||||
store = store_dir(root, location)
|
||||
if store is None:
|
||||
return None
|
||||
return store.joinpath(*parts[1:])
|
||||
|
||||
|
||||
def _no_link_below(store, path):
|
||||
"""No folder from *store* down to *path* (and *store* itself) is a link,
|
||||
junction or reparse point (lexical walk, one lstat each)."""
|
||||
store = Path(os.path.abspath(store))
|
||||
path = Path(os.path.abspath(path))
|
||||
try:
|
||||
relative = path.relative_to(store)
|
||||
except ValueError:
|
||||
return False
|
||||
current = store
|
||||
for part in ("",) + relative.parts:
|
||||
current = current / part if part else current
|
||||
if preview_location.is_link_like(current):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def owns(root, path, location=None):
|
||||
"""Whether *path* lies inside the store of *root*, resolved: the store
|
||||
stays inside its container and no link takes *path* out of the store.
|
||||
In folder mode no link, junction or reparse point may sit anywhere from
|
||||
the store down to *path*."""
|
||||
location = location or current()
|
||||
store = store_dir(root, location)
|
||||
if store is None:
|
||||
return False
|
||||
container = store_container(root, location)
|
||||
if not (_within(store, container) and _within(path, store)):
|
||||
return False
|
||||
if location.mode == SOURCE_MODE:
|
||||
return True
|
||||
return may_write(path, location)
|
||||
|
||||
|
||||
# ── preview text ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def is_preview_text(filepath):
|
||||
"""URL text naming a preview: ``thumbnails/<...>/<name>_thumb<ext>``.
|
||||
|
||||
Text only, before any join: no backslash, drive or stream colon, and no
|
||||
empty, ``.`` or ``..`` segment; ``thumbnails`` exactly as the first part
|
||||
and nowhere else among the folders; the extension, in any case, one a
|
||||
preview may carry (``X_thumb.JPG`` of an older release included).
|
||||
"""
|
||||
if not filepath or "\\" in filepath or ":" in filepath:
|
||||
return False
|
||||
parts = filepath.split("/")
|
||||
if len(parts) < 2 or parts[0] != PREVIEW_FOLDER:
|
||||
return False
|
||||
if any(part in ("", ".", "..") for part in parts):
|
||||
return False
|
||||
if PREVIEW_FOLDER in parts[1:-1]:
|
||||
return False
|
||||
stem, extension = os.path.splitext(parts[-1])
|
||||
return (
|
||||
stem.endswith(thumbnail_settings.THUMB_SUFFIX)
|
||||
and len(stem) > len(thumbnail_settings.THUMB_SUFFIX)
|
||||
and extension.lower() in thumbnail_settings.THUMBNAIL_EXTENSIONS
|
||||
)
|
||||
|
||||
|
||||
# ── writes ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _WriteGuard:
|
||||
"""``guard()`` for one preview write: true while the location has not
|
||||
changed since the write started and *hold* (if any) allows it."""
|
||||
|
||||
__slots__ = ("generation", "_hold")
|
||||
|
||||
def __init__(self, hold=None):
|
||||
self.generation = _generation
|
||||
self._hold = hold
|
||||
|
||||
def __call__(self):
|
||||
if self.generation != _generation:
|
||||
return False
|
||||
return self._hold is None or bool(self._hold())
|
||||
|
||||
|
||||
def write_guard(hold=None):
|
||||
"""Take when a preview write starts; hand to its final rename."""
|
||||
if isinstance(hold, _WriteGuard):
|
||||
return hold
|
||||
return _WriteGuard(hold)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def committed(guard=None):
|
||||
"""Around a preview's final rename: holds ``write_lock`` and yields
|
||||
whether the write may land (``guard()``, re-checked under the lock)."""
|
||||
with write_lock:
|
||||
yield guard is None or bool(guard())
|
||||
@@ -30,6 +30,7 @@ from . import thumbnail_settings
|
||||
from .admin import _current_image_monitor
|
||||
from .job_routes import busy_response
|
||||
from .jobs import THUMBNAILS_JOB_KIND, JobBusy
|
||||
from .thumbnail_location import needs_move_response
|
||||
|
||||
try:
|
||||
from ...utils.validators import validate_result_timeout, validate_worker_threads
|
||||
@@ -264,6 +265,12 @@ class SettingsRoutesMixin:
|
||||
"sources": source_status,
|
||||
"monitor": monitor,
|
||||
"thumbnail_encoders": encoders,
|
||||
# #320: the full path only here (D6A).
|
||||
"thumbnail_location": await self._run_in_executor(
|
||||
functools.partial(
|
||||
self._thumbnail_location_payload, with_path=True
|
||||
)
|
||||
),
|
||||
"settings": {
|
||||
"monitoring_enabled": GalleryConfig.MONITORING_ENABLED,
|
||||
"save_foreign_runs": GalleryConfig.SAVE_FOREIGN_RUNS,
|
||||
@@ -377,6 +384,9 @@ class SettingsRoutesMixin:
|
||||
if error_response is not None:
|
||||
return error_response
|
||||
|
||||
if "thumbnail_location" in data:
|
||||
# #320: only the Move thumbnails job changes the location.
|
||||
return needs_move_response()
|
||||
expected_revision = data.get("expected_revision")
|
||||
if expected_revision is not None and not isinstance(expected_revision, str):
|
||||
return web.json_response(
|
||||
@@ -481,7 +491,8 @@ class SettingsRoutesMixin:
|
||||
for key in _THUMBNAIL_SETTINGS
|
||||
)
|
||||
job = self._jobs.active(THUMBNAILS_JOB_KIND) if changed else None
|
||||
if job is None or job.scope.get("mode") != "convert":
|
||||
# A move copies previews of the current settings (#320).
|
||||
if job is None or job.scope.get("mode") not in ("convert", "move"):
|
||||
return None
|
||||
return web.json_response(
|
||||
{
|
||||
|
||||
+22
-1
@@ -30,6 +30,8 @@ import ntpath
|
||||
import os
|
||||
import re
|
||||
|
||||
from .preview_location import HOLDS_THUMBNAILS_REASON
|
||||
|
||||
logger = logging.getLogger("prompt_manager.api")
|
||||
|
||||
MAX_SOURCE_PATH = 4096 # characters
|
||||
@@ -114,7 +116,26 @@ def source_problem(text):
|
||||
code = source_form_error(text)
|
||||
if code is not None:
|
||||
return code, FORM_REASONS[code]
|
||||
return GalleryConfig.gallery_root_problem(text)
|
||||
code, reason = GalleryConfig.gallery_root_problem(text)
|
||||
if code is None and _holds_thumbnails(GalleryConfig.resolve_gallery_root(text)):
|
||||
return "holds-thumbnails", HOLDS_THUMBNAILS_REASON
|
||||
return code, reason
|
||||
|
||||
|
||||
def _holds_thumbnails(path):
|
||||
"""Whether *path* is inside or around the thumbnail folder in use (#320):
|
||||
a source there would list Prompt Manager's own previews."""
|
||||
from . import preview_store
|
||||
|
||||
try:
|
||||
from ...utils.library_location import _inside_any
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils.library_location import _inside_any
|
||||
|
||||
setting = preview_store.setting()
|
||||
if setting.mode != preview_store.FOLDER_MODE or not setting.path:
|
||||
return False
|
||||
return _inside_any(path, [setting.path]) or _inside_any(setting.path, [path])
|
||||
|
||||
|
||||
def public_spelling(text):
|
||||
|
||||
+193
-50
@@ -8,7 +8,8 @@ previews behind those URLs are made here, in three ways:
|
||||
- **On first view.** ``GET /prompt_manager/images/serve/thumbnails/<rel>_thumb
|
||||
<ext>`` for a preview that does not exist yet is admitted on the event loop
|
||||
(``_PreviewGate``): at most ``ON_DEMAND_LIMIT`` previews are written at once,
|
||||
on this API's own small pool (never the default executor), one flight runs
|
||||
on this API's interactive image pool (``IMAGE_POOL_SIZE`` threads, shared
|
||||
with card facts and inspector reads; never the default executor), one flight runs
|
||||
per file, and the requests waiting are capped (``MAX_WAITING``); past the
|
||||
cap a request gets its fallback at once. Inside the admission the preview is
|
||||
mapped back to its media file under the same root (the request text is
|
||||
@@ -24,7 +25,8 @@ previews behind those URLs are made here, in three ways:
|
||||
- **On capture.** The image monitor hands every new output it settles to
|
||||
``queue_capture_thumbnail``, which queues its preview for one background
|
||||
worker thread (``AutoThumbnailer``). Each wake takes at most Worker threads
|
||||
(``WORKER_THREADS``) files and writes them on that many threads; enqueuing
|
||||
(``WORKER_THREADS``) files and writes them on the long-lived job pool
|
||||
(``utils.parallel``, shared with scans and thumbnail jobs); enqueuing
|
||||
is O(1), so the monitor and ComfyUI's queue never wait. A file already
|
||||
queued is not queued again, the queue is bounded (``MAX_QUEUED``) and a
|
||||
preview left out is made on its first view instead.
|
||||
@@ -49,7 +51,7 @@ write starts (after its slot) and again just before its rename.
|
||||
|
||||
Shutdown: ``close_auto_thumbnails`` stops the worker at a file boundary,
|
||||
kills a running ffmpeg, waits a bounded time (and logs when the wait ran
|
||||
out) and shuts the view pool down; after the app's cleanup
|
||||
out) and shuts the image pool down; after the app's cleanup
|
||||
(``final=True``) nothing starts a worker again. Importing this module again
|
||||
(a package reload) stops the workers the previous import left.
|
||||
"""
|
||||
@@ -60,12 +62,12 @@ import functools
|
||||
import io
|
||||
import os
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import CancelledError, ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import thumbnail_settings
|
||||
from . import preview_store, thumbnail_settings
|
||||
from .images import (
|
||||
in_trash_below_roots,
|
||||
MEDIA_EXTENSIONS,
|
||||
@@ -76,7 +78,7 @@ from .images import (
|
||||
_thumbnail_is_fresh,
|
||||
_video_placeholder_image,
|
||||
_worker_threads,
|
||||
media_thumbnail_rel,
|
||||
media_preview_path,
|
||||
)
|
||||
from .jobs import (
|
||||
DUPLICATE_CLEANUP_JOB_KIND,
|
||||
@@ -104,8 +106,11 @@ PLACEHOLDER_SIZE = (thumbnail_settings.DEFAULT_EDGE, thumbnail_settings.DEFAULT_
|
||||
PLACEHOLDER_QUALITY = 85
|
||||
# Source extensions in the order a shared preview name looks for them (R2).
|
||||
_SOURCE_ORDER = thumbnail_settings.IMAGE_SOURCE_ORDER + tuple(sorted(VIDEO_EXTENSIONS))
|
||||
# Previews views may write at once (the size of the view pool).
|
||||
ON_DEMAND_LIMIT = 3
|
||||
# Threads of the interactive image pool (#523): view previews, card facts and
|
||||
# inspector reads. Fixed, so image buffers stay in a few threads' arenas.
|
||||
IMAGE_POOL_SIZE = 3
|
||||
# Previews views may write at once (the whole image pool).
|
||||
ON_DEMAND_LIMIT = IMAGE_POOL_SIZE
|
||||
# View requests that may wait on the gate; one more gets its fallback at once.
|
||||
MAX_WAITING = 64
|
||||
# How long a pool thread waits for the capture worker's flight of the same file.
|
||||
@@ -124,7 +129,7 @@ AUTO_REFUSED_KINDS = frozenset(
|
||||
{RESTORE_JOB_KIND, DUPLICATE_CLEANUP_JOB_KIND, SOURCE_CHANGE_JOB_KIND}
|
||||
)
|
||||
WORKER_NAME = "PromptManagerThumbnailWorker"
|
||||
POOL_PREFIX = "pm-preview"
|
||||
POOL_PREFIX = "pm-image"
|
||||
# Identifies this import's workers: a reload stops the ones it did not start.
|
||||
_MODULE_TOKEN = object()
|
||||
|
||||
@@ -153,6 +158,36 @@ def _dst_key(dst):
|
||||
return flight_key(Path(dst).parent, (Path(dst).name,))
|
||||
|
||||
|
||||
class DeferredPreview:
|
||||
"""The preview of *rel* under *root*, resolved when its write starts
|
||||
(#320): a queued capture follows the thumbnail location at write time,
|
||||
not the one when it was queued."""
|
||||
|
||||
__slots__ = ("root", "rel", "strict")
|
||||
|
||||
def __init__(self, root, rel, *, strict=True):
|
||||
self.root = Path(root)
|
||||
self.rel = Path(rel)
|
||||
# A capture refuses a store linked out of its container (the serve
|
||||
# route's rule); a planned target keeps the plan's rule: inside the
|
||||
# store, as the thumbnail job writes it.
|
||||
self.strict = strict
|
||||
|
||||
def resolve(self):
|
||||
"""The preview's path now, or None when nothing may be written."""
|
||||
dst = media_preview_path(self.root, self.rel, create=True)
|
||||
if dst is None:
|
||||
return None
|
||||
if self.strict:
|
||||
return dst if preview_store.owns(self.root, dst) else None
|
||||
store = preview_store.store_dir(self.root)
|
||||
return dst if _is_within(dst, store) else None
|
||||
|
||||
|
||||
def _resolved(dst):
|
||||
return dst.resolve() if isinstance(dst, DeferredPreview) else Path(dst)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def video_placeholder_jpeg():
|
||||
"""The placeholder frame as JPEG bytes (a video whose preview is missing)."""
|
||||
@@ -210,10 +245,19 @@ def _candidates(parts):
|
||||
def _current_preview(root, parts):
|
||||
"""The preview *parts* names, in the format written now (#317): a request
|
||||
for an older name (``_thumb.png`` before a conversion) is answered with
|
||||
the current file."""
|
||||
the current file. None when the store of *root* has no place for it."""
|
||||
_folder, base, _wanted = _candidates(parts)
|
||||
name = base + thumbnail_settings.THUMB_SUFFIX + writer_settings().extension
|
||||
return Path(root).joinpath(*parts[:-1], name)
|
||||
return preview_store.request_path(root, (*parts[:-1], name))
|
||||
|
||||
|
||||
def _request_key(root, parts):
|
||||
"""The flight key of the preview *parts* under *root*: the key of its
|
||||
disk path (so the serve and history routes share one flight)."""
|
||||
path = preview_store.request_path(root, parts)
|
||||
if path is None:
|
||||
return flight_key(root, parts)
|
||||
return _dst_key(path)
|
||||
|
||||
|
||||
def _quick_source(root, parts):
|
||||
@@ -254,12 +298,14 @@ def _thumbnail_source(root, parts):
|
||||
if in_trash_below_roots(found, [root]):
|
||||
return None
|
||||
rel = Path(*folder_parts, found.name)
|
||||
dst = root / media_thumbnail_rel(rel)
|
||||
dst = media_preview_path(root, rel, create=True)
|
||||
if dst is None:
|
||||
# Nowhere to write (the thumbnail folder is unavailable): the
|
||||
# original is served and nothing is written.
|
||||
return found, None, found.suffix.lower() in VIDEO_EXTENSIONS
|
||||
if dst != _current_preview(root, parts):
|
||||
return None
|
||||
if not _is_within(root / "thumbnails", root) or not _is_within(
|
||||
dst, root / "thumbnails"
|
||||
):
|
||||
if not preview_store.owns(root, dst):
|
||||
dst = None
|
||||
return found, dst, found.suffix.lower() in VIDEO_EXTENSIONS
|
||||
|
||||
@@ -279,6 +325,21 @@ def _scan_folder(folder, base, extensions):
|
||||
return None
|
||||
|
||||
|
||||
def _deferred(roots, src, dst):
|
||||
"""The ``DeferredPreview`` of a planned ``(src, dst)``: the root whose
|
||||
store the plan wrote *dst* into (nested roots plan one each); *dst* as
|
||||
planned when none matches."""
|
||||
src = Path(src)
|
||||
for root in roots:
|
||||
root = Path(root)
|
||||
if not src.is_relative_to(root):
|
||||
continue
|
||||
rel = src.relative_to(root)
|
||||
if media_preview_path(root, rel) == Path(dst):
|
||||
return DeferredPreview(root, rel, strict=False)
|
||||
return dst
|
||||
|
||||
|
||||
def _stop_stale_workers():
|
||||
"""Stop capture workers a previous import of this module started."""
|
||||
for thread in threading.enumerate():
|
||||
@@ -294,7 +355,7 @@ class AutoThumbnailer:
|
||||
``generate(src, dst, is_video, *, stop, guard, track)`` writes one
|
||||
preview and returns ``_generate_one``'s result dict; ``may_write()``
|
||||
says whether automatic previews may be written now. Views call
|
||||
``ensure`` on the view pool, the monitor calls ``enqueue`` from its
|
||||
``ensure`` on the image pool, the monitor calls ``enqueue`` from its
|
||||
thread.
|
||||
"""
|
||||
|
||||
@@ -420,16 +481,23 @@ class AutoThumbnailer:
|
||||
|
||||
def enqueue(self, src, dst, is_video):
|
||||
"""Queue *dst*'s preview for the worker; False when already queued,
|
||||
full, closed or its worker is refused at interpreter exit. Never
|
||||
blocks on a write."""
|
||||
key = _dst_key(dst)
|
||||
full, closed, nowhere to write it or its worker is refused at
|
||||
interpreter exit. Never blocks on a write. *dst* may be a
|
||||
``DeferredPreview``: its path is resolved again when the write
|
||||
starts."""
|
||||
now = _resolved(dst)
|
||||
if now is None:
|
||||
return False
|
||||
key = _dst_key(now)
|
||||
if not isinstance(dst, DeferredPreview):
|
||||
dst = now
|
||||
with self._changed:
|
||||
if self._closed or key in self._queued:
|
||||
return False
|
||||
if len(self._queue) >= self._max_queued:
|
||||
return False
|
||||
self._queued.add(key)
|
||||
self._queue.append((key, Path(src), Path(dst), bool(is_video)))
|
||||
self._queue.append((key, Path(src), dst, bool(is_video)))
|
||||
self._busy = True
|
||||
started = True
|
||||
if self._start_worker and not self._worker_running:
|
||||
@@ -491,12 +559,15 @@ class AutoThumbnailer:
|
||||
self._worker_running = False
|
||||
self._changed.notify_all()
|
||||
return
|
||||
take = min(len(self._queue), max(1, int(self._workers() or 1)))
|
||||
workers = max(1, int(self._workers() or 1))
|
||||
take = min(len(self._queue), workers)
|
||||
batch = [self._queue.popleft() for _ in range(take)]
|
||||
self._batch = len(batch)
|
||||
self._changed.notify_all()
|
||||
try:
|
||||
map_parallel(self._write_queued, batch, len(batch))
|
||||
# The job pool's size is the setting; a smaller batch uses
|
||||
# fewer of its threads (#523: one long-lived pool).
|
||||
map_parallel(self._write_queued, batch, workers)
|
||||
finally:
|
||||
with self._changed:
|
||||
self._batch = 0
|
||||
@@ -506,6 +577,9 @@ class AutoThumbnailer:
|
||||
def _write_queued(self, item):
|
||||
_key, src, dst, is_video = item
|
||||
try:
|
||||
dst = _resolved(dst)
|
||||
if dst is None:
|
||||
return # nowhere to write it now: made on its first view
|
||||
self.ensure(src, dst, is_video)
|
||||
except Exception: # one file never stops the worker
|
||||
self._logger.warning(
|
||||
@@ -567,7 +641,7 @@ class _PreviewGate:
|
||||
"""Admission of view requests on the event loop (#249 I2).
|
||||
|
||||
Waiters are coroutines: at most ``limit`` previews are written at once
|
||||
on the view pool, one flight runs per key, and past ``MAX_WAITING``
|
||||
on the image pool, one flight runs per key, and past ``MAX_WAITING``
|
||||
waiting requests ``run`` answers ``OVER`` at once.
|
||||
"""
|
||||
|
||||
@@ -622,13 +696,17 @@ class _PreviewGate:
|
||||
flight.set_result(result)
|
||||
|
||||
|
||||
def _on_image_thread():
|
||||
return threading.current_thread().name.startswith(POOL_PREFIX + "_")
|
||||
|
||||
|
||||
class AutoThumbnailMixin:
|
||||
"""Automatic previews on the API: views, the monitor's queue, scans."""
|
||||
|
||||
_auto_thumbnailer = None
|
||||
_auto_thumbnails_shut = False
|
||||
_preview_gate_obj = None
|
||||
_preview_pool_obj = None
|
||||
_image_pool_obj = None
|
||||
_auto_thumbnailer_lock = threading.Lock()
|
||||
|
||||
def _auto_thumbnails(self):
|
||||
@@ -651,7 +729,7 @@ class AutoThumbnailMixin:
|
||||
self._auto_thumbnails_shut = True
|
||||
|
||||
def close_auto_thumbnails(self, final=False, timeout=None):
|
||||
"""Stop the capture worker and the view pool, and forget them.
|
||||
"""Stop the capture worker and the image pool, and forget them.
|
||||
|
||||
Bounded by ``timeout`` (see AutoThumbnailer.close). ``final`` (the
|
||||
package teardown) also keeps a late monitor event from starting a
|
||||
@@ -662,7 +740,7 @@ class AutoThumbnailMixin:
|
||||
if final:
|
||||
self._auto_thumbnails_shut = True
|
||||
thumbnailer, self._auto_thumbnailer = self._auto_thumbnailer, None
|
||||
pool, self._preview_pool_obj = self._preview_pool_obj, None
|
||||
pool, self._image_pool_obj = self._image_pool_obj, None
|
||||
self._preview_gate_obj = None
|
||||
closed = True
|
||||
if thumbnailer is not None:
|
||||
@@ -671,6 +749,8 @@ class AutoThumbnailMixin:
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
if final:
|
||||
shutdown_encoders() # kill JPEG XL / ffmpeg encoders still running
|
||||
# The thumbnail folder's probe (#320), bounded like the rest.
|
||||
closed = preview_store.stop_probe() and closed
|
||||
return closed
|
||||
|
||||
def _auto_thumbnail_failures(self):
|
||||
@@ -695,6 +775,9 @@ class AutoThumbnailMixin:
|
||||
return job
|
||||
if job.kind == THUMBNAILS_JOB_KIND and job.title == THUMBNAIL_CLEAR_TITLE:
|
||||
return job
|
||||
# Move thumbnails holds automatic previews for its whole run (D2A).
|
||||
if job.kind == THUMBNAILS_JOB_KIND and job.scope.get("mode") == "move":
|
||||
return job
|
||||
return None
|
||||
|
||||
# ── the view admission ────────────────────────────────────────────
|
||||
@@ -706,16 +789,67 @@ class AutoThumbnailMixin:
|
||||
gate = self._preview_gate_obj = _PreviewGate(ON_DEMAND_LIMIT, self.logger)
|
||||
return gate
|
||||
|
||||
def _preview_pool(self):
|
||||
with AutoThumbnailMixin._auto_thumbnailer_lock:
|
||||
if self._preview_pool_obj is None:
|
||||
self._preview_pool_obj = ThreadPoolExecutor(
|
||||
ON_DEMAND_LIMIT, thread_name_prefix=POOL_PREFIX
|
||||
)
|
||||
return self._preview_pool_obj
|
||||
def _image_pool(self):
|
||||
"""The interactive image pool (#523): IMAGE_POOL_SIZE threads for view
|
||||
previews (behind their gate), card facts and inspector reads, so
|
||||
image buffers never spread over the default executor's threads.
|
||||
|
||||
def _preview_pool_threads(self):
|
||||
pool = self._preview_pool_obj
|
||||
None after the package teardown (``close_auto_thumbnails(final=True)``):
|
||||
a late request then runs on the default executor (``run_in_executor``
|
||||
with None) and starts no thread the teardown no longer stops.
|
||||
"""
|
||||
with AutoThumbnailMixin._auto_thumbnailer_lock:
|
||||
if self._image_pool_obj is None and not self._auto_thumbnails_shut:
|
||||
self._image_pool_obj = ThreadPoolExecutor(
|
||||
IMAGE_POOL_SIZE, thread_name_prefix=POOL_PREFIX
|
||||
)
|
||||
return self._image_pool_obj
|
||||
|
||||
async def _run_image(self, func, *args, **kwargs):
|
||||
"""``_run_in_executor`` for interactive image work: the image pool.
|
||||
|
||||
After the teardown, or with the pool shut down in between (a test
|
||||
reset), the call runs on the default executor as before.
|
||||
"""
|
||||
call = functools.partial(func, *args, **kwargs)
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
future = loop.run_in_executor(self._image_pool(), call)
|
||||
except RuntimeError: # submit refused: the pool was shut down
|
||||
future = loop.run_in_executor(None, call)
|
||||
return await future
|
||||
|
||||
def _map_image(self, func, items):
|
||||
"""``func`` over *items* on the image pool, in order (blocking).
|
||||
|
||||
Called from an image pool thread, after the teardown, or with the
|
||||
pool shut down in between (its queued items are cancelled), the
|
||||
items it did not run run on the calling thread.
|
||||
"""
|
||||
items = list(items)
|
||||
pool = None if not items or _on_image_thread() else self._image_pool()
|
||||
if pool is None:
|
||||
return [func(item) for item in items]
|
||||
futures = []
|
||||
try:
|
||||
for item in items:
|
||||
futures.append(pool.submit(func, item))
|
||||
except RuntimeError:
|
||||
pass # shut down in between: the rest run here
|
||||
results = []
|
||||
for index, item in enumerate(items):
|
||||
future = futures[index] if index < len(futures) else None
|
||||
if future is None or future.cancelled():
|
||||
results.append(func(item))
|
||||
continue
|
||||
try:
|
||||
results.append(future.result())
|
||||
except CancelledError: # cancelled while this caller waited
|
||||
results.append(func(item))
|
||||
return results
|
||||
|
||||
def _image_pool_threads(self):
|
||||
pool = self._image_pool_obj
|
||||
return [] if pool is None else list(getattr(pool, "_threads", ()))
|
||||
|
||||
def _video_placeholder_bytes(self):
|
||||
@@ -730,7 +864,7 @@ class AutoThumbnailMixin:
|
||||
return await self._preview_gate().run(
|
||||
flight_key(Path(dst).parent, (Path(dst).name,)),
|
||||
functools.partial(self._ensure_view, src, dst, is_video),
|
||||
self._preview_pool(),
|
||||
self._image_pool(),
|
||||
)
|
||||
|
||||
def _ensure_view(self, src, dst, is_video):
|
||||
@@ -774,17 +908,17 @@ class AutoThumbnailMixin:
|
||||
The flight answers only the outcome (a history flight may be the one
|
||||
joined); the file it previews is derived here: the preview path is
|
||||
the request's own, and a fallback looks its source up again on the
|
||||
view pool (bounded by it, never the default executor).
|
||||
image pool (bounded by it, never the default executor).
|
||||
"""
|
||||
if _has_traversal(filepath) or not _is_preview_request(filepath):
|
||||
return None
|
||||
parts = tuple(filepath.split("/"))
|
||||
gate = self._preview_gate()
|
||||
pool = self._preview_pool()
|
||||
pool = self._image_pool()
|
||||
for root in output_dirs:
|
||||
root = Path(root)
|
||||
outcome = await gate.run(
|
||||
flight_key(root, parts),
|
||||
_request_key(root, parts),
|
||||
functools.partial(self._preview_on_demand, root, parts),
|
||||
pool,
|
||||
)
|
||||
@@ -797,7 +931,7 @@ class AutoThumbnailMixin:
|
||||
continue
|
||||
return self._preview_response(BUSY, quick[0], None, quick[1])
|
||||
dst = _current_preview(root, parts)
|
||||
if outcome == READY:
|
||||
if outcome == READY and dst is not None:
|
||||
return self._file_response(dst)
|
||||
located = await gate.loop.run_in_executor(
|
||||
pool, _thumbnail_source, root, parts
|
||||
@@ -820,17 +954,31 @@ class AutoThumbnailMixin:
|
||||
"""
|
||||
if self._auto_thumbnails_shut:
|
||||
return "skipped"
|
||||
located = self._capture_target(image_path)
|
||||
located = self._capture_source(image_path)
|
||||
if located is None:
|
||||
return "skipped"
|
||||
src, dst, is_video = located
|
||||
src, root, rel, is_video = located
|
||||
dst = DeferredPreview(root, rel).resolve()
|
||||
if dst is None:
|
||||
return "skipped"
|
||||
if _thumbnail_is_fresh(src, dst):
|
||||
return "fresh"
|
||||
if self._auto_thumbnails().enqueue(src, dst, is_video):
|
||||
if self._auto_thumbnails().enqueue(src, DeferredPreview(root, rel), is_video):
|
||||
return "queued"
|
||||
return "skipped"
|
||||
|
||||
def _capture_target(self, image_path):
|
||||
"""``(src, dst, is_video)`` of a new output's preview now, or None."""
|
||||
located = self._capture_source(image_path)
|
||||
if located is None:
|
||||
return None
|
||||
src, root, rel, is_video = located
|
||||
dst = DeferredPreview(root, rel).resolve()
|
||||
return None if dst is None else (src, dst, is_video)
|
||||
|
||||
def _capture_source(self, image_path):
|
||||
"""``(src, root, rel, is_video)`` of a media file under a current
|
||||
root (not a preview, a link or in a trash folder), or None."""
|
||||
path = Path(os.path.abspath(str(image_path)))
|
||||
if path.suffix.lower() not in MEDIA_EXTENSIONS or not _regular_file(path):
|
||||
return None
|
||||
@@ -843,12 +991,7 @@ class AutoThumbnailMixin:
|
||||
rel = Path(os.path.relpath(path, root))
|
||||
if "thumbnails" in rel.parts[:-1] or in_trash_below_roots(path, [root]):
|
||||
return None
|
||||
dst = root / media_thumbnail_rel(rel)
|
||||
if not _is_within(root / "thumbnails", root) or not _is_within(
|
||||
dst, root / "thumbnails"
|
||||
):
|
||||
return None
|
||||
return path, dst, path.suffix.lower() in VIDEO_EXTENSIONS
|
||||
return path, root, rel, path.suffix.lower() in VIDEO_EXTENSIONS
|
||||
return None
|
||||
|
||||
# ── after a scan ──────────────────────────────────────────────────
|
||||
@@ -890,7 +1033,7 @@ class AutoThumbnailMixin:
|
||||
return "none"
|
||||
thumbnailer = self._auto_thumbnails()
|
||||
for src, dst, is_video in plan["targets"]:
|
||||
thumbnailer.enqueue(src, dst, is_video)
|
||||
thumbnailer.enqueue(src, _deferred(output_dirs, src, dst), is_video)
|
||||
self.logger.info(f"{missing:,} previews missing after the scan; queued")
|
||||
return "queued"
|
||||
|
||||
|
||||
@@ -153,6 +153,7 @@ class ThumbnailConvertMixin:
|
||||
job.log_line("Conversion failed · internal error")
|
||||
state = "failed"
|
||||
job.finish(state)
|
||||
await self._release_job_memory()
|
||||
|
||||
async def _convert_job_body(self, job, options):
|
||||
output_dirs = await self._run_in_executor(self._get_all_output_dirs)
|
||||
|
||||
@@ -46,7 +46,7 @@ from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from . import thumbnail_settings
|
||||
from . import preview_store, thumbnail_settings
|
||||
|
||||
try:
|
||||
from ...utils.logging_config import get_logger
|
||||
@@ -101,14 +101,20 @@ def temp_sibling(dst, tag="part"):
|
||||
|
||||
|
||||
def replace_unless_held(tmp, dst, guard=None):
|
||||
"""Rename *tmp* over *dst*, unless ``guard()`` is false (PreviewWriteHeld)."""
|
||||
if guard is not None and not guard():
|
||||
raise PreviewWriteHeld(PREVIEW_HELD)
|
||||
os.replace(tmp, dst)
|
||||
"""Rename *tmp* over *dst*, unless ``guard()`` is false (PreviewWriteHeld).
|
||||
|
||||
The check and the rename run under ``preview_store.write_lock`` (#320): a
|
||||
guard from ``preview_store.write_guard`` also refuses a write that started
|
||||
before the thumbnail location changed."""
|
||||
with preview_store.committed(guard) as allowed:
|
||||
if not allowed:
|
||||
raise PreviewWriteHeld(PREVIEW_HELD)
|
||||
os.replace(tmp, dst)
|
||||
|
||||
|
||||
# ``temp_sibling``'s names: ``.<stem>.<tag>-<pid>-<thread id><ext>``.
|
||||
_TEMP_NAME = re.compile(r"^\..+\.(?:part|src)-(\d+)-\d+\.[A-Za-z0-9]+$")
|
||||
_MOVE_TEMP_NAME = re.compile(r"\.pm-move-[0-9a-f]{8}\.tmp$")
|
||||
# A temp at least this old (seconds) outlived its write: no encode runs longer.
|
||||
STRAY_TEMP_AGE = 4 * ENCODE_TIMEOUT
|
||||
# When this process started (this module's import): a temp carrying its PID
|
||||
@@ -133,7 +139,9 @@ def sweep_stray_temps(folder, min_age=None):
|
||||
for root, _dirs, files in os.walk(folder):
|
||||
for name in files:
|
||||
match = _TEMP_NAME.match(name)
|
||||
if match is None:
|
||||
# A move's copy temp (#320): ``<name>.pm-move-<8 hex>.tmp``.
|
||||
move_temp = match is None and _MOVE_TEMP_NAME.search(name)
|
||||
if match is None and not move_temp:
|
||||
continue
|
||||
path = Path(root) / name
|
||||
try:
|
||||
@@ -142,7 +150,13 @@ def sweep_stray_temps(folder, min_age=None):
|
||||
continue
|
||||
if min_age and (
|
||||
now - stat.st_mtime < min_age
|
||||
or (match.group(1) == own and stat.st_mtime >= PROCESS_STARTED)
|
||||
or move_temp
|
||||
and stat.st_mtime >= PROCESS_STARTED
|
||||
or (
|
||||
match is not None
|
||||
and match.group(1) == own
|
||||
and stat.st_mtime >= PROCESS_STARTED
|
||||
)
|
||||
):
|
||||
continue
|
||||
path.unlink()
|
||||
|
||||
@@ -49,6 +49,7 @@ from .images import (
|
||||
_worker_threads,
|
||||
)
|
||||
from .job_routes import busy_response
|
||||
from .thumbnail_location import location_refusal
|
||||
from .jobs import THUMBNAIL_GROUP_NAMES, THUMBNAILS_JOB_KIND, JobBusy
|
||||
from .thumbnail_convert import CONVERT_JOB_TITLE
|
||||
from .thumbnail_encode import writer_settings
|
||||
@@ -131,7 +132,10 @@ class ThumbnailJobsMixin:
|
||||
"""The thumbnail plan, the thumbnail job and the legacy stream on it."""
|
||||
|
||||
async def get_thumbnail_plan(self, request):
|
||||
"""What a thumbnail run would do (read-only; capped like a run)."""
|
||||
"""What a thumbnail run would do (read-only; capped like a run).
|
||||
``mode=prune`` lists the removed sources' thumbnails (#320)."""
|
||||
if request.query.get("mode") == "prune":
|
||||
return await self.get_prune_plan()
|
||||
options, error = thumbnail_options(request.query, from_query=True)
|
||||
if error:
|
||||
return _invalid(error)
|
||||
@@ -179,15 +183,39 @@ class ThumbnailJobsMixin:
|
||||
)
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return _invalid(_OPTIONS_ERROR)
|
||||
if isinstance(body, dict) and body.get("mode") in ("move", "prune"):
|
||||
return await self._start_location_job(body)
|
||||
options, error = thumbnail_options(body)
|
||||
if error:
|
||||
return _invalid(error)
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
try:
|
||||
job = self._start_thumbnail_job(options)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
async def _start_location_job(self, body):
|
||||
"""Move thumbnails or Remove thumbnails of removed sources (#320)."""
|
||||
if body["mode"] == "prune":
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
try:
|
||||
if body["mode"] == "move":
|
||||
started = self.start_thumbnail_move(body)
|
||||
else:
|
||||
started = self.start_thumbnail_prune()
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
if isinstance(started, web.Response):
|
||||
return started
|
||||
return web.json_response(
|
||||
{"success": True, "job": started.summary()}, status=202
|
||||
)
|
||||
|
||||
def _start_thumbnail_job(self, options):
|
||||
"""Register the thumbnail job synchronously; raises JobBusy.
|
||||
|
||||
@@ -226,6 +254,7 @@ class ThumbnailJobsMixin:
|
||||
)
|
||||
state = "failed"
|
||||
job.finish(state)
|
||||
await self._release_job_memory()
|
||||
|
||||
async def _thumbnail_job_body(self, job, options):
|
||||
"""Plan, then generate in worker batches; returns the final state.
|
||||
@@ -458,8 +487,12 @@ class ThumbnailJobsMixin:
|
||||
or attaches to the running thumbnail job, and streams its frames. A
|
||||
legacy hold (the blocking generate or clear) or a record-changing
|
||||
job refuses it with the stream's error frame. The job goes on when
|
||||
the client goes away.
|
||||
the client goes away. Refused (409) while the thumbnail folder is
|
||||
unavailable (#320).
|
||||
"""
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
response = web.StreamResponse(
|
||||
status=200,
|
||||
headers={
|
||||
@@ -506,8 +539,12 @@ class ThumbnailJobsMixin:
|
||||
the way.
|
||||
"""
|
||||
running = self._jobs.active(THUMBNAILS_JOB_KIND)
|
||||
if running is not None and running.scope.get("mode") == "convert":
|
||||
return None, False, running # no legacy frames to follow (#317)
|
||||
if running is not None and running.scope.get("mode") in (
|
||||
"convert",
|
||||
"move",
|
||||
"prune",
|
||||
):
|
||||
return None, False, running # no legacy frames to follow (#317, #320)
|
||||
if running is not None and running.cancellable:
|
||||
return running, False, None
|
||||
busy = self._jobs.blocker(THUMBNAILS_JOB_KIND, admission="guarded")
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
"""The thumbnail location on the API (#320): startup, the settings payload
|
||||
and the refusals while the thumbnail folder is unavailable.
|
||||
|
||||
The setting itself is config.json ``thumbnails.location``
|
||||
(``preview_location``); the runtime state is ``preview_store``. POST
|
||||
``/settings`` never changes it (409 ``location-needs-move``): the Move
|
||||
thumbnails job is its only writer.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import preview_location, preview_store
|
||||
|
||||
try:
|
||||
from ...utils import library_location
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils import library_location
|
||||
|
||||
# The node folder (its web/ is served and a reinstall wipes it).
|
||||
NODE_FOLDER = os.path.dirname(
|
||||
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
)
|
||||
|
||||
|
||||
def default_path():
|
||||
"""``<data folder>/data/thumbnails`` (D7B), or None when unknown."""
|
||||
try:
|
||||
return library_location.cache_dir("thumbnails")
|
||||
except Exception: # no data folder can be decided: no default
|
||||
return None
|
||||
|
||||
|
||||
def location_refusal(location=None):
|
||||
"""The 409 ``location-unavailable`` response while the thumbnail folder
|
||||
is unavailable, else None (thumbnail generate, convert, clear, prune)."""
|
||||
location = location or preview_store.current()
|
||||
if location.mode != preview_store.FOLDER_MODE or location.available:
|
||||
return None
|
||||
status, text = preview_location.message("location-unavailable")
|
||||
return web.json_response(
|
||||
{"success": False, "code": "location-unavailable", "error": text},
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
def needs_move_response():
|
||||
status, text = preview_location.message("location-needs-move")
|
||||
return web.json_response(
|
||||
{"success": False, "code": "location-needs-move", "error": text},
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
class ThumbnailLocationMixin:
|
||||
"""Startup check and payloads of the thumbnail location."""
|
||||
|
||||
def _all_source_roots(self):
|
||||
"""Every image source of every root list: the Images roots, the
|
||||
configured sources and the extra gallery roots (as strings)."""
|
||||
from ..config import GalleryConfig, _extra_gallery_roots
|
||||
|
||||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||||
roots += [str(root) for root in GalleryConfig.MONITORING_DIRECTORIES]
|
||||
roots += list(_extra_gallery_roots())
|
||||
return tuple(dict.fromkeys(root for root in roots if root))
|
||||
|
||||
def _thumbnail_location_context(self):
|
||||
"""What a thumbnail folder is validated against (blocking)."""
|
||||
contained = []
|
||||
try:
|
||||
import folder_paths
|
||||
|
||||
for name in (
|
||||
"get_output_directory",
|
||||
"get_input_directory",
|
||||
"get_temp_directory",
|
||||
):
|
||||
getter = getattr(folder_paths, name, None)
|
||||
folder = getter() if getter is not None else None
|
||||
if folder:
|
||||
contained.append(str(folder))
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
contained.append(library_location.config_location()[1])
|
||||
except Exception: # no data folder can be decided
|
||||
self.logger.debug("No library folder for the thumbnail check")
|
||||
setting = preview_store.setting()
|
||||
return preview_location.Context(
|
||||
roots=self._all_source_roots(),
|
||||
contained=tuple(contained),
|
||||
node_folder=NODE_FOLDER,
|
||||
default_path=default_path(),
|
||||
current_path=(
|
||||
setting.path if setting.mode == preview_store.FOLDER_MODE else None
|
||||
),
|
||||
)
|
||||
|
||||
def use_preview_location(self):
|
||||
"""Put the configured location in force at once, unchecked (the
|
||||
package's import, before any request): folder mode is unavailable
|
||||
until ``start_preview_location`` has checked it, so nothing is ever
|
||||
written to the sources meanwhile. No filesystem access."""
|
||||
from ..config import PromptManagerConfig
|
||||
|
||||
try:
|
||||
return preview_store.configure(
|
||||
PromptManagerConfig.THUMBNAIL_LOCATION, check=False
|
||||
)
|
||||
except Exception:
|
||||
self.logger.warning("The thumbnail location is unusable", exc_info=True)
|
||||
preview_store.mark_invalid()
|
||||
return preview_store.current()
|
||||
|
||||
def start_preview_location(self):
|
||||
"""Parse and check the configured location once (startup thread);
|
||||
the probe takes over. Never raises; a check that fails leaves a
|
||||
folder location unavailable (``invalid``), never the sources."""
|
||||
from ..config import PromptManagerConfig
|
||||
|
||||
try:
|
||||
value = PromptManagerConfig.THUMBNAIL_LOCATION
|
||||
parsed = preview_location.parse_location(value)
|
||||
context = None
|
||||
if parsed.mode == preview_store.FOLDER_MODE:
|
||||
context = self._thumbnail_location_context()
|
||||
location = preview_store.configure(value, context=context)
|
||||
except Exception:
|
||||
self.logger.warning("The thumbnail location check failed", exc_info=True)
|
||||
if (
|
||||
preview_location.parse_location(
|
||||
PromptManagerConfig.THUMBNAIL_LOCATION
|
||||
).mode
|
||||
== preview_store.FOLDER_MODE
|
||||
):
|
||||
preview_store.mark_invalid()
|
||||
return preview_store.current()
|
||||
if location.mode == preview_store.FOLDER_MODE and not location.available:
|
||||
self.logger.warning(
|
||||
"The thumbnail folder is unavailable "
|
||||
f"({location.problem or 'checking'}): originals are shown and "
|
||||
"no thumbnails are written until it is back"
|
||||
)
|
||||
return location
|
||||
|
||||
def _thumbnail_location_payload(self, *, with_path):
|
||||
"""``{mode, available, problem}`` plus, for the settings GET only,
|
||||
``path`` and ``default_path`` (D6A)."""
|
||||
location = preview_store.current()
|
||||
payload = {
|
||||
"mode": location.mode,
|
||||
"available": location.available,
|
||||
"problem": location.problem,
|
||||
}
|
||||
if with_path:
|
||||
payload["path"] = location.path
|
||||
payload["default_path"] = default_path()
|
||||
# Old copies a move still has to remove (#320 Part 3).
|
||||
payload["cleanup_pending"] = len(preview_store.setting().cleanup)
|
||||
return payload
|
||||
@@ -0,0 +1,535 @@
|
||||
"""Move thumbnails (#320 Part 3): the reviewed job that changes where previews
|
||||
live, the removal of old copies after it, recovery at startup and the
|
||||
"Remove thumbnails of removed sources" scope (Part 5).
|
||||
|
||||
Kind ``thumbnails`` with ``mode: "move"`` (D1A): admission ``guarded``, so
|
||||
scans, deletes, tagging, source changes and every other thumbnail job wait
|
||||
for it, while the monitor's Needs review recording runs beside it.
|
||||
Automatic previews are held for the whole job (D2A, ``_auto_thumbnail_blocker``).
|
||||
|
||||
Phases: review (``POST /images/thumbnails/plan``, no writes but the
|
||||
validation probe), start (a plan token), hold (generation bump, journal,
|
||||
target folder), copy (cancellable at every file boundary, on the job pool
|
||||
in small chunks, D3A), switch (one config commit sets the location and
|
||||
records the old stores under ``cleanup``; not cancellable from here),
|
||||
remove old copies, end. Per-file errors switch anyway (D10A); a folder-wide
|
||||
error or a cancel before the switch removes what the job wrote and changes
|
||||
nothing. The move is the only writer of the setting (D4B).
|
||||
|
||||
Moving away from a thumbnail folder also removes the previews of this
|
||||
install's sources that are no longer listed (Q320-8: remove).
|
||||
"""
|
||||
|
||||
import functools
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import preview_location, preview_store
|
||||
from . import thumbnail_move_files as files
|
||||
from .jobs import THUMBNAILS_JOB_KIND
|
||||
from .thumbnail_move_plan import (
|
||||
MOVE_GROUPS,
|
||||
MOVE_TITLE,
|
||||
PLAN_TTL,
|
||||
_number,
|
||||
_plural,
|
||||
_remove,
|
||||
_roots_revision,
|
||||
_volume,
|
||||
_is_network,
|
||||
_write_json,
|
||||
)
|
||||
from . import thumbnail_move_plan as move_plan
|
||||
|
||||
try:
|
||||
from ...utils import exit_signal
|
||||
from ...utils.parallel import map_parallel
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils import exit_signal
|
||||
from utils.parallel import map_parallel
|
||||
|
||||
|
||||
class ThumbnailMoveMixin:
|
||||
"""Start and run the move job; the removal of old copies."""
|
||||
|
||||
# ── start ─────────────────────────────────────────────────────────
|
||||
|
||||
def _take_move_plan(self, body):
|
||||
"""The plan a start names, or ``(status, code)``."""
|
||||
token = body.get("token")
|
||||
plan = (self._move_plans or {}).get(token) if isinstance(token, str) else None
|
||||
if plan is None or time.monotonic() - plan.created > PLAN_TTL:
|
||||
return 409, "plan-stale"
|
||||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||||
if _roots_revision(roots) != plan.revision:
|
||||
return 409, "plan-stale"
|
||||
if preview_store.setting() != plan.old:
|
||||
return 409, "plan-stale"
|
||||
if plan.confirm_required and body.get("confirmed") is not True:
|
||||
return 400, "confirmation-required"
|
||||
return plan
|
||||
|
||||
def start_thumbnail_move(self, body):
|
||||
"""Start the move job (sync, on the loop); raises JobBusy. Returns
|
||||
the job or a refusal response."""
|
||||
if not self._move_recovered:
|
||||
blocker = self._jobs.active(THUMBNAILS_JOB_KIND)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "busy",
|
||||
"error": "Prompt Manager is still checking an earlier move. "
|
||||
"Try again in a moment.",
|
||||
"running": None if blocker is None else {"id": blocker.id},
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
taken = self._take_move_plan(body)
|
||||
if isinstance(taken, tuple):
|
||||
status, code = taken
|
||||
text = {
|
||||
"plan-stale": "The review is out of date. Review the move again.",
|
||||
"confirmation-required": "Confirm that Prompt Manager may "
|
||||
"create its folders there.",
|
||||
}[code]
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": text}, status=status
|
||||
)
|
||||
plan = taken
|
||||
self._move_plans.pop(plan.token, None)
|
||||
return self._jobs.start(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
MOVE_TITLE,
|
||||
admission="guarded",
|
||||
cancellable=True,
|
||||
scope={
|
||||
"mode": "move",
|
||||
"from": plan.old.mode,
|
||||
"to": plan.target["mode"],
|
||||
"to_path": plan.target["path"],
|
||||
},
|
||||
groups=MOVE_GROUPS,
|
||||
runner=functools.partial(self._run_thumbnail_move, plan=plan),
|
||||
)
|
||||
|
||||
# ── the job ───────────────────────────────────────────────────────
|
||||
|
||||
async def _run_thumbnail_move(self, job, *, plan):
|
||||
state = {"written": [], "created": [], "created_folder": False}
|
||||
try:
|
||||
await self._run_in_executor(self._move_hold, job, plan, state)
|
||||
except (OSError, ValueError) as error:
|
||||
await self._run_in_executor(self._move_rollback, plan, state)
|
||||
self._end_move(
|
||||
job, "failed", "The new folder could not be made; nothing changed."
|
||||
)
|
||||
self.logger.warning(f"Move thumbnails could not start: {error}")
|
||||
return
|
||||
left = [
|
||||
{
|
||||
"path": "/".join(item),
|
||||
"root": s.label,
|
||||
"detail": "Not a thumbnail; left in place",
|
||||
}
|
||||
for s in plan.sources
|
||||
if s.scan is not None
|
||||
for item in s.scan.left
|
||||
]
|
||||
if left:
|
||||
job.record_counted("left", len(left), left)
|
||||
items = [
|
||||
(source, item)
|
||||
for source in plan.sources
|
||||
if source.scan is not None
|
||||
for item in source.scan.copy
|
||||
]
|
||||
total = len(items)
|
||||
job.update(total=total, processed=0, progress=0)
|
||||
chunk = self._move_chunk(plan)
|
||||
workers = self._move_workers()
|
||||
done = 0
|
||||
for start in range(0, total, chunk):
|
||||
if job.cancelled:
|
||||
return await self._move_cancelled(job, plan, state)
|
||||
batch = items[start : start + chunk]
|
||||
try:
|
||||
results = await self._run_in_executor(
|
||||
map_parallel,
|
||||
functools.partial(self._move_copy_one, state=state),
|
||||
batch,
|
||||
workers,
|
||||
)
|
||||
except exit_signal.ExitRequested:
|
||||
return # exiting: the next start undoes the copies (journal)
|
||||
except files.FolderGone as error:
|
||||
self.logger.warning(f"Move thumbnails stopped: {error}")
|
||||
await self._run_in_executor(self._move_rollback, plan, state)
|
||||
self._end_move(
|
||||
job,
|
||||
"failed",
|
||||
"The new folder could not be reached or is full; nothing changed.",
|
||||
)
|
||||
return
|
||||
for (source, item), (outcome, detail) in zip(batch, results):
|
||||
row = {
|
||||
"path": "/".join(item[0]),
|
||||
"root": source.label,
|
||||
"detail": detail,
|
||||
}
|
||||
job.record(
|
||||
"moved" if outcome == "moved" else outcome,
|
||||
row,
|
||||
f"{outcome}: {source.label} · {'/'.join(item[0])}",
|
||||
)
|
||||
done += len(batch)
|
||||
label = batch[-1][0].label
|
||||
job.update(
|
||||
processed=done,
|
||||
progress=int(done * 90 / total) if total else 90,
|
||||
status_line=(
|
||||
f"Copying {_number(done)} of {_number(total)} previews · {label}"
|
||||
),
|
||||
)
|
||||
job.publish({"type": "progress", "summary": job.summary()})
|
||||
if job.cancelled:
|
||||
return await self._move_cancelled(job, plan, state)
|
||||
# The switch: not cancellable from here (an exit still stops between
|
||||
# files and leaves the cleanup to the next start).
|
||||
job.cancellable = False
|
||||
job.update(status_line="Switching to the new location", progress=92)
|
||||
job.publish({"type": "progress", "summary": job.summary()})
|
||||
try:
|
||||
await self._run_in_executor(self._move_switch, plan)
|
||||
except Exception:
|
||||
self.logger.exception("Move thumbnails could not switch")
|
||||
await self._run_in_executor(self._move_rollback, plan, state)
|
||||
self._end_move(
|
||||
job, "failed", "The setting could not be saved; nothing changed."
|
||||
)
|
||||
return
|
||||
await self._remove_old_copies(job, plan)
|
||||
moved = job.counters.get("moved", 0) + job.counters.get("skipped", 0)
|
||||
errors = job.counters.get("errors", 0)
|
||||
where = plan.target["path"] or "inside each source"
|
||||
parts = [f"{_number(moved)} moved"]
|
||||
if errors:
|
||||
parts.append(_plural(errors, "error"))
|
||||
parts.append(
|
||||
"old copies removed"
|
||||
if not job.scope.get("cleanup_left")
|
||||
else "some old copies are removed at the next start"
|
||||
)
|
||||
self._end_move(
|
||||
job, "finished", f"Thumbnails moved to {where}: " + " · ".join(parts) + "."
|
||||
)
|
||||
|
||||
def _end_move(self, job, state, line):
|
||||
job.update(status_line=line, progress=100)
|
||||
job.log_line(line)
|
||||
job.publish({"type": "complete", "summary": job.summary()})
|
||||
job.finish(state)
|
||||
|
||||
def _move_workers(self):
|
||||
from .images import _worker_threads
|
||||
|
||||
return max(1, int(_worker_threads() or 1))
|
||||
|
||||
def _move_chunk(self, plan):
|
||||
"""2 files at a time across volumes or to/from a share, else 4, never
|
||||
more than Worker threads (D3A: the pool keeps its size)."""
|
||||
sides = [plan.target["path"] or ""] + [s.old_store or "" for s in plan.sources]
|
||||
sides += [s.new_store for s in plan.sources]
|
||||
network = any(_is_network(side) for side in sides if side)
|
||||
volumes = {_volume(side) for side in sides if side}
|
||||
size = 2 if network or len(volumes) > 1 else 4
|
||||
return max(1, min(size, self._move_workers()))
|
||||
|
||||
def _move_hold(self, job, plan, state):
|
||||
"""Phase 3: bump the generation, write the journal, make the target."""
|
||||
preview_store.bump_generation()
|
||||
if plan.to_folder:
|
||||
folder = plan.target["path"]
|
||||
if not os.path.isdir(folder):
|
||||
if plan.target_default:
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
else:
|
||||
os.mkdir(folder)
|
||||
state["created_folder"] = True
|
||||
plan.target_id = preview_location.init_folder(
|
||||
folder, created_by_pm=state["created_folder"]
|
||||
)
|
||||
install = plan.install
|
||||
|
||||
def add_keys(marker):
|
||||
for source in plan.sources:
|
||||
if source.key and source.key not in marker["keys"]:
|
||||
marker["keys"][source.key] = {
|
||||
"path": source.root,
|
||||
"identity": os.path.normcase(os.path.realpath(source.root)),
|
||||
"dev_ino": preview_location.file_identity(source.root),
|
||||
"install": install,
|
||||
}
|
||||
|
||||
with preview_store.write_lock:
|
||||
preview_location.write_marker(folder, add_keys)
|
||||
_write_json(
|
||||
move_plan.journal_path(),
|
||||
{
|
||||
"state": "copying",
|
||||
"from": plan.old.to_config(),
|
||||
"to": plan.target,
|
||||
"stores": [s.new_store for s in plan.sources],
|
||||
# Stores that do not exist yet are made by this job only:
|
||||
# after a crash everything in them goes (B4).
|
||||
"created_dirs": [
|
||||
s.new_store for s in plan.sources if not os.path.isdir(s.new_store)
|
||||
],
|
||||
"created_folder": state["created_folder"],
|
||||
"install": plan.install,
|
||||
},
|
||||
)
|
||||
job.log_line(
|
||||
f"Move thumbnails started: {plan.old.path or 'inside each source'} → "
|
||||
f"{plan.target['path'] or 'inside each source'}"
|
||||
)
|
||||
|
||||
def _move_copy_one(self, pair, *, state):
|
||||
"""Copy one preview (a pool thread); ``(group, detail)``."""
|
||||
source, (relative, size, mtime_ns) = pair
|
||||
destination = Path(source.new_store).joinpath(*relative)
|
||||
try:
|
||||
if not os.path.isdir(destination.parent):
|
||||
base = Path(source.new_store)
|
||||
if not base.is_dir():
|
||||
if plan_parent_missing(base):
|
||||
raise files.FolderGone("the new folder is gone")
|
||||
created = []
|
||||
files._make_folders(base.parent, (base.name,), created)
|
||||
state["created"].extend(created)
|
||||
created = []
|
||||
files._make_folders(base, relative[:-1], created)
|
||||
state["created"].extend(created)
|
||||
outcome = files.copy_one(
|
||||
Path(source.old_store).joinpath(*relative),
|
||||
destination,
|
||||
size,
|
||||
mtime_ns,
|
||||
state["created"],
|
||||
)
|
||||
except files.CopyFailed as error:
|
||||
return "errors", str(error)
|
||||
if outcome == "moved":
|
||||
state["written"].append(str(destination))
|
||||
return "moved", "Copied and checked"
|
||||
return "skipped", "Already there"
|
||||
|
||||
async def _move_cancelled(self, job, plan, state):
|
||||
if exit_signal.requested():
|
||||
return # the next start removes the partial copies (journal)
|
||||
await self._run_in_executor(self._move_rollback, plan, state)
|
||||
self._end_move(job, "cancelled", "Move cancelled: nothing changed.")
|
||||
|
||||
def _move_rollback(self, plan, state):
|
||||
"""Remove what this job wrote (files, folders it made, a folder and
|
||||
marker it created) and the journal: nothing changed."""
|
||||
for path in state["written"]:
|
||||
files.remove_file(path)
|
||||
for folder in reversed(state["created"]):
|
||||
files.remove_empty_folders(folder)
|
||||
if plan.to_folder and state.get("created_folder"):
|
||||
folder = plan.target["path"]
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
names = None
|
||||
if names is not None and set(names) <= {preview_location.MARKER_NAME}:
|
||||
_remove(preview_location.marker_path(folder))
|
||||
files.remove_empty_folders(folder)
|
||||
_remove(move_plan.journal_path())
|
||||
|
||||
def _move_switch(self, plan):
|
||||
"""Phase 5: one config commit sets the location and records the old
|
||||
stores under ``cleanup``; the runtime follows under the write lock."""
|
||||
from ..config import PromptManagerConfig
|
||||
from ..config_store import get_config_store
|
||||
|
||||
entry = {
|
||||
"from": plan.old.mode,
|
||||
"folder": plan.old.path if plan.from_folder else None,
|
||||
"stores": [s.old_store for s in plan.sources if s.old_store],
|
||||
"drop_keys": self._old_keys_to_drop(plan),
|
||||
"install": plan.install,
|
||||
}
|
||||
# An older entry never removes a store the move just filled (a move
|
||||
# back while old copies still waited).
|
||||
filled = {os.path.normcase(s.new_store) for s in plan.sources}
|
||||
cleanup = []
|
||||
for old_entry in plan.old.cleanup:
|
||||
stores = [
|
||||
store
|
||||
for store in old_entry.get("stores") or []
|
||||
if isinstance(store, str) and os.path.normcase(store) not in filled
|
||||
]
|
||||
cleanup.append({**old_entry, "stores": stores})
|
||||
cleanup.append(entry)
|
||||
if plan.to_folder:
|
||||
value = {
|
||||
"mode": preview_store.FOLDER_MODE,
|
||||
"path": plan.target["path"],
|
||||
"id": plan.target_id,
|
||||
"install": plan.install,
|
||||
"cleanup": cleanup,
|
||||
}
|
||||
else:
|
||||
value = {
|
||||
"mode": preview_store.SOURCE_MODE,
|
||||
"install": plan.install,
|
||||
"cleanup": cleanup,
|
||||
}
|
||||
|
||||
def mutate(config):
|
||||
section = config.get("thumbnails")
|
||||
if not isinstance(section, dict):
|
||||
section = {}
|
||||
section["location"] = value
|
||||
config["thumbnails"] = section
|
||||
return config
|
||||
|
||||
def publish(_config):
|
||||
PromptManagerConfig.THUMBNAIL_LOCATION = value
|
||||
preview_store.configure(value, check=False)
|
||||
|
||||
get_config_store().commit(mutate, expected_revision=None, publish=publish)
|
||||
preview_store.refresh()
|
||||
_remove(move_plan.journal_path())
|
||||
|
||||
def _old_keys_to_drop(self, plan):
|
||||
if not plan.from_folder:
|
||||
return []
|
||||
keys = [s.old_store and os.path.basename(s.old_store) for s in plan.sources]
|
||||
keys = [key for key in keys if key]
|
||||
if plan.to_folder and os.path.normcase(plan.old.path or "") == os.path.normcase(
|
||||
plan.target["path"] or ""
|
||||
):
|
||||
return []
|
||||
return keys + [key for key, _store, _scan in plan.unlisted]
|
||||
|
||||
async def _remove_old_copies(self, job, plan):
|
||||
"""Phase 6: run the cleanup the switch recorded (and earlier ones)."""
|
||||
job.update(status_line="Removing old copies", progress=95)
|
||||
job.publish({"type": "progress", "summary": job.summary()})
|
||||
left = await self._run_in_executor(self._run_cleanup, job)
|
||||
if left:
|
||||
job.scope["cleanup_left"] = True
|
||||
|
||||
def _run_cleanup(self, job=None):
|
||||
"""Remove every recorded old store's previews, temps and empty
|
||||
folders; keep in ``cleanup`` what could not go. Returns how many
|
||||
entries are left (blocking)."""
|
||||
setting = preview_store.setting()
|
||||
remaining = []
|
||||
for entry in setting.cleanup:
|
||||
if not self._cleanup_entry(entry, job):
|
||||
remaining.append(entry)
|
||||
if list(setting.cleanup) != remaining:
|
||||
self._commit_cleanup(remaining)
|
||||
return len(remaining)
|
||||
|
||||
def _cleanup_entry(self, entry, job):
|
||||
"""One recorded removal; True when it is complete."""
|
||||
folder = entry.get("folder")
|
||||
stores = [
|
||||
store for store in entry.get("stores") or [] if isinstance(store, str)
|
||||
]
|
||||
if folder:
|
||||
if not os.path.isdir(folder):
|
||||
return False # the old folder is not reachable: kept for later
|
||||
folder_marker = preview_location.read_marker(folder) or {}
|
||||
keys = folder_marker.get("keys") or {}
|
||||
stores += [
|
||||
os.path.join(folder, key)
|
||||
for key in entry.get("drop_keys") or []
|
||||
if preview_location.valid_key(key) and key in keys
|
||||
]
|
||||
complete = True
|
||||
for store in dict.fromkeys(stores):
|
||||
if not self._remove_store(store, folder, job):
|
||||
complete = False
|
||||
if folder and complete:
|
||||
drop = set(entry.get("drop_keys") or [])
|
||||
|
||||
def forget(marker):
|
||||
for key in drop:
|
||||
marker["keys"].pop(key, None)
|
||||
|
||||
try:
|
||||
with preview_store.write_lock:
|
||||
marker = preview_location.write_marker(folder, forget)
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
if not marker["keys"] and marker.get("created_by_pm"):
|
||||
names = set(os.listdir(folder))
|
||||
rest = {n for n in names if not files.is_os_metadata(n)}
|
||||
if rest <= {preview_location.MARKER_NAME}:
|
||||
_remove(preview_location.marker_path(folder))
|
||||
files.remove_empty_folders(folder)
|
||||
return complete
|
||||
|
||||
def _remove_store(self, store, folder, job):
|
||||
"""Remove the previews and temps of one old store; True when done."""
|
||||
store = Path(store)
|
||||
if folder and not preview_location.valid_key(store.name):
|
||||
return True
|
||||
if not store.exists():
|
||||
return True
|
||||
if preview_location.is_link_like(store):
|
||||
return True # never through a link; nothing of ours there
|
||||
complete = True
|
||||
removed = 0
|
||||
for current, dirs, names in os.walk(store):
|
||||
dirs[:] = [
|
||||
d for d in dirs if not preview_location.is_link_like(Path(current) / d)
|
||||
]
|
||||
for name in names:
|
||||
path = Path(current) / name
|
||||
if files.is_temp_name(name) or files._preview_parts(name):
|
||||
if preview_location.is_link_like(path):
|
||||
continue
|
||||
if files.remove_file(path):
|
||||
removed += 1
|
||||
else:
|
||||
complete = False
|
||||
if job is not None:
|
||||
job.log_line(
|
||||
"In use by another program; removed at the next "
|
||||
f"start: {name}"
|
||||
)
|
||||
files.remove_empty_folders(store)
|
||||
if job is not None and removed:
|
||||
job.log_line(f"Removed {_number(removed)} old files from {store.name}")
|
||||
return complete
|
||||
|
||||
def _commit_cleanup(self, remaining):
|
||||
from ..config import PromptManagerConfig
|
||||
from ..config_store import get_config_store
|
||||
|
||||
def mutate(config):
|
||||
section = config.get("thumbnails")
|
||||
if isinstance(section, dict) and isinstance(section.get("location"), dict):
|
||||
section["location"]["cleanup"] = remaining
|
||||
return config
|
||||
|
||||
def publish(config):
|
||||
value = (config.get("thumbnails") or {}).get("location")
|
||||
PromptManagerConfig.THUMBNAIL_LOCATION = value
|
||||
preview_store.set_cleanup(remaining)
|
||||
|
||||
get_config_store().commit(mutate, expected_revision=None, publish=publish)
|
||||
|
||||
|
||||
def plan_parent_missing(store):
|
||||
"""A key folder's parent (the thumbnail folder) is gone mid-copy."""
|
||||
return not Path(store).parent.is_dir()
|
||||
@@ -0,0 +1,316 @@
|
||||
"""The file side of Move thumbnails (#320 Part 3): what is copied, how a
|
||||
copy is made and checked, and how old copies are removed.
|
||||
|
||||
No job, HTTP or config code lives here; ``thumbnail_move`` drives it.
|
||||
|
||||
Copy set (B5): each old store is walked with ``os.scandir``, never through a
|
||||
symlink, junction or reparse point (counted as "skipped link"). A regular
|
||||
file is
|
||||
|
||||
- copied when it has the preview grammar (``<name>_thumb<ext>``, any written
|
||||
or legacy extension) and its original still exists in the source;
|
||||
- removed with the old copies, not copied, when it is an orphan (no
|
||||
original), lies under a trash folder, or is a temp sibling of a write
|
||||
(``.<stem>.part-…``) or of a move (``….pm-move-<8 hex>.tmp``);
|
||||
- left in place otherwise ("Left in place"). OS metadata files (``Thumbs.db``,
|
||||
``desktop.ini``, ``.DS_Store``, ``._*``) are removed only when they alone
|
||||
keep a preview folder from being removed.
|
||||
|
||||
A copy goes to ``<name>.pm-move-<8 hex>.tmp`` beside its destination, its
|
||||
size is compared, its mtime is set to the source preview's rounded up to the
|
||||
next 2-second step (FAT/exFAT keep it fresh, a stale preview stays stale),
|
||||
then it is renamed into place. Every handle is closed before a rename or an
|
||||
unlink (Windows).
|
||||
"""
|
||||
|
||||
import errno
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import stat as stat_module
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from . import preview_location, thumbnail_settings
|
||||
from .thumbnail_encode import _TEMP_NAME
|
||||
|
||||
try:
|
||||
from ...utils.media_folders import in_trash_folder
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils.media_folders import in_trash_folder
|
||||
|
||||
MOVE_TEMP = re.compile(r"\.pm-move-[0-9a-f]{8}\.tmp$")
|
||||
MOVE_TEMP_SUFFIX = ".pm-move-{token}.tmp"
|
||||
# The temp name adds this much to a destination path (".pm-move-xxxxxxxx.tmp").
|
||||
TEMP_EXTRA = 21
|
||||
WINDOWS_FILE_LIMIT = 259
|
||||
MTIME_STEP_NS = 2_000_000_000
|
||||
# A file another program holds open (Windows) is retried this often.
|
||||
REMOVE_RETRIES = (0.5, 1.0, 2.0)
|
||||
_WINDOWS_IN_USE = {32, 33} # ERROR_SHARING_VIOLATION, ERROR_LOCK_VIOLATION
|
||||
|
||||
|
||||
def is_os_metadata(name):
|
||||
folded = name.lower()
|
||||
return folded in preview_location.OS_METADATA or folded.startswith("._")
|
||||
|
||||
|
||||
def is_temp_name(name):
|
||||
return bool(_TEMP_NAME.match(name) or MOVE_TEMP.search(name))
|
||||
|
||||
|
||||
def _preview_parts(name):
|
||||
"""``(base, extension)`` of a preview file name, or None."""
|
||||
stem, extension = os.path.splitext(name)
|
||||
suffix = thumbnail_settings.THUMB_SUFFIX
|
||||
if not stem.endswith(suffix) or len(stem) <= len(suffix):
|
||||
return None
|
||||
if extension.lower() not in thumbnail_settings.THUMBNAIL_EXTENSIONS:
|
||||
return None
|
||||
return stem[: -len(suffix)], extension
|
||||
|
||||
|
||||
def _has_original(root, rel_dir, base, extension):
|
||||
"""Whether a media file in ``<root>/<rel_dir>`` owns this preview."""
|
||||
folder = Path(root).joinpath(*rel_dir)
|
||||
try:
|
||||
with os.scandir(folder) as entries:
|
||||
for entry in entries:
|
||||
stem, source_ext = os.path.splitext(entry.name)
|
||||
if stem != base or not source_ext:
|
||||
continue
|
||||
if not thumbnail_settings.owns(extension, source_ext):
|
||||
continue
|
||||
if entry.is_file(follow_symlinks=False):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
@dataclass
|
||||
class StoreScan:
|
||||
"""One old store's files, sorted into what the move does with them."""
|
||||
|
||||
store: Path
|
||||
copy: list = field(default_factory=list) # (relative parts, size, mtime_ns)
|
||||
orphans: list = field(default_factory=list) # relative parts
|
||||
temps: list = field(default_factory=list)
|
||||
left: list = field(default_factory=list)
|
||||
links: int = 0
|
||||
bytes: int = 0
|
||||
|
||||
@property
|
||||
def removable(self):
|
||||
return [item[0] for item in self.copy] + self.orphans + self.temps
|
||||
|
||||
|
||||
def scan_store(store, root, *, stop=None):
|
||||
"""Sort the files of *store* (the previews of the source *root*).
|
||||
|
||||
A store that does not exist, or is itself a link, scans empty (a link
|
||||
counts once).
|
||||
"""
|
||||
store = Path(store)
|
||||
scan = StoreScan(store)
|
||||
try:
|
||||
if preview_location.is_link_like(store):
|
||||
scan.links += 1
|
||||
return scan
|
||||
if not store.is_dir():
|
||||
return scan
|
||||
except OSError:
|
||||
return scan
|
||||
_walk(scan, store, (), root, stop)
|
||||
return scan
|
||||
|
||||
|
||||
def _walk(scan, folder, rel_dir, root, stop):
|
||||
try:
|
||||
with os.scandir(folder) as iterator:
|
||||
entries = list(iterator)
|
||||
except OSError:
|
||||
return
|
||||
for entry in entries:
|
||||
if stop is not None and stop():
|
||||
return
|
||||
relative = rel_dir + (entry.name,)
|
||||
path = Path(entry.path)
|
||||
try:
|
||||
if preview_location.is_link_like(path):
|
||||
scan.links += 1
|
||||
continue
|
||||
info = entry.stat(follow_symlinks=False)
|
||||
except OSError:
|
||||
continue
|
||||
if stat_module.S_ISDIR(info.st_mode):
|
||||
_walk(scan, path, relative, root, stop)
|
||||
continue
|
||||
if not stat_module.S_ISREG(info.st_mode):
|
||||
scan.left.append(relative)
|
||||
continue
|
||||
if is_temp_name(entry.name):
|
||||
scan.temps.append(relative)
|
||||
continue
|
||||
if is_os_metadata(entry.name):
|
||||
continue # removed only when it alone keeps a folder (rmdir)
|
||||
parts = _preview_parts(entry.name)
|
||||
if parts is None:
|
||||
scan.left.append(relative)
|
||||
continue
|
||||
if in_trash_folder("/".join(relative)) or not _has_original(
|
||||
root, rel_dir, *parts
|
||||
):
|
||||
scan.orphans.append(relative)
|
||||
continue
|
||||
scan.copy.append((relative, info.st_size, info.st_mtime_ns))
|
||||
scan.bytes += info.st_size
|
||||
|
||||
|
||||
# ── copying ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def rounded_mtime_ns(mtime_ns):
|
||||
"""*mtime_ns* rounded up to the next 2-second step (FAT/exFAT)."""
|
||||
return -(-mtime_ns // MTIME_STEP_NS) * MTIME_STEP_NS
|
||||
|
||||
|
||||
def too_long(path, long_paths=None):
|
||||
"""A destination (with its temp name) over Windows' file limit."""
|
||||
if os.name != "nt" and long_paths is not False:
|
||||
return False
|
||||
if long_paths is None:
|
||||
long_paths = preview_location.long_paths_enabled()
|
||||
if long_paths:
|
||||
return False
|
||||
return len(str(path)) + TEMP_EXTRA > WINDOWS_FILE_LIMIT
|
||||
|
||||
|
||||
class CopyFailed(Exception):
|
||||
"""One preview could not be copied (a per-file error, D10A)."""
|
||||
|
||||
|
||||
class FolderGone(Exception):
|
||||
"""The destination cannot take writes any more (abort the move)."""
|
||||
|
||||
|
||||
def _make_folders(base, relative_dir, created):
|
||||
"""Make ``base/relative_dir`` one folder at a time (*base* must exist);
|
||||
every folder made is added to *created*."""
|
||||
current = Path(base)
|
||||
for part in relative_dir:
|
||||
current = current / part
|
||||
try:
|
||||
os.mkdir(current)
|
||||
created.append(str(current))
|
||||
except FileExistsError:
|
||||
pass
|
||||
if preview_location.is_link_like(current):
|
||||
raise CopyFailed("a link in the destination")
|
||||
|
||||
|
||||
def copy_one(source, destination, size, mtime_ns, created, *, long_paths=None):
|
||||
"""Copy one preview; returns ``"moved"`` or ``"skipped"`` (already
|
||||
there). Raises CopyFailed (this file) or FolderGone (the destination)."""
|
||||
destination = Path(destination)
|
||||
if too_long(destination, long_paths):
|
||||
raise CopyFailed("Path too long for Windows; the original is shown")
|
||||
wanted = rounded_mtime_ns(mtime_ns)
|
||||
try:
|
||||
existing = os.stat(destination, follow_symlinks=False)
|
||||
except FileNotFoundError:
|
||||
existing = None
|
||||
except OSError as error:
|
||||
raise FolderGone(str(error)) from error
|
||||
if (
|
||||
existing is not None
|
||||
and stat_module.S_ISREG(existing.st_mode)
|
||||
and existing.st_size == size
|
||||
and existing.st_mtime_ns == wanted
|
||||
):
|
||||
return "skipped"
|
||||
temp = destination.with_name(
|
||||
destination.name + MOVE_TEMP_SUFFIX.format(token=secrets.token_hex(4))
|
||||
)
|
||||
try:
|
||||
shutil.copyfile(source, temp)
|
||||
if os.stat(temp).st_size != size:
|
||||
raise CopyFailed("The copy did not match; made again on view")
|
||||
os.utime(temp, ns=(wanted, wanted))
|
||||
os.replace(temp, destination)
|
||||
except CopyFailed:
|
||||
_unlink(temp)
|
||||
raise
|
||||
except FileNotFoundError as error:
|
||||
_unlink(temp)
|
||||
if not Path(source).exists():
|
||||
raise CopyFailed("Could not be read; made again on view") from error
|
||||
raise FolderGone(str(error)) from error
|
||||
except OSError as error:
|
||||
_unlink(temp)
|
||||
if error.errno in (errno.ENOSPC, errno.EDQUOT, errno.EROFS):
|
||||
raise FolderGone(str(error)) from error
|
||||
raise CopyFailed("Could not be copied; made again on view") from error
|
||||
return "moved"
|
||||
|
||||
|
||||
def _unlink(path):
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# ── removing ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _in_use(error):
|
||||
return getattr(error, "winerror", None) in _WINDOWS_IN_USE
|
||||
|
||||
|
||||
def remove_file(path, *, sleep=time.sleep):
|
||||
"""Remove one file; a file another program holds open (Windows) is
|
||||
retried 3 times. Returns True when it is gone."""
|
||||
for wait in (0.0,) + REMOVE_RETRIES:
|
||||
if wait:
|
||||
sleep(wait)
|
||||
try:
|
||||
os.unlink(path)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return True
|
||||
except OSError as error:
|
||||
if not _in_use(error):
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def remove_empty_folders(top, *, keep_top=False):
|
||||
"""``rmdir`` the empty folders below *top* (and *top*), bottom-up, never
|
||||
through a link. A folder holding only OS metadata files loses them first.
|
||||
Returns the folders that could not be removed."""
|
||||
top = Path(top)
|
||||
left = []
|
||||
if not top.is_dir() or preview_location.is_link_like(top):
|
||||
return left
|
||||
for folder, dirs, _files in os.walk(top, topdown=False):
|
||||
folder = Path(folder)
|
||||
if folder == top and keep_top:
|
||||
continue
|
||||
if preview_location.is_link_like(folder):
|
||||
continue
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
left.append(folder)
|
||||
continue
|
||||
if names and all(is_os_metadata(name) for name in names):
|
||||
for name in names:
|
||||
remove_file(folder / name)
|
||||
try:
|
||||
os.rmdir(folder)
|
||||
except OSError:
|
||||
left.append(folder)
|
||||
return left
|
||||
@@ -0,0 +1,490 @@
|
||||
"""Move thumbnails, the review (#320 Part 3 phase 1): what a move would copy,
|
||||
remove and leave, checked before anything is written."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import preview_location, preview_store
|
||||
from . import thumbnail_move_files as files
|
||||
from .job_routes import busy_response
|
||||
from .jobs import THUMBNAILS_JOB_KIND, JobBusy
|
||||
from .thumbnail_location import default_path
|
||||
|
||||
try:
|
||||
from ...utils import library_location
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils import library_location
|
||||
|
||||
MOVE_TITLE = "Move thumbnails"
|
||||
FINISHING_TITLE = "Move thumbnails (finishing)"
|
||||
PRUNE_TITLE = "Remove thumbnails of removed sources"
|
||||
MOVE_GROUPS = ("moved", "skipped", "left", "errors")
|
||||
PRUNE_GROUPS = ("removed", "left", "errors")
|
||||
PLAN_TTL = 600.0
|
||||
JOURNAL_NAME = "thumbnail-move.json"
|
||||
SPACE_MARGIN = 64 * 2**20
|
||||
SPACE_SHARE = 0.05
|
||||
LISTED = 5
|
||||
SENTENCE = (
|
||||
"Every thumbnail is copied and checked before Prompt Manager switches. "
|
||||
"Scans, deletes and tagging wait until it finishes; closing this page does "
|
||||
"not stop it."
|
||||
)
|
||||
|
||||
|
||||
def _number(value):
|
||||
return f"{value:,}"
|
||||
|
||||
|
||||
def _plural(count, one, many=None):
|
||||
return f"{_number(count)} {one if count == 1 else (many or one + 's')}"
|
||||
|
||||
|
||||
def _size(value):
|
||||
size = float(value)
|
||||
for unit in ("B", "KB", "MB", "GB"):
|
||||
if size < 1024 or unit == "GB":
|
||||
return f"{size:.1f} {unit}" if unit != "B" else f"{int(size)} B"
|
||||
size /= 1024
|
||||
return f"{size:.1f} GB"
|
||||
|
||||
|
||||
def _refusal(code, status=None, **extra):
|
||||
known_status, text = preview_location.MESSAGES.get(code, (400, code))
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": text, **extra},
|
||||
status=status or known_status,
|
||||
)
|
||||
|
||||
|
||||
def journal_path():
|
||||
"""``<data folder>/data/thumbnail-move.json`` (only while a move copies)."""
|
||||
return library_location.cache_dir(JOURNAL_NAME)
|
||||
|
||||
|
||||
def _read_json(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def _write_json(path, data):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
temp = f"{path}.{os.getpid()}.tmp"
|
||||
with open(temp, "w", encoding="utf-8") as handle:
|
||||
json.dump(data, handle, indent=2)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temp, path)
|
||||
|
||||
|
||||
def _remove(path):
|
||||
try:
|
||||
os.unlink(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def _is_network(path):
|
||||
text = str(path)
|
||||
if text.startswith(("\\\\", "//")):
|
||||
return True
|
||||
if os.name == "nt":
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
drive = os.path.splitdrive(os.path.abspath(text))[0] + "\\"
|
||||
return ctypes.windll.kernel32.GetDriveTypeW(drive) == 4 # DRIVE_REMOTE
|
||||
except (AttributeError, OSError):
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _volume(path):
|
||||
current = Path(os.path.abspath(path))
|
||||
while not current.exists() and current.parent != current:
|
||||
current = current.parent
|
||||
try:
|
||||
return os.stat(current).st_dev
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _free(path):
|
||||
current = Path(os.path.abspath(path))
|
||||
while not current.exists() and current.parent != current:
|
||||
current = current.parent
|
||||
try:
|
||||
return shutil.disk_usage(current).free
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
# ── the plan ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourcePlan:
|
||||
"""One listed source: its old store, its new store and what moves."""
|
||||
|
||||
root: str
|
||||
label: str
|
||||
key: str | None
|
||||
old_store: str | None
|
||||
new_store: str
|
||||
scan: files.StoreScan | None
|
||||
available: bool = True
|
||||
too_long: list = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MovePlan:
|
||||
"""A reviewed move: from, to, the sources and the removals."""
|
||||
|
||||
token: str
|
||||
created: float
|
||||
revision: str
|
||||
old: preview_location.ParsedLocation
|
||||
target: dict # {"mode", "path"}
|
||||
target_id: str | None
|
||||
target_exists: bool
|
||||
target_default: bool
|
||||
install: str
|
||||
sources: list
|
||||
unlisted: list # (key, store path, scan)
|
||||
needs_bytes: int
|
||||
confirm_required: bool
|
||||
|
||||
@property
|
||||
def to_folder(self):
|
||||
return self.target["mode"] == preview_store.FOLDER_MODE
|
||||
|
||||
@property
|
||||
def from_folder(self):
|
||||
return self.old.mode == preview_store.FOLDER_MODE
|
||||
|
||||
|
||||
def _roots_revision(roots):
|
||||
text = json.dumps(sorted(str(root) for root in roots))
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
class ThumbnailMovePlanMixin:
|
||||
"""The move's review and its routes."""
|
||||
|
||||
_move_plans = None
|
||||
|
||||
# ── routes ────────────────────────────────────────────────────────
|
||||
|
||||
def _register_thumbnail_move_routes(self, routes):
|
||||
@routes.post("/prompt_manager/images/thumbnails/plan")
|
||||
async def thumbnail_move_plan_route(request):
|
||||
return await self.plan_thumbnail_move(request)
|
||||
|
||||
@routes.post("/prompt_manager/images/thumbnails/location/recreate")
|
||||
async def thumbnail_location_recreate_route(request):
|
||||
return await self.recreate_thumbnail_folder(request)
|
||||
|
||||
@routes.post("/prompt_manager/images/thumbnails/location/adopt")
|
||||
async def thumbnail_location_adopt_route(request):
|
||||
return await self.adopt_thumbnail_folder(request)
|
||||
|
||||
# ── review ────────────────────────────────────────────────────────
|
||||
|
||||
async def plan_thumbnail_move(self, request):
|
||||
try:
|
||||
body = await request.json()
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
body = None
|
||||
if not isinstance(body, dict) or body.get("mode") != "move":
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "invalid-request",
|
||||
"error": "mode must be move",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
target = body.get("location")
|
||||
if not isinstance(target, dict) or target.get("mode") not in (
|
||||
preview_store.SOURCE_MODE,
|
||||
preview_store.FOLDER_MODE,
|
||||
):
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "invalid-request",
|
||||
"error": "location must name a mode",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
blocker = self._jobs.blocker(THUMBNAILS_JOB_KIND, admission="guarded")
|
||||
if blocker is not None:
|
||||
return busy_response(JobBusy(blocker))
|
||||
try:
|
||||
outcome = await self._run_in_executor(self._build_move_plan, target)
|
||||
except Exception:
|
||||
self.logger.exception("The thumbnail move could not be planned")
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": None,
|
||||
"error": "The move could not be planned",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
if isinstance(outcome, str):
|
||||
return _refusal(outcome)
|
||||
plan, payload = outcome
|
||||
if payload.get("insufficient"):
|
||||
return _refusal(
|
||||
"insufficient-space",
|
||||
status=507,
|
||||
error_detail=payload["insufficient"],
|
||||
plan=payload,
|
||||
)
|
||||
self._store_move_plan(plan)
|
||||
return web.json_response({"success": True, **payload})
|
||||
|
||||
def _store_move_plan(self, plan):
|
||||
if self._move_plans is None:
|
||||
self._move_plans = {}
|
||||
now = time.monotonic()
|
||||
self._move_plans = {
|
||||
token: kept
|
||||
for token, kept in self._move_plans.items()
|
||||
if now - kept.created < PLAN_TTL
|
||||
}
|
||||
self._move_plans[plan.token] = plan
|
||||
|
||||
def _build_move_plan(self, target):
|
||||
"""``(plan, payload)`` or a refusal code (blocking)."""
|
||||
old = preview_store.setting()
|
||||
location = preview_store.current()
|
||||
install = old.install or str(uuid.uuid4())
|
||||
target_path = None
|
||||
default = default_path()
|
||||
if target["mode"] == preview_store.FOLDER_MODE:
|
||||
raw = target.get("path")
|
||||
if not isinstance(raw, str):
|
||||
return "location-not-absolute"
|
||||
context = self._thumbnail_location_context()
|
||||
code = preview_location.location_problem(
|
||||
raw, purpose="move", context=context
|
||||
)
|
||||
if code is not None:
|
||||
return code
|
||||
target_path = os.path.abspath(raw)
|
||||
elif old.mode == preview_store.SOURCE_MODE and not old.problem:
|
||||
return "location-same"
|
||||
target_default = bool(default) and os.path.normcase(
|
||||
target_path or ""
|
||||
) == os.path.normcase(os.path.abspath(default))
|
||||
target_exists = bool(target_path) and os.path.isdir(target_path)
|
||||
target_marker = (
|
||||
preview_location.read_marker(target_path) if target_exists else None
|
||||
)
|
||||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||||
labels = self._source_labels(roots)
|
||||
old_keys = {}
|
||||
if old.mode == preview_store.FOLDER_MODE and location.available:
|
||||
marker = preview_location.read_marker(old.path) or {}
|
||||
old_keys = marker.get("keys") or {}
|
||||
new_keys = dict((target_marker or {}).get("keys") or {})
|
||||
taken = {key: entry.get("identity") for key, entry in new_keys.items()}
|
||||
sources = []
|
||||
for root in roots:
|
||||
identity = os.path.normcase(os.path.realpath(root))
|
||||
dev_ino = preview_location.file_identity(root)
|
||||
old_key = None
|
||||
old_store = None
|
||||
available = True
|
||||
if old.mode == preview_store.SOURCE_MODE:
|
||||
old_store = str(preview_store.source_folder(root))
|
||||
elif location.available:
|
||||
old_key = preview_location.find_key(old_keys, root, identity, dev_ino)
|
||||
if old_key:
|
||||
old_store = os.path.join(old.path, old_key)
|
||||
else:
|
||||
available = False
|
||||
key = None
|
||||
if target_path:
|
||||
key = old_key or preview_location.find_key(
|
||||
new_keys, root, identity, dev_ino
|
||||
)
|
||||
if key is None or taken.get(key, identity) != identity:
|
||||
key = preview_location.source_key(root, identity, taken)
|
||||
taken[key] = identity
|
||||
new_store = os.path.join(target_path, key)
|
||||
else:
|
||||
new_store = str(preview_store.source_folder(root))
|
||||
scan = None
|
||||
if old_store:
|
||||
scan = files.scan_store(old_store, root)
|
||||
plan = SourcePlan(
|
||||
root, labels.get(root, root), key, old_store, new_store, scan, available
|
||||
)
|
||||
if scan is not None and target_path:
|
||||
plan.too_long = [
|
||||
item[0]
|
||||
for item in scan.copy
|
||||
if files.too_long(Path(new_store).joinpath(*item[0]))
|
||||
]
|
||||
sources.append(plan)
|
||||
unlisted = self._unlisted_keys(old, location, old_keys, install)
|
||||
needs = sum(source.scan.bytes for source in sources if source.scan is not None)
|
||||
plan = MovePlan(
|
||||
token=secrets.token_urlsafe(16),
|
||||
created=time.monotonic(),
|
||||
revision=_roots_revision(roots),
|
||||
old=old,
|
||||
target={"mode": target["mode"], "path": target_path},
|
||||
target_id=(target_marker or {}).get("id"),
|
||||
target_exists=target_exists,
|
||||
target_default=target_default,
|
||||
install=install,
|
||||
sources=sources,
|
||||
unlisted=unlisted,
|
||||
needs_bytes=needs,
|
||||
confirm_required=bool(target_path) and not target_default,
|
||||
)
|
||||
return plan, self._plan_payload(plan, location)
|
||||
|
||||
def _unlisted_keys(self, old, location, keys, install):
|
||||
"""This install's keys whose source no longer is in any root list
|
||||
(``(key, store, scan)``), in the folder in use."""
|
||||
if old.mode != preview_store.FOLDER_MODE or not location.available:
|
||||
return []
|
||||
listed = {
|
||||
os.path.normcase(os.path.realpath(root))
|
||||
for root in self._all_source_roots()
|
||||
}
|
||||
found = []
|
||||
for key, entry in sorted(keys.items()):
|
||||
if entry.get("install") not in (install, None):
|
||||
continue
|
||||
if entry.get("identity") in listed:
|
||||
continue
|
||||
store = os.path.join(old.path, key)
|
||||
found.append(
|
||||
(key, store, files.scan_store(store, entry.get("path") or store))
|
||||
)
|
||||
return found
|
||||
|
||||
def _source_labels(self, roots):
|
||||
from . import sources
|
||||
|
||||
labels = sources._comfyui_labels()
|
||||
return {root: sources.source_label(root, labels) for root in roots}
|
||||
|
||||
def _plan_payload(self, plan, location):
|
||||
sources = plan.sources
|
||||
copies = sum(len(s.scan.copy) for s in sources if s.scan)
|
||||
orphans = sum(len(s.scan.orphans) for s in sources if s.scan)
|
||||
temps = sum(len(s.scan.temps) for s in sources if s.scan)
|
||||
links = sum(s.scan.links for s in sources if s.scan)
|
||||
left = [
|
||||
f"{s.label} · {'/'.join(item)}"
|
||||
for s in sources
|
||||
if s.scan
|
||||
for item in s.scan.left
|
||||
]
|
||||
too_long = [
|
||||
f"{s.label} · {'/'.join(item)}" for s in sources for item in s.too_long
|
||||
]
|
||||
unlisted_count = sum(
|
||||
len(scan.copy) + len(scan.orphans) for _k, _s, scan in plan.unlisted
|
||||
)
|
||||
free = None
|
||||
insufficient = None
|
||||
if plan.needs_bytes:
|
||||
volumes = {}
|
||||
for source in sources:
|
||||
if source.scan is None or not source.scan.bytes:
|
||||
continue
|
||||
volume = _volume(source.new_store)
|
||||
entry = volumes.setdefault(volume, [source.new_store, 0])
|
||||
entry[1] += source.scan.bytes
|
||||
for where, needed in volumes.values():
|
||||
available = _free(where)
|
||||
if available is None:
|
||||
continue
|
||||
free = available if free is None else min(free, available)
|
||||
margin = max(SPACE_MARGIN, int(needed * SPACE_SHARE))
|
||||
if needed + margin > available:
|
||||
insufficient = {
|
||||
"needed": needed + margin,
|
||||
"free": available,
|
||||
"text": (
|
||||
f"There is not enough space for the thumbnails: "
|
||||
f"{_size(available)} free, {_size(needed + margin)} "
|
||||
"needed. Free some space or choose another folder."
|
||||
),
|
||||
}
|
||||
payload = {
|
||||
"mode": "move",
|
||||
"from": {
|
||||
"mode": plan.old.mode,
|
||||
"path": plan.old.path,
|
||||
"available": location.available,
|
||||
},
|
||||
"to": {
|
||||
"mode": plan.target["mode"],
|
||||
"path": plan.target["path"],
|
||||
"exists": plan.target_exists,
|
||||
"will_create": bool(plan.target["path"]) and not plan.target_exists,
|
||||
"default": plan.target_default,
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"label": s.label,
|
||||
"key": s.key,
|
||||
"files": len(s.scan.copy) if s.scan else 0,
|
||||
"bytes": s.scan.bytes if s.scan else 0,
|
||||
"too_long": len(s.too_long),
|
||||
"links": s.scan.links if s.scan else 0,
|
||||
"orphans": len(s.scan.orphans) if s.scan else 0,
|
||||
"left_in_place": len(s.scan.left) if s.scan else 0,
|
||||
"available": s.available,
|
||||
}
|
||||
for s in sources
|
||||
],
|
||||
"totals": {
|
||||
"sources": len(sources),
|
||||
"files": copies,
|
||||
"bytes": plan.needs_bytes,
|
||||
"orphans": orphans,
|
||||
"temps": temps,
|
||||
"links": links,
|
||||
"left_in_place": len(left),
|
||||
"too_long": len(too_long),
|
||||
"unlisted_sources": len(plan.unlisted),
|
||||
"unlisted_files": unlisted_count,
|
||||
},
|
||||
"too_long_listed": too_long[:LISTED],
|
||||
"left_listed": left[:LISTED],
|
||||
"unlisted": [
|
||||
{"key": key, "files": len(scan.copy) + len(scan.orphans)}
|
||||
for key, _store, scan in plan.unlisted
|
||||
],
|
||||
"needs_bytes": plan.needs_bytes,
|
||||
"free_bytes": free,
|
||||
"confirm_required": plan.confirm_required,
|
||||
"token": plan.token,
|
||||
"sentence": SENTENCE,
|
||||
}
|
||||
if insufficient:
|
||||
payload["insufficient"] = insufficient
|
||||
return payload
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Move thumbnails after a restart and the unavailable folder's actions
|
||||
(#320 Parts 3 and 4): undo an interrupted copy, finish a removal that
|
||||
waited, Recreate folder and Use this folder."""
|
||||
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import preview_location, preview_store
|
||||
from . import thumbnail_move_files as files
|
||||
from .job_routes import busy_response
|
||||
from .jobs import THUMBNAILS_JOB_KIND, JobBusy
|
||||
from .thumbnail_move_plan import (
|
||||
FINISHING_TITLE,
|
||||
MOVE_GROUPS,
|
||||
_read_json,
|
||||
_refusal,
|
||||
_remove,
|
||||
)
|
||||
from . import thumbnail_move_plan as move_plan
|
||||
|
||||
|
||||
class ThumbnailMoveRecoveryMixin:
|
||||
"""Startup recovery, the finishing job, Recreate and Use this folder."""
|
||||
|
||||
_move_recover_lock = threading.Lock()
|
||||
# Per API: set once recover_thumbnail_move has run (a move waits for it).
|
||||
_move_recovered = False
|
||||
|
||||
# ── startup ───────────────────────────────────────────────────────
|
||||
|
||||
def recover_thumbnail_move(self):
|
||||
"""Startup (B4): undo an interrupted copy, or forget a journal the
|
||||
switch already made true. Idempotent; the move start waits for it."""
|
||||
with self._move_recover_lock:
|
||||
if self._move_recovered:
|
||||
return
|
||||
try:
|
||||
self._recover_journal()
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"The interrupted thumbnail move check failed", exc_info=True
|
||||
)
|
||||
self._move_recovered = True
|
||||
|
||||
def _recover_journal(self):
|
||||
path = move_plan.journal_path()
|
||||
journal = _read_json(path)
|
||||
if journal is None:
|
||||
return
|
||||
setting = preview_store.setting()
|
||||
target = journal.get("to") or {}
|
||||
now = setting.path if setting.mode == preview_store.FOLDER_MODE else None
|
||||
if setting.mode == target.get("mode") and (
|
||||
now is None
|
||||
or os.path.normcase(now) == os.path.normcase(target.get("path") or "")
|
||||
):
|
||||
_remove(path) # the switch happened; only the journal was left
|
||||
return
|
||||
for store in journal.get("stores") or []:
|
||||
if not isinstance(store, str) or not os.path.isdir(store):
|
||||
continue
|
||||
for current, dirs, names in os.walk(store):
|
||||
dirs[:] = [
|
||||
d
|
||||
for d in dirs
|
||||
if not preview_location.is_link_like(Path(current) / d)
|
||||
]
|
||||
for name in names:
|
||||
if files.MOVE_TEMP.search(name):
|
||||
files.remove_file(Path(current) / name)
|
||||
for folder in reversed(journal.get("created_dirs") or []):
|
||||
if not isinstance(folder, str) or not os.path.isdir(folder):
|
||||
continue
|
||||
for current, dirs, names in os.walk(folder):
|
||||
dirs[:] = [
|
||||
d
|
||||
for d in dirs
|
||||
if not preview_location.is_link_like(Path(current) / d)
|
||||
]
|
||||
for name in names:
|
||||
if files._preview_parts(name) or files.is_temp_name(name):
|
||||
files.remove_file(Path(current) / name)
|
||||
files.remove_empty_folders(folder)
|
||||
folder = target.get("path")
|
||||
if journal.get("created_folder") and folder and os.path.isdir(folder):
|
||||
names = set(os.listdir(folder))
|
||||
if names <= {preview_location.MARKER_NAME}:
|
||||
_remove(preview_location.marker_path(folder))
|
||||
files.remove_empty_folders(folder)
|
||||
_remove(path)
|
||||
self.logger.info("An interrupted thumbnail move was undone; nothing changed")
|
||||
|
||||
def _cleanup_reachable(self, entry):
|
||||
folder = entry.get("folder")
|
||||
if folder:
|
||||
return os.path.isdir(folder)
|
||||
return True
|
||||
|
||||
async def finish_thumbnail_move(self, _app=None):
|
||||
"""Startup: recover, then start "Move thumbnails (finishing)" when old
|
||||
copies wait and every one is reachable (on the server's loop)."""
|
||||
await self._run_in_executor(self.recover_thumbnail_move)
|
||||
cleanup = list(preview_store.setting().cleanup)
|
||||
if not cleanup:
|
||||
return None
|
||||
reachable = await self._run_in_executor(
|
||||
lambda: all(self._cleanup_reachable(entry) for entry in cleanup)
|
||||
)
|
||||
if not reachable:
|
||||
return None
|
||||
try:
|
||||
return self._jobs.start(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
FINISHING_TITLE,
|
||||
admission="guarded",
|
||||
cancellable=False,
|
||||
scope={"mode": "move", "phase": "remove"},
|
||||
groups=MOVE_GROUPS,
|
||||
runner=self._run_finishing,
|
||||
)
|
||||
except JobBusy:
|
||||
return None
|
||||
|
||||
async def _run_finishing(self, job):
|
||||
await self._remove_old_copies(job, None)
|
||||
left = len(preview_store.setting().cleanup)
|
||||
self._end_move(
|
||||
job,
|
||||
"finished",
|
||||
(
|
||||
"Old thumbnails removed."
|
||||
if not left
|
||||
else (
|
||||
"Some old thumbnails are still in use; they are removed at "
|
||||
"the next start."
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
# ── Recreate and Use this folder (Part 4) ─────────────────────────
|
||||
|
||||
async def recreate_thumbnail_folder(self, _request):
|
||||
blocker = self._jobs.blocker(THUMBNAILS_JOB_KIND, admission="guarded")
|
||||
if blocker is not None:
|
||||
return busy_response(JobBusy(blocker))
|
||||
code = await self._run_in_executor(self._recreate_folder)
|
||||
if code is not None:
|
||||
status = (
|
||||
409 if code in ("location-has-marker", "location-not-empty") else 400
|
||||
)
|
||||
if code == "location-has-marker":
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": code,
|
||||
"error": "The folder is back with its thumbnails. Nothing "
|
||||
"was recreated.",
|
||||
},
|
||||
status=status,
|
||||
)
|
||||
return _refusal(code, status=status)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"location": self._thumbnail_location_payload(with_path=True),
|
||||
}
|
||||
)
|
||||
|
||||
def _recreate_folder(self):
|
||||
setting = preview_store.setting()
|
||||
if setting.mode != preview_store.FOLDER_MODE or not setting.path:
|
||||
return "location-same"
|
||||
folder = setting.path
|
||||
if os.path.isdir(folder):
|
||||
if preview_location.read_marker(folder) is not None:
|
||||
return "location-has-marker"
|
||||
if not preview_location.holds_only_metadata(folder):
|
||||
return "location-not-empty"
|
||||
elif not os.path.isdir(os.path.dirname(folder)):
|
||||
return "location-parent-missing"
|
||||
else:
|
||||
try:
|
||||
os.mkdir(folder)
|
||||
except OSError:
|
||||
return "location-not-writable"
|
||||
try:
|
||||
marker_id = preview_location.init_folder(folder, created_by_pm=True)
|
||||
except OSError:
|
||||
return "location-not-writable"
|
||||
return self._commit_location_id(marker_id)
|
||||
|
||||
async def adopt_thumbnail_folder(self, _request):
|
||||
blocker = self._jobs.blocker(THUMBNAILS_JOB_KIND, admission="guarded")
|
||||
if blocker is not None:
|
||||
return busy_response(JobBusy(blocker))
|
||||
code = await self._run_in_executor(self._adopt_folder)
|
||||
if code is not None:
|
||||
return _refusal(code)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"location": self._thumbnail_location_payload(with_path=True),
|
||||
}
|
||||
)
|
||||
|
||||
def _adopt_folder(self):
|
||||
setting = preview_store.setting()
|
||||
if setting.mode != preview_store.FOLDER_MODE or not setting.path:
|
||||
return "location-same"
|
||||
try:
|
||||
marker = preview_location.read_marker(setting.path)
|
||||
except OSError:
|
||||
return "location-unavailable"
|
||||
if marker is None:
|
||||
return "location-unavailable"
|
||||
return self._commit_location_id(marker["id"])
|
||||
|
||||
def _commit_location_id(self, marker_id):
|
||||
from ..config import PromptManagerConfig
|
||||
from ..config_store import get_config_store
|
||||
|
||||
value = dict(preview_store.setting().to_config())
|
||||
value["id"] = marker_id
|
||||
|
||||
def mutate(config):
|
||||
section = config.get("thumbnails")
|
||||
if not isinstance(section, dict):
|
||||
section = {}
|
||||
section["location"] = value
|
||||
config["thumbnails"] = section
|
||||
return config
|
||||
|
||||
def publish(_config):
|
||||
PromptManagerConfig.THUMBNAIL_LOCATION = value
|
||||
preview_store.configure(value, check=False)
|
||||
|
||||
try:
|
||||
get_config_store().commit(mutate, expected_revision=None, publish=publish)
|
||||
except Exception:
|
||||
self.logger.exception("The thumbnail folder could not be saved")
|
||||
return "location-not-writable"
|
||||
preview_store.refresh()
|
||||
return None
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Remove thumbnails of removed sources (#320 Part 5, D200): a Thumbnail
|
||||
maintenance scope, offered only in folder mode (31A).
|
||||
|
||||
*Removed* means a key of this install in the thumbnail folder's marker whose
|
||||
identity matches no root in any root list. The job removes the
|
||||
preview-grammar files and temps of those key folders, ``rmdir``s the empty
|
||||
folders, drops the keys from the marker and lists anything else as "Left in
|
||||
place". Kind ``thumbnails``, ``mode: "prune"``.
|
||||
"""
|
||||
|
||||
import functools
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import preview_location, preview_store
|
||||
from . import thumbnail_move_files as files
|
||||
from .jobs import THUMBNAILS_JOB_KIND
|
||||
from .thumbnail_location import location_refusal
|
||||
from .thumbnail_move_plan import _number, _plural, _size
|
||||
|
||||
PRUNE_TITLE = "Remove thumbnails of removed sources"
|
||||
PRUNE_GROUPS = ("removed", "left", "errors")
|
||||
|
||||
|
||||
def _prune_line(count, files_count, size):
|
||||
if not count:
|
||||
return "No sources were removed: nothing to remove."
|
||||
return (
|
||||
f"{_plural(count, 'source')} no longer listed: "
|
||||
f"{_plural(files_count, 'thumbnail')} ({_size(size)}) are removed."
|
||||
)
|
||||
|
||||
|
||||
class ThumbnailPruneMixin:
|
||||
"""The prune plan and job."""
|
||||
|
||||
def _prune_targets(self):
|
||||
"""``[(key, store, scan)]`` of this install's unlisted keys (blocking)."""
|
||||
setting = preview_store.setting()
|
||||
location = preview_store.current()
|
||||
if setting.mode != preview_store.FOLDER_MODE or not location.available:
|
||||
return []
|
||||
marker = preview_location.read_marker(setting.path) or {}
|
||||
install = setting.install
|
||||
return self._unlisted_keys(setting, location, marker.get("keys") or {}, install)
|
||||
|
||||
def _prune_payload(self, targets):
|
||||
files_count = sum(
|
||||
len(scan.copy) + len(scan.orphans) for _k, _s, scan in targets
|
||||
)
|
||||
size = sum(scan.bytes for _k, _s, scan in targets)
|
||||
return {
|
||||
"success": True,
|
||||
"mode": "prune",
|
||||
"sources": [
|
||||
{
|
||||
"key": key,
|
||||
"files": len(scan.copy) + len(scan.orphans),
|
||||
"bytes": scan.bytes,
|
||||
"left_in_place": len(scan.left),
|
||||
}
|
||||
for key, _store, scan in targets
|
||||
],
|
||||
"targets": files_count,
|
||||
"bytes": size,
|
||||
"line": _prune_line(len(targets), files_count, size),
|
||||
"folder": preview_store.setting().path,
|
||||
}
|
||||
|
||||
async def get_prune_plan(self):
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
if preview_store.current().mode != preview_store.FOLDER_MODE:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "invalid-request",
|
||||
"error": "Thumbnails are kept inside each source: no removed "
|
||||
"source has thumbnails here.",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
targets = await self._run_in_executor(self._prune_targets)
|
||||
return web.json_response(
|
||||
self._prune_payload(targets), headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
|
||||
def start_thumbnail_prune(self):
|
||||
"""Start the prune job (sync, on the loop); raises JobBusy."""
|
||||
return self._jobs.start(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
PRUNE_TITLE,
|
||||
admission="guarded",
|
||||
cancellable=True,
|
||||
scope={"mode": "prune"},
|
||||
groups=PRUNE_GROUPS,
|
||||
runner=self._run_thumbnail_prune,
|
||||
)
|
||||
|
||||
async def _run_thumbnail_prune(self, job):
|
||||
targets = await self._run_in_executor(self._prune_targets)
|
||||
job.update(total=len(targets), processed=0)
|
||||
setting = preview_store.setting()
|
||||
for done, (key, store, scan) in enumerate(targets, 1):
|
||||
if job.cancelled:
|
||||
break
|
||||
removed, failed = await self._run_in_executor(
|
||||
functools.partial(self._prune_store, store)
|
||||
)
|
||||
job.record_counted(
|
||||
"removed", removed, [{"path": key, "detail": "Thumbnails removed"}]
|
||||
)
|
||||
if failed:
|
||||
job.record_counted("errors", failed)
|
||||
if scan.left:
|
||||
job.record_counted(
|
||||
"left",
|
||||
len(scan.left),
|
||||
[{"path": f"{key}/{'/'.join(item)}"} for item in scan.left],
|
||||
)
|
||||
await self._run_in_executor(
|
||||
functools.partial(self._forget_key, setting.path, key, store)
|
||||
)
|
||||
job.update(
|
||||
processed=done,
|
||||
progress=int(done * 100 / len(targets)),
|
||||
status_line=f"Removing thumbnails of {_number(done)} of "
|
||||
f"{_number(len(targets))} removed sources",
|
||||
)
|
||||
job.publish({"type": "progress", "summary": job.summary()})
|
||||
line = f"{_number(job.counters.get('removed', 0))} thumbnails removed"
|
||||
if job.counters.get("errors"):
|
||||
line += f" · {_plural(job.counters['errors'], 'error')}"
|
||||
job.update(status_line=line, progress=100)
|
||||
job.log_line(line)
|
||||
job.finish("cancelled" if job.cancelled else "finished")
|
||||
|
||||
@staticmethod
|
||||
def _prune_store(store):
|
||||
"""Remove the previews and temps of one key folder; ``(removed, failed)``."""
|
||||
removed = failed = 0
|
||||
store = Path(store)
|
||||
if not store.is_dir() or preview_location.is_link_like(store):
|
||||
return 0, 0
|
||||
for current, dirs, names in os.walk(store):
|
||||
dirs[:] = [
|
||||
d for d in dirs if not preview_location.is_link_like(Path(current) / d)
|
||||
]
|
||||
for name in names:
|
||||
path = Path(current) / name
|
||||
if not (files.is_temp_name(name) or files._preview_parts(name)):
|
||||
continue
|
||||
if preview_location.is_link_like(path):
|
||||
continue
|
||||
if files.remove_file(path):
|
||||
removed += 1
|
||||
else:
|
||||
failed += 1
|
||||
files.remove_empty_folders(store)
|
||||
return removed, failed
|
||||
|
||||
@staticmethod
|
||||
def _forget_key(folder, key, store):
|
||||
"""Drop *key* from the marker once its folder is gone."""
|
||||
if os.path.exists(store):
|
||||
return
|
||||
|
||||
def forget(marker):
|
||||
marker["keys"].pop(key, None)
|
||||
|
||||
with preview_store.write_lock:
|
||||
preview_location.write_marker(folder, forget)
|
||||
@@ -114,9 +114,16 @@ def validate_edge(value):
|
||||
# ── names ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def preview_stem(rel_path):
|
||||
"""``<rel without extension>_thumb`` (POSIX): a preview's name inside the
|
||||
store of its source (``preview_store``)."""
|
||||
return f"{Path(rel_path).with_suffix('').as_posix()}{THUMB_SUFFIX}"
|
||||
|
||||
|
||||
def thumbnail_stem_rel(rel_path):
|
||||
"""``thumbnails/<rel without extension>_thumb`` (POSIX)."""
|
||||
return f"thumbnails/{Path(rel_path).with_suffix('').as_posix()}{THUMB_SUFFIX}"
|
||||
"""``thumbnails/<rel without extension>_thumb`` (POSIX): URL text only.
|
||||
Where the preview is on disk is ``preview_store``'s answer."""
|
||||
return f"thumbnails/{preview_stem(rel_path)}"
|
||||
|
||||
|
||||
def thumbnail_rel(rel_path, extension=None):
|
||||
@@ -207,9 +214,20 @@ def owned_in_folder(folder, stem, source_extension, preferred=None):
|
||||
|
||||
|
||||
def _owned(root, rel_path, preferred):
|
||||
from . import preview_store
|
||||
|
||||
rel_path = Path(rel_path)
|
||||
stem = Path(root) / thumbnail_stem_rel(rel_path)
|
||||
return owned_in_folder(stem.parent, stem.name, rel_path.suffix, preferred)
|
||||
store = preview_store.store_dir(root)
|
||||
if store is None:
|
||||
return iter(())
|
||||
stem = store / preview_stem(rel_path)
|
||||
found = owned_in_folder(stem.parent, stem.name, rel_path.suffix, preferred)
|
||||
if preview_store.current().mode == preview_store.SOURCE_MODE:
|
||||
return found
|
||||
# Folder mode: timed, so a stalled share marks the folder unreachable,
|
||||
# and only previews no link takes out of the key folder (deletes use it).
|
||||
with preview_store.store_io():
|
||||
return iter([path for path in found if preview_store.may_write(path)])
|
||||
|
||||
|
||||
def owned_thumbnails(root, rel_path, preferred=None):
|
||||
|
||||
@@ -767,6 +767,8 @@ class PromptManagerConfig:
|
||||
THUMBNAIL_QUALITY = 80
|
||||
THUMBNAIL_EDGE = 512
|
||||
THUMBNAIL_EDGE_RANGE = (128, 2048)
|
||||
# #320 thumbnail location (config.json ``thumbnails.location``), as read.
|
||||
THUMBNAIL_LOCATION = None
|
||||
|
||||
@classmethod
|
||||
def max_worker_threads(cls) -> int:
|
||||
@@ -828,6 +830,12 @@ class PromptManagerConfig:
|
||||
"format": cls.THUMBNAIL_FORMAT,
|
||||
"quality": cls.THUMBNAIL_QUALITY,
|
||||
"edge": cls.THUMBNAIL_EDGE,
|
||||
# #320: as read; only the Move thumbnails job writes it.
|
||||
**(
|
||||
{"location": cls.THUMBNAIL_LOCATION}
|
||||
if cls.THUMBNAIL_LOCATION is not None
|
||||
else {}
|
||||
),
|
||||
},
|
||||
"gallery": GalleryConfig.get_config(),
|
||||
"integrations": IntegrationConfig.get_config(),
|
||||
@@ -1002,6 +1010,11 @@ class PromptManagerConfig:
|
||||
low, high = cls.THUMBNAIL_EDGE_RANGE
|
||||
if type(edge) is int and low <= edge <= high:
|
||||
cls.THUMBNAIL_EDGE = edge
|
||||
# #320: kept as read (``preview_location.parse_location`` checks it;
|
||||
# a bad value becomes an unavailable ``invalid`` location, never a
|
||||
# silent fall-back to the sources).
|
||||
if "location" in section:
|
||||
cls.THUMBNAIL_LOCATION = section["location"]
|
||||
|
||||
|
||||
# Load configuration on import
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ description = "A powerful ComfyUI custom node that extends the standard text enc
|
||||
version = "3.4.1"
|
||||
license = {file = "LICENSE"}
|
||||
requires-python = ">=3.12"
|
||||
dependencies = ["watchdog>=2.1.0,<7", "Pillow>=8.0.0"]
|
||||
dependencies = ["watchdog>=2.1.0", "Pillow>=8.0.0"]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=84.2.1"]
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Runtime dependencies
|
||||
watchdog>=6.0.0,<7
|
||||
watchdog>=6.0.0
|
||||
Pillow>=12.3.0
|
||||
aiohttp>=3.14.5
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# Runtime dependencies for PromptManager.
|
||||
# aiohttp, numpy and torch come from ComfyUI itself and are not listed here.
|
||||
watchdog>=2.1.0,<7 # Output directory monitoring
|
||||
watchdog>=2.1.0 # Output directory monitoring
|
||||
Pillow>=8.0.0 # Image metadata extraction and thumbnails
|
||||
|
||||
@@ -320,3 +320,44 @@ test('copy toast and fallback dialog',async({page})=>{
|
||||
await expect(fallback).toContainText('Automatic clipboard access requires a secure browser context.');
|
||||
await page.keyboard.press('Escape');await expect(fallback).toBeHidden();await expect(copy).toBeFocused();
|
||||
});
|
||||
|
||||
// #542 D191: "All N eligible nodes" previews every node, sends one after another and names each refusal.
|
||||
test('All eligible nodes previews each node, sends in node id order and names a refusal',async({page,context},testInfo)=>{
|
||||
const host=await context.newPage();await host.goto('/prompt_manager/workspace#/overview');
|
||||
await host.evaluate(async()=>{
|
||||
const {createBridgeHost}=await import('/prompt_manager/js/shared/toolkit/prompt-bridge.js');
|
||||
const targets=['12','3','7'].map(id=>({id,label:id==='7'?'Edited in ComfyUI':'Prompt Manager (+/-)',type:'PromptManagerPair',roles:['positive','negative']}));
|
||||
window.hostValues=Object.fromEntries(targets.map(target=>[target.id,{positive:`Existing ${target.id}`,negative:'Existing negative'}]));window.applied=[];
|
||||
window.fixtureHost=createBridgeHost({origin:location.origin,namespace:'/',label:'Test ComfyUI',adapter:{
|
||||
listTargets:()=>targets,
|
||||
preview:(id,roles)=>{const target=targets.find(item=>item.id===id);return{preview:{target,fields:Object.entries(roles).map(([role,after])=>({role,before:window.hostValues[id][role],after}))},snapshot:{id,roles}};},
|
||||
apply:({id,roles})=>{if(id==='7')throw new Error('Workflow or prompt changed. Preview again.');Object.assign(window.hostValues[id],roles);window.applied.push(id);return{targetId:id,values:roles};},
|
||||
}});
|
||||
});
|
||||
try {
|
||||
await page.goto(workspace);await select(page,'0:shared/same.png');await details(page).getByRole('button',{name:/^Send prompt pair to ComfyUI: /}).click();
|
||||
const sender=dialog(page);const nodes=sender.getByRole('combobox',{name:'Destination node',exact:true});
|
||||
await sender.getByRole('combobox',{name:'ComfyUI window',exact:true}).selectOption({label:'Test ComfyUI'});
|
||||
await expect(nodes.locator('option')).toHaveText(['Choose a destination node','#12 Prompt Manager (+/-)','#3 Prompt Manager (+/-)','#7 Edited in ComfyUI','All 3 eligible nodes']);
|
||||
await expect(nodes).toHaveValue('');
|
||||
await nodes.selectOption({label:'All 3 eligible nodes'});
|
||||
await expect(sender.getByRole('status')).toHaveText('Sending to all 3 eligible nodes in Test ComfyUI. Apply to send it, or preview the changes first.');
|
||||
await page.screenshot({path:testInfo.outputPath('send-all-option.png')});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();
|
||||
const reviews=sender.locator('section.prompt-sender-node');
|
||||
await expect(reviews.locator('> h4')).toHaveText(['#3 Prompt Manager (+/-)','#7 Edited in ComfyUI','#12 Prompt Manager (+/-)']);
|
||||
await expect(sender.getByRole('region',{name:'#12 Prompt Manager (+/-)',exact:true})).toContainText('Existing 12');
|
||||
expect(await host.evaluate(()=>window.applied)).toEqual([]);
|
||||
await sender.getByRole('button',{name:'Apply to ComfyUI',exact:true}).click();
|
||||
await expect(sender.getByRole('alert')).toHaveText('Sent to 2 of 3 nodes in Test ComfyUI. #7 Edited in ComfyUI: Workflow or prompt changed. Preview again.');
|
||||
expect(await host.evaluate(()=>window.applied)).toEqual(['3','12']);
|
||||
expect(await host.evaluate(()=>window.hostValues['7'].positive)).toBe('Existing 7');
|
||||
await page.screenshot({path:testInfo.outputPath('send-all-result.png')});
|
||||
await page.setViewportSize({width:390,height:844});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();await expect(reviews).toHaveCount(3);
|
||||
await reviews.last().scrollIntoViewIfNeeded();
|
||||
expect(await page.evaluate(()=>document.querySelector('dialog.prompt-sender').scrollWidth<=document.querySelector('dialog.prompt-sender').clientWidth+1)).toBe(true);
|
||||
await page.screenshot({path:testInfo.outputPath('send-all-mobile.png')});
|
||||
const apply=sender.getByRole('button',{name:'Apply to ComfyUI',exact:true});await apply.focus();await expect(apply).toBeFocused();
|
||||
} finally {await host.evaluate(()=>window.fixtureHost.dispose());await host.close();}
|
||||
});
|
||||
|
||||
@@ -22,6 +22,9 @@ from PIL.PngImagePlugin import PngInfo
|
||||
from tests.browser.folder_trash import close_trash, replace_folder
|
||||
from tests.browser.malloc_tuning import pin_mmap_threshold
|
||||
from tests.browser.request_hold import register_request_holds
|
||||
from tests.browser.thumbnail_location_fixture import (
|
||||
register_thumbnail_location_commands,
|
||||
)
|
||||
from tests.browser import slow_reset_watch
|
||||
from tests.browser.fast_reset import SEED_ON_LIVE_ENV, ScratchSeeder
|
||||
from tests.browser.workspace_activity_fixture import register_activity_commands
|
||||
@@ -240,6 +243,7 @@ def create_app(root: Path) -> web.Application:
|
||||
library_editor_commands = register_library_editor_commands(routes, api, root)
|
||||
register_vocabulary_commands(routes, api, root)
|
||||
restore_sources = register_source_change_commands(routes, api)
|
||||
reset_thumbnail_location = register_thumbnail_location_commands(routes, api, root)
|
||||
register_tagging_commands(routes, api, root, model, on_reset=restore_sources)
|
||||
register_review_commands(routes, api, root)
|
||||
register_collections_commands(routes, api, root)
|
||||
@@ -451,6 +455,8 @@ def create_app(root: Path) -> web.Application:
|
||||
restore_sources()
|
||||
# No preview queued by the last test may land in the new fixture.
|
||||
await api._run_in_executor(api.close_auto_thumbnails)
|
||||
# Previews inside each source again (#320).
|
||||
await api._run_in_executor(reset_thumbnail_location)
|
||||
library_editor_commands.reset()
|
||||
await api._run_in_executor(reset_logs)
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
@@ -595,6 +601,7 @@ def create_app(root: Path) -> web.Application:
|
||||
history_commands.reset()
|
||||
library_editor_commands.reset()
|
||||
reveal_stub.reset()
|
||||
await api._run_in_executor(reset_thumbnail_location)
|
||||
history_fixture = None
|
||||
manifest = await api._run_in_executor(
|
||||
seeder.run,
|
||||
|
||||
@@ -162,13 +162,15 @@ async function serverJson(request, url) {
|
||||
}
|
||||
|
||||
const number = value => value.toLocaleString('en-US');
|
||||
// The sidebar's Images count (state.mjs), which System's Images metric shows (#555).
|
||||
const IMAGES_COUNT_URL = '/prompt_manager/images/output?mode=workspace&limit=1';
|
||||
|
||||
/** The concept's shell and Overview with the recorded deviations applied, given the server's real overview. */
|
||||
function mapOverviewConcept(reference, overview) {
|
||||
function mapOverviewConcept(reference, overview, images) {
|
||||
// D165: the discreet legacy-pages line follows the settings layout.
|
||||
reference = { ...reference, tree: [...reference.tree, 'p.legacy-pages.small'] };
|
||||
const relabel = { 'Database integrity': 'Not checked', 'Positive prompts': number(overview.library.positive_prompts),
|
||||
'Negative variants': number(overview.library.negative_variants), Images: number(overview.images.indexed) };
|
||||
'Negative variants': number(overview.library.negative_variants), Images: number(images) };
|
||||
const rows = {
|
||||
// D93: the real pending count; Review metadata opens Needs review (the concept's go-review).
|
||||
'3 ambiguous image links': row => ({ ...row,
|
||||
@@ -179,8 +181,9 @@ function mapOverviewConcept(reference, overview) {
|
||||
'One display configuration': row => ({ ...row, description: 'Grid and density settings are shared across views.' }),
|
||||
// D93: the real sources, not samples.
|
||||
'2 sample output sources': row => ({ ...row, title: `${number(overview.sources.effective)} output sources` }),
|
||||
// D96: the dialog shows runs by category and library totals, not prompt, tag or date usage (D153: no ratings).
|
||||
Statistics: row => ({ ...row, description: 'Runs by category, with library totals.' }),
|
||||
// D96: the dialog shows runs by category and database record counts, not prompt, tag or date usage (D153: no
|
||||
// ratings; #555: the counts are labelled as records, not as the Library or Images page counts).
|
||||
Statistics: row => ({ ...row, description: 'Runs by category, with database record counts.' }),
|
||||
};
|
||||
const surfaces = reference.columns.surfaces.map(surface => ({ ...surface,
|
||||
rows: surface.rows.map(row => (rows[row.title] ?? (value => value))(row)) }));
|
||||
@@ -218,6 +221,8 @@ async function conceptOverview(page, theme) {
|
||||
async function productionOverview(page, theme) {
|
||||
await openProduction(page, 'system/overview', theme);
|
||||
await expect(page.locator('#workspace-main .summary-metrics')).toBeVisible();
|
||||
// #555: the Images metric is the shell's Images count: compare once it was read.
|
||||
await expect(page.locator('#workspace-main .summary-metrics > .metric:nth-child(3) strong')).toHaveText(/^[\d,]+$/);
|
||||
// The thumbnail row is its own read (I2): compare once it answered.
|
||||
await expect(page.locator('#workspace-main .operation h3').filter({ hasText: 'thumbnails' })).toHaveText('No missing thumbnails');
|
||||
// So is the backup row (S9).
|
||||
@@ -231,6 +236,7 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`system shell and overview match the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const overview = await serverJson(request, '/prompt_manager/system/overview');
|
||||
const images = (await serverJson(request, IMAGES_COUNT_URL)).total;
|
||||
// The fixture's thumbnails all exist, so the concept's "No missing thumbnails" is also the real title.
|
||||
expect((await serverJson(request, '/prompt_manager/images/thumbnails/status')).pending).toBe(0);
|
||||
const concept = await conceptOverview(page, theme);
|
||||
@@ -239,7 +245,7 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-system-${theme}-${width}.png`), fullPage: true });
|
||||
const production = await productionOverview(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-system-${theme}-${width}.png`), fullPage: true });
|
||||
expect(diffInventory(mapOverviewConcept(concept, overview), production)).toEqual([]);
|
||||
expect(diffInventory(mapOverviewConcept(concept, overview, images), production)).toEqual([]);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
}
|
||||
@@ -265,8 +271,9 @@ const UNMAPPED_PATHS = [
|
||||
test('system comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const overview = await serverJson(request, '/prompt_manager/system/overview');
|
||||
const images = (await serverJson(request, IMAGES_COUNT_URL)).total;
|
||||
const concept = await conceptOverview(page, 'dark');
|
||||
const expected = mapOverviewConcept(concept, overview);
|
||||
const expected = mapOverviewConcept(concept, overview, images);
|
||||
const production = await productionOverview(page, 'dark');
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Unmapped, the concept differs exactly where a deviation is recorded, and nowhere else.
|
||||
@@ -290,7 +297,7 @@ test('system comparison detects mutations', async ({ page, request }) => {
|
||||
});
|
||||
|
||||
/**
|
||||
* D96: the concept's table, its rows the real ones; then "Library totals" in the same table markup. D153 (#447): the
|
||||
* D96: the concept's table, its rows the real ones; then "Database records" (#555) in the same table markup. D153 (#447): the
|
||||
* concept's "Avg. rating" column and a "Rated images" total are not drawn.
|
||||
*/
|
||||
function mapStatisticsConcept(reference, statistics) {
|
||||
@@ -302,9 +309,10 @@ function mapStatisticsConcept(reference, statistics) {
|
||||
tables: [
|
||||
{ ...runs, head: runs.head.filter(([, , text]) => text !== 'Avg. rating'),
|
||||
rows: statistics.by_category.map(row => [row.category ?? 'Uncategorized', number(row.runs)]) },
|
||||
{ caption: 'Library totals', head: [['th', 'col', 'Total'], ['th', 'col', 'Count']],
|
||||
rows: [['Prompts', number(statistics.totals.total_prompts)], ['Tags', number(statistics.totals.total_tags)],
|
||||
['Images', number(statistics.totals.total_images)]] },
|
||||
{ caption: 'Database records', head: [['th', 'col', 'Total'], ['th', 'col', 'Count']],
|
||||
rows: [['Prompt records', number(statistics.totals.total_prompts)],
|
||||
['Tags on prompts', number(statistics.totals.total_tags)],
|
||||
['Image records', number(statistics.totals.total_images)]] },
|
||||
],
|
||||
};
|
||||
}
|
||||
@@ -420,7 +428,7 @@ for (const theme of THEMES) test(`statistics and health dialogs match the concep
|
||||
expect(diffInventory(mapHealthConcept(concept.health), await page.evaluate(measureDialog)).map(item => item.path))
|
||||
.toEqual(['operations.rows.0.title']);
|
||||
await page.evaluate(() => { document.querySelector('#modal .operation h3').textContent = 'Integrity check'; });
|
||||
// And a removed "Library totals" table and a changed caption in the statistics dialog.
|
||||
// And a removed "Database records" table and a changed caption in the statistics dialog.
|
||||
await page.keyboard.press('Escape');
|
||||
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Explore', exact: true }).click();
|
||||
@@ -648,9 +656,10 @@ function mapSourcesConcept(reference, { settings, sources }) {
|
||||
// D113: the range ends at the server's core count (the concept's fixed 16) and shows the real worker count.
|
||||
return [{ ...row, control: { ...row.control, max: String(settings.cpu_count), value: String(settings.worker_threads) } }];
|
||||
}),
|
||||
// D158: the inserted row after the first setting row; D98: "Refresh files" closes the surface.
|
||||
// D158: the inserted row after the first setting row; D196 (#320): the Thumbnail location block after
|
||||
// "Thumbnail generation"; D98: "Refresh files" closes the surface.
|
||||
surface: { ...reference.surface, tree: [...reference.surface.tree.slice(0, rowAt + 1), 'div.setting-row',
|
||||
...reference.surface.tree.slice(rowAt + 1), 'div.operation'] },
|
||||
...reference.surface.tree.slice(rowAt + 1), 'div.setting-block', 'div.operation'] },
|
||||
operations: [...reference.operations, { ...reference.operations[0], title: REFRESH_FILES.title,
|
||||
description: REFRESH_FILES.description, actions: [{ ...reference.operations[0].actions[0], text: REFRESH_FILES.label }] }],
|
||||
};
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
"""The thumbnail location on the test server (#320), undone by every reset.
|
||||
|
||||
Folder mode is reachable from config.json and tests only until the Move
|
||||
thumbnails job lands, so page tests set it here, inside the server's root:
|
||||
|
||||
- ``POST /__test__/thumbnail-location`` ``{"mode": "folder"}`` (or
|
||||
``"source"``): previews live in ``<root>/thumbs/<key>`` from now on (a new
|
||||
folder with Prompt Manager's marker).
|
||||
- ``GET /__test__/thumbnail-files``: the preview files in the thumbnail
|
||||
folder and whether any source grew a ``thumbnails/`` folder.
|
||||
- ``POST /__test__/thumbnail-folder/remove``: the thumbnail folder goes, as
|
||||
when its drive is unplugged; the server notices it on its next check.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Callable
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
|
||||
FOLDER_NAME = "thumbs"
|
||||
|
||||
|
||||
def register_thumbnail_location_commands(
|
||||
routes: web.RouteTableDef, api: PromptManagerAPI, root: Path
|
||||
) -> Callable[[], None]:
|
||||
"""Register the commands; returns the reset every reset runs."""
|
||||
from py.api import preview_location, preview_store
|
||||
from py.api import thumbnail_location, thumbnail_move_plan
|
||||
from py.config import PromptManagerConfig
|
||||
from py.config_store import get_config_store
|
||||
|
||||
folder = root / FOLDER_NAME
|
||||
default = root / "default-thumbs"
|
||||
custom = root / "custom-thumbs" # a folder a page types in
|
||||
journal = root / "data" / "thumbnail-move.json"
|
||||
# The move's journal and default folder live in this server's root, never
|
||||
# in the checkout's data folder.
|
||||
thumbnail_move_plan.journal_path = lambda: str(journal)
|
||||
thumbnail_move_plan.default_path = lambda: str(default)
|
||||
thumbnail_location.default_path = lambda: str(default)
|
||||
api.recover_thumbnail_move()
|
||||
|
||||
def forget_config(config: dict) -> dict:
|
||||
section = config.get("thumbnails")
|
||||
if isinstance(section, dict):
|
||||
section.pop("location", None)
|
||||
return config
|
||||
|
||||
def reset() -> None:
|
||||
preview_store.configure(None)
|
||||
PromptManagerConfig.THUMBNAIL_LOCATION = None
|
||||
store = get_config_store()
|
||||
config, _revision = store.read()
|
||||
if (
|
||||
isinstance(config.get("thumbnails"), dict)
|
||||
and "location" in config["thumbnails"]
|
||||
):
|
||||
store.commit(forget_config, expected_revision=None)
|
||||
for path in (folder, default, custom):
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
journal.unlink(missing_ok=True)
|
||||
|
||||
def use(mode: str) -> dict:
|
||||
reset()
|
||||
if mode == "source":
|
||||
return {"mode": "source"}
|
||||
folder.mkdir()
|
||||
marker_id = preview_location.init_folder(str(folder), created_by_pm=True)
|
||||
location = preview_store.configure(
|
||||
{
|
||||
"mode": "folder",
|
||||
"path": str(folder),
|
||||
"id": marker_id,
|
||||
"install": "browser-tests",
|
||||
}
|
||||
)
|
||||
return {"mode": location.mode, "available": location.available}
|
||||
|
||||
@routes.post("/__test__/thumbnail-location")
|
||||
async def set_location(request: web.Request) -> web.Response:
|
||||
data = await request.json()
|
||||
mode = data.get("mode") if isinstance(data, dict) else None
|
||||
if mode not in ("source", "folder"):
|
||||
raise web.HTTPBadRequest()
|
||||
return web.json_response(await api._run_in_executor(use, mode))
|
||||
|
||||
def files() -> dict:
|
||||
setting = preview_store.setting()
|
||||
where = Path(setting.path) if setting.path else folder
|
||||
previews = sorted(
|
||||
path.relative_to(where).as_posix()
|
||||
for path in where.rglob("*_thumb.*")
|
||||
if path.is_file()
|
||||
)
|
||||
sources = [str(path) for path in api._get_all_output_dirs()]
|
||||
return {
|
||||
"folder": previews,
|
||||
"folder_exists": folder.exists() or default.exists() or custom.exists(),
|
||||
"custom_path": str(custom),
|
||||
"source_thumbnails": any(
|
||||
(Path(source) / "thumbnails").exists() for source in sources
|
||||
),
|
||||
}
|
||||
|
||||
@routes.get("/__test__/thumbnail-files")
|
||||
async def thumbnail_files(_request: web.Request) -> web.Response:
|
||||
return web.json_response(await api._run_in_executor(files))
|
||||
|
||||
def remove() -> bool:
|
||||
setting = preview_store.setting()
|
||||
shutil.rmtree(setting.path or folder, ignore_errors=True)
|
||||
return preview_store.refresh()
|
||||
|
||||
@routes.post("/__test__/thumbnail-folder/remove")
|
||||
async def remove_folder(_request: web.Request) -> web.Response:
|
||||
available = await api._run_in_executor(remove)
|
||||
return web.json_response({"available": available})
|
||||
|
||||
def foreign() -> bool:
|
||||
"""Another install recreated the folder: its marker has another id."""
|
||||
setting = preview_store.setting()
|
||||
|
||||
def other(marker: dict) -> None:
|
||||
marker["id"] = "another-install"
|
||||
|
||||
preview_location.write_marker(setting.path, other)
|
||||
return preview_store.refresh()
|
||||
|
||||
@routes.post("/__test__/thumbnail-folder/foreign")
|
||||
async def foreign_folder(_request: web.Request) -> web.Response:
|
||||
return web.json_response({"available": await api._run_in_executor(foreign)})
|
||||
|
||||
return reset
|
||||
@@ -261,3 +261,20 @@ test('prompt deletion requires confirmation and can retry after a network failur
|
||||
await expect(await request.get(url)).toBeOK();
|
||||
}
|
||||
});
|
||||
|
||||
// #320: with previews in one folder, the legacy gallery still shows them, made
|
||||
// and served from that folder (the fixture's own previews inside the sources
|
||||
// are never served then).
|
||||
test('the legacy gallery shows thumbnails kept in one folder', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/thumbnail-location', { data: { mode: 'folder' } })).toBeOK();
|
||||
const served = page.waitForResponse(response =>
|
||||
response.url().includes('/prompt_manager/images/serve/thumbnails/') && response.status() === 200);
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2);
|
||||
const response = await served; // made in the folder before it was served
|
||||
const name = decodeURIComponent(new URL(response.url()).pathname.split('/serve/thumbnails/')[1]);
|
||||
const files = await (await request.get('/__test__/thumbnail-files')).json();
|
||||
const stem = name.replace(/\.[^.]+$/, '');
|
||||
expect(files.folder.some(file => /^[a-z0-9_-]+-[0-9a-f]{8}\//.test(file)
|
||||
&& file.slice(file.indexOf('/') + 1).replace(/\.[^.]+$/, '') === stem)).toBe(true);
|
||||
});
|
||||
|
||||
@@ -156,8 +156,9 @@ test('at 390px the window fits the screen, actions stay reachable and the page s
|
||||
expect(frame.bottom-frame.top).toBeLessThan(844);
|
||||
expect(frame.bottom-frame.top).toBeGreaterThanOrEqual(280);
|
||||
expect(await page.evaluate(()=>document.documentElement.scrollWidth)).toBeLessThanOrEqual(390);
|
||||
// The last row's actions can be scrolled to inside the window and fit its width.
|
||||
const last=rows(page).last().getByRole('button',{name:'Rename / merge',exact:true});
|
||||
// The first page's last row's actions can be scrolled to inside the window and fit its width.
|
||||
// Pinned to row 100: reaching it loads the next page (D183), and last() would then follow row 200.
|
||||
const last=rows(page).nth(99).getByRole('button',{name:'Rename / merge',exact:true});
|
||||
await last.scrollIntoViewIfNeeded();await expect(last).toBeInViewport();
|
||||
const [button]=await boxes(last);const [after]=await boxes(tagWindow(page));
|
||||
expect(button.right).toBeLessThanOrEqual(after.right);
|
||||
|
||||
@@ -284,6 +284,23 @@ test('a restore seen finishing elsewhere closes an open restore review and disca
|
||||
expect((await serverJson(request, '/__test__/system/backups')).stages).toBe(0);
|
||||
});
|
||||
|
||||
// #583: in WebKit, while a page.route is armed, a file part of a FormData body is read only when the request goes on;
|
||||
// a garbage collection in between dropped the form once nothing referenced it, and the server got an empty file
|
||||
// ("Uploaded file is empty"). The page keeps the form until its upload is answered. The route holds the upload
|
||||
// through a collection; without the fix WebKit lost the file every time (0/10 in the pinned image). Chromium and
|
||||
// Firefox keep the body anyway, so there it only checks that a held upload still opens the review.
|
||||
test('a review upload survives a garbage collection while it is in flight', async ({ page, request }) => {
|
||||
const bytes = await backupBytes(request);
|
||||
await page.route(`**${PREVIEW_URL}`, async route => {
|
||||
await page.requestGC();
|
||||
await route.continue();
|
||||
});
|
||||
await openBackups(page);
|
||||
expect((await chooseBackup(page, bytes)).status()).toBe(200);
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Restore preview');
|
||||
expect((await serverJson(request, '/__test__/system/backups')).stages).toBe(1);
|
||||
});
|
||||
|
||||
test('invalid file shows the validator message; nothing is staged', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
await openBackups(page);
|
||||
|
||||
@@ -8,6 +8,8 @@ import { readCoverageTable, parseControl, CONTROL_EXEMPTIONS } from '../../tools
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const OVERVIEW_URL = '/prompt_manager/system/overview';
|
||||
const THUMBNAILS_URL = '/prompt_manager/images/thumbnails/status';
|
||||
// The sidebar's Images count (state.mjs NAV_COUNT_SOURCES): the Images page's own total.
|
||||
const IMAGES_COUNT_URL = '/prompt_manager/images/output?mode=workspace&limit=1';
|
||||
const number = value => value.toLocaleString('en-US');
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
@@ -55,6 +57,7 @@ test('overview shows real counts and Not checked', async ({ page, request }) =>
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) writes.push(item.url()); });
|
||||
const overview = await serverJson(request, OVERVIEW_URL);
|
||||
const thumbnails = await serverJson(request, THUMBNAILS_URL);
|
||||
const images = (await serverJson(request, IMAGES_COUNT_URL)).total;
|
||||
// The concept fixture: six positives, their saved negatives and 24 linked images, all with thumbnails.
|
||||
expect([overview.library.positive_prompts, overview.images.indexed, thumbnails.pending]).toEqual([6, 24, 0]);
|
||||
// I2: the Overview never walks the sources for thumbnails; the row reads the status route itself.
|
||||
@@ -66,7 +69,8 @@ test('overview shows real counts and Not checked', async ({ page, request }) =>
|
||||
[item.querySelector('strong').textContent, item.querySelector('span').textContent]))).toEqual([
|
||||
[number(overview.library.positive_prompts), 'Positive prompts'],
|
||||
[number(overview.library.negative_variants), 'Negative variants'],
|
||||
[number(overview.images.indexed), 'Images'],
|
||||
// #555: the Images page's own count, the same number as its sidebar badge.
|
||||
[number(images), 'Images'],
|
||||
['Not checked', 'Database integrity'],
|
||||
]);
|
||||
expect(await rowsOf(layout, 'Library health')).toEqual([
|
||||
@@ -79,11 +83,31 @@ test('overview shows real counts and Not checked', async ({ page, request }) =>
|
||||
expect(await rowsOf(layout, 'Workspace preferences')).toEqual([
|
||||
['One display configuration', 'Grid and density settings are shared across views.', 'Appearance'],
|
||||
['2 output sources', 'Monitoring and enrichment share one source list.', 'Manage sources'],
|
||||
['Statistics', 'Runs by category, with library totals.', 'Explore'],
|
||||
['Statistics', 'Runs by category, with database record counts.', 'Explore'],
|
||||
]);
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
|
||||
test('#555: the Images metric is the sidebar Images count, not the indexed image rows', async ({ page, request }) => {
|
||||
const overview = await serverJson(request, OVERVIEW_URL);
|
||||
// The Images page lists files no row links (the demo library: 122 files, 110 rows); the listing says so.
|
||||
const listed = overview.images.indexed + 12;
|
||||
await page.route(`**${IMAGES_COUNT_URL}`, async route => {
|
||||
const response = await route.fetch();
|
||||
await route.fulfill({ response, json: { ...(await response.json()), total: listed } });
|
||||
});
|
||||
const layout = await openOverview(page);
|
||||
const imagesMetric = layout.locator('.summary-metrics > .metric').filter({ has: page.locator('span', { hasText: /^Images$/ }) });
|
||||
await expect(imagesMetric.locator('strong')).toHaveText(number(listed));
|
||||
await expect(page.locator('#primary-nav a[data-route="images"] .nav-count')).toHaveText(number(listed));
|
||||
// A failed count reads as a dash, never the row count.
|
||||
await page.unroute(`**${IMAGES_COUNT_URL}`);
|
||||
await page.route(`**${IMAGES_COUNT_URL}`, route => route.fulfill({ status: 503, json: { success: false, error: 'Unavailable' } }));
|
||||
await page.goto(`${workspace}#/system/appearance`);
|
||||
await page.locator('#workspace-main .section-nav').getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect(imagesMetric.locator('strong')).toHaveText('–');
|
||||
});
|
||||
|
||||
// D129 (#249 S6b): where the library lives. The hook makes the fixture's database read as a node-folder library; its
|
||||
// Move library is a stand-in that touches no file (workspace_system_fixture.py).
|
||||
const banner = page => page.locator('#library-banner');
|
||||
@@ -364,7 +388,7 @@ test('View backups / Appearance / Manage sources switch sections', async ({ page
|
||||
}
|
||||
});
|
||||
|
||||
test('statistics dialog lists runs by category and library totals', async ({ page, request }) => {
|
||||
test('statistics dialog lists runs by category and database record counts', async ({ page, request }) => {
|
||||
const statistics = await serverJson(request, '/prompt_manager/system/statistics');
|
||||
expect(statistics.by_category.length).toBeGreaterThan(1);
|
||||
const layout = await openOverview(page);
|
||||
@@ -373,13 +397,14 @@ test('statistics dialog lists runs by category and library totals', async ({ pag
|
||||
const dialog = page.locator('dialog#modal[open]');
|
||||
await expect(dialog.locator('#modal-title')).toHaveText('Usage statistics');
|
||||
const tables = dialog.locator('#modal-body > table');
|
||||
await expect(tables.locator('caption')).toHaveText(['Runs by category', 'Library totals']);
|
||||
await expect(tables.locator('caption')).toHaveText(['Runs by category', 'Database records']);
|
||||
// D153 (#447): no rating column and no rated-images total.
|
||||
await expect(tables.first().locator('thead th')).toHaveText(['Category', 'Runs']);
|
||||
const cells = table => table.locator('tbody tr').evaluateAll(rows => rows.map(row => [...row.cells].map(cell => cell.textContent)));
|
||||
expect(await cells(tables.first())).toEqual(statistics.by_category.map(row => [row.category ?? 'Uncategorized', number(row.runs)]));
|
||||
expect(await cells(tables.nth(1))).toEqual([['Prompts', number(statistics.totals.total_prompts)],
|
||||
['Tags', number(statistics.totals.total_tags)], ['Images', number(statistics.totals.total_images)]]);
|
||||
// #555: raw rows of the legacy /stats, labelled as rows so they never read as the Library or Images page counts.
|
||||
expect(await cells(tables.nth(1))).toEqual([['Prompt records', number(statistics.totals.total_prompts)],
|
||||
['Tags on prompts', number(statistics.totals.total_tags)], ['Image records', number(statistics.totals.total_images)]]);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(explore).toBeFocused();
|
||||
// A statistics read that fails says so; the Overview stays.
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
|
||||
// #320 Thumbnail location (D196–D201, the approved 320 mock) on the concept-shaped real fixture: the row in System ›
|
||||
// Sources & scans reviews and runs Move thumbnails both ways, the review refuses a folder in place, and an unplugged
|
||||
// thumbnail folder shows the notice whose Recreate folder brings it back. The hooks are thumbnail_location_fixture.py's.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const NOTICE_KEY = 'pm_update_notice_dismissed';
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/thumbnail-location', { data: { mode: 'source' } })).toBeOK();
|
||||
page.locationErrors = [];
|
||||
page.on('pageerror', error => page.locationErrors.push(error.message));
|
||||
await page.addInitScript(key => { if (location.protocol.startsWith('http')) localStorage.setItem(key, '3.2.0'); }, NOTICE_KEY);
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/thumbnail-location', { data: { mode: 'source' } });
|
||||
expect(page.locationErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const block = page => page.locator('#location-block');
|
||||
const dialog = page => page.locator('dialog#modal[open]');
|
||||
|
||||
async function files(request) {
|
||||
const response = await request.get('/__test__/thumbnail-files');
|
||||
await expect(response).toBeOK();
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function openSources(page) {
|
||||
await page.goto(`${workspace}#/system/sources`);
|
||||
await expect(block(page).locator('#location-status')).not.toHaveText('');
|
||||
}
|
||||
|
||||
/** Review the row's choice and start the move; resolves once its job view says it finished. */
|
||||
async function moveFromRow(page) {
|
||||
const planned = page.waitForResponse(response => response.url().endsWith('/images/thumbnails/plan')
|
||||
&& response.request().method() === 'POST');
|
||||
await block(page).getByRole('button', { name: 'Review move…' }).click();
|
||||
expect((await planned).status()).toBe(200);
|
||||
await expect(dialog(page).locator('h2')).toHaveText('Move thumbnails');
|
||||
await dialog(page).getByRole('button', { name: 'Move thumbnails' }).click();
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Move thumbnails');
|
||||
await expect(page.locator('#workspace-main')).toContainText('Thumbnails moved to');
|
||||
}
|
||||
|
||||
test('the row moves thumbnails into one folder and back', async ({ page, request }) => {
|
||||
await openSources(page);
|
||||
const row = block(page);
|
||||
await expect(row.getByRole('radio', { name: /Inside each source/ })).toBeChecked();
|
||||
await expect(row.locator('#location-status')).toHaveText('Now: inside each source.');
|
||||
await expect(row.getByRole('button', { name: 'Review move…' })).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(row.locator('#location-path')).toBeDisabled();
|
||||
|
||||
await row.getByRole('radio', { name: /One folder/ }).check();
|
||||
await expect(row.locator('#location-path')).toBeEnabled();
|
||||
await expect(row.locator('#location-path')).toHaveValue(/default-thumbs$/);
|
||||
await expect(row.locator('#location-status')).toHaveText(/^Now: inside each source\. The suggested folder is next to your library/);
|
||||
await expect(row.getByRole('button', { name: 'Review move…' })).toHaveAttribute('aria-disabled', 'false');
|
||||
await moveFromRow(page);
|
||||
await expect(page.locator('#job-phase')).toHaveText(/^Thumbnails are now read from the new location/);
|
||||
const moved = await files(request);
|
||||
expect(moved.folder.length).toBeGreaterThan(0);
|
||||
expect(moved.folder.every(name => /^[a-z0-9_-]+-[0-9a-f]{8}\//.test(name))).toBe(true);
|
||||
expect(moved.source_thumbnails).toBe(false);
|
||||
|
||||
// Images shows thumbnails served from the folder.
|
||||
const served = page.waitForResponse(response => response.url().includes('/images/serve/thumbnails/')
|
||||
&& response.status() === 200);
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await served;
|
||||
|
||||
await openSources(page);
|
||||
await expect(block(page).locator('#location-status')).toHaveText(/^Now: one folder, .*default-thumbs\. Using the default folder/);
|
||||
await block(page).getByRole('radio', { name: /Inside each source/ }).check();
|
||||
await moveFromRow(page);
|
||||
const back = await files(request);
|
||||
expect(back.source_thumbnails).toBe(true);
|
||||
expect(back.folder_exists).toBe(false);
|
||||
});
|
||||
|
||||
test('a folder the server refuses is explained under the field', async ({ page }) => {
|
||||
await openSources(page);
|
||||
const row = block(page);
|
||||
await row.getByRole('radio', { name: /One folder/ }).check();
|
||||
await row.locator('#location-path').fill('thumbnails');
|
||||
await row.getByRole('button', { name: 'Review move…' }).click();
|
||||
await expect(row.locator('#location-problem')).toHaveText(
|
||||
'Enter a full folder path, such as D:\\Thumbnails or /mnt/fast/thumbnails.');
|
||||
await expect(row.locator('.location-path')).toHaveAttribute('aria-invalid', 'true');
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await row.locator('#location-path').fill('/somewhere/else');
|
||||
await expect(row.locator('#location-problem')).toHaveText('');
|
||||
});
|
||||
|
||||
test('an unplugged folder shows the notice and Recreate folder brings it back', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/thumbnail-location', { data: { mode: 'folder' } })).toBeOK();
|
||||
const removed = await request.post('/__test__/thumbnail-folder/remove');
|
||||
expect((await removed.json()).available).toBe(false);
|
||||
await openSources(page);
|
||||
const notice = page.locator('#unavailable-notice');
|
||||
await expect(notice.locator('h2')).toHaveText('The thumbnail folder is unavailable');
|
||||
await expect(notice).toContainText('is missing. Originals are shown and no thumbnails are written until it is back.');
|
||||
await expect(block(page).locator('#location-status')).toHaveText(/· unavailable\. Originals are shown\.$/);
|
||||
await notice.getByRole('button', { name: 'Recreate folder' }).click();
|
||||
await expect(dialog(page).locator('h2')).toHaveText('Recreate the thumbnail folder?');
|
||||
const recreated = page.waitForResponse(response => response.url().endsWith('/location/recreate'));
|
||||
await dialog(page).getByRole('button', { name: 'Recreate folder' }).click();
|
||||
expect((await recreated).status()).toBe(200);
|
||||
await expect(notice).toHaveCount(0);
|
||||
await expect(block(page).locator('#location-status')).toHaveText(/^Now: one folder, .*thumbs\.$/);
|
||||
});
|
||||
|
||||
test('a folder of your own needs the confirmation before it moves', async ({ page, request }) => {
|
||||
const custom = (await files(request)).custom_path;
|
||||
await openSources(page);
|
||||
const row = block(page);
|
||||
await row.getByRole('radio', { name: /One folder/ }).check();
|
||||
await row.locator('#location-path').fill(custom);
|
||||
await row.getByRole('button', { name: 'Review move…' }).click();
|
||||
const review = dialog(page);
|
||||
await expect(review.locator('dt')).toContainText(['From', 'To', 'Thumbnails', 'Free space', 'Removed after the move']);
|
||||
await expect(review.locator('dd').nth(1)).toHaveText(`${custom} (new folder)`);
|
||||
const move = review.getByRole('button', { name: 'Move thumbnails' });
|
||||
await expect(move).toHaveAttribute('aria-disabled', 'true');
|
||||
await review.getByRole('checkbox', { name: /may create its own folders/ }).check();
|
||||
await expect(move).toHaveAttribute('aria-disabled', 'false');
|
||||
await move.click();
|
||||
await expect(page.locator('#workspace-main')).toContainText('Thumbnails moved to');
|
||||
expect((await files(request)).folder.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('another folder’s marker offers Use this folder', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/thumbnail-location', { data: { mode: 'folder' } })).toBeOK();
|
||||
await expect(await request.post('/__test__/thumbnail-folder/foreign')).toBeOK();
|
||||
await openSources(page);
|
||||
const notice = page.locator('#unavailable-notice');
|
||||
await expect(notice).toContainText('holds another thumbnail folder');
|
||||
await notice.getByRole('button', { name: 'Use this folder' }).click();
|
||||
await expect(notice).toHaveCount(0);
|
||||
await expect(block(page).locator('#location-status')).toHaveText(/^Now: one folder, .*thumbs\.$/);
|
||||
});
|
||||
@@ -145,9 +145,10 @@ test('long vocabularies append and Load more is keyboard reachable',async({page,
|
||||
await expect(loadMore(page)).toHaveCount(0);
|
||||
await expect(rows(page).nth(200).getByRole('button',{name:'Browse',exact:true})).toBeFocused();
|
||||
await expect(rows(page).locator('td:nth-child(1)')).toHaveText(manifest.tags);
|
||||
// Reload with the window at its top: Firefox restores a scrolled window, whose end then appends a page before the check.
|
||||
await main(page).locator('.two-col .table-wrap').evaluate(node=>new Promise(resolve=>{node.scrollTop=0;requestAnimationFrame(()=>requestAnimationFrame(resolve));}));
|
||||
await page.reload();
|
||||
// Open the screen as a new history entry, not a reload: Firefox restores a reloaded page's scrolled tag window, and
|
||||
// its end then appends a page before the check (#567). Scrolling the window to its top first does not always stick.
|
||||
await page.goto('about:blank');
|
||||
await page.goto(route);
|
||||
await expect(rows(page)).toHaveCount(100);
|
||||
// D183: the end of the list arrives inside the tag window.
|
||||
const scroll=top=>main(page).locator('.two-col .table-wrap').evaluate((node,y)=>new Promise(resolve=>{node.scrollTop=y ?? node.scrollHeight;requestAnimationFrame(()=>requestAnimationFrame(resolve));}),top);
|
||||
|
||||
@@ -131,11 +131,40 @@ test('native image inspector previews and sends a pair into real visible widgets
|
||||
const twinId=await page.evaluate(async()=>{const {app}=await import('/scripts/app.js');const node=LiteGraph.createNode('PromptManagerPair');app.graph.add(node);return node.id;});
|
||||
await find0.click();
|
||||
const targets=sender.getByRole('combobox',{name:'Destination node',exact:true});
|
||||
await expect(targets.locator('option')).toHaveCount(3);
|
||||
await expect(targets.locator('option')).toHaveCount(4);
|
||||
await expect(targets).toHaveValue('');
|
||||
await expect(apply).toBeDisabled();
|
||||
await expect(targets.locator('option').nth(2)).toHaveText(new RegExp(`^#${twinId} `));
|
||||
await page.evaluate(async twinId=>{const {app}=await import('/scripts/app.js');app.graph.remove(app.graph.getNodeById(twinId));},twinId);
|
||||
// #542 D191: "All 2 eligible nodes" writes both real nodes, one after another; a linked third node is not offered.
|
||||
const linkedId=await page.evaluate(async()=>{
|
||||
const {app}=await import('/scripts/app.js');const {convertToInput}=await import('/extensions/core/widgetInputs.js');
|
||||
const node=LiteGraph.createNode('PromptManagerPair');app.graph.add(node);
|
||||
const widget=node.widgets.find(widget=>widget.name==='text');
|
||||
if(!node.inputs.some(input=>input.name==='text'))convertToInput(node,widget);
|
||||
const source=LiteGraph.createNode('PrimitiveString');app.graph.add(source);
|
||||
source.connect(0,node,node.inputs.findIndex(input=>input.name==='text'));
|
||||
for(const each of app.graph._nodes.filter(item=>item.type==='PromptManagerPair')){const text=each.widgets.find(widget=>widget.name==='text');if(text)text.value=`Before all ${each.id}`;}
|
||||
return node.id;
|
||||
});
|
||||
await find0.click();
|
||||
await expect(targets.locator('option')).toHaveText(['Choose a destination node',new RegExp(`^#${id} `),new RegExp(`^#${twinId} `),'All 2 eligible nodes']);
|
||||
await expect(sender.getByRole('status')).toContainText('1 other Prompt Manager node cannot receive this prompt.');
|
||||
await targets.selectOption({label:'All 2 eligible nodes'});
|
||||
await sender.getByRole('button',{name:'Preview changes',exact:true}).click();
|
||||
await expect(sender.locator('section.prompt-sender-node > h4')).toHaveText([new RegExp(`^#${id} `),new RegExp(`^#${twinId} `)]);
|
||||
await apply.click();
|
||||
// The tab title gains ComfyUI's unsaved mark once the workflow changed, so the window is matched loosely.
|
||||
await expect(sender.getByRole('status')).toHaveText(/^Sent to 2 of 2 nodes in .*ComfyUI\.$/);
|
||||
await expect(apply).toBeDisabled();
|
||||
const all=await page.evaluate(async ids=>{
|
||||
const {app}=await import('/scripts/app.js');const output=(await app.graphToPrompt()).output;
|
||||
return ids.map(nodeId=>{const node=app.graph.getNodeById(nodeId);const serialized=node.serialize();const index=node.widgets.findIndex(widget=>widget.name==='negative_text');
|
||||
return{prompt:output[String(nodeId)]?.inputs.negative_text,serialized:serialized.widgets_values[index],text:node.widgets.find(widget=>widget.name==='text')?.value};});
|
||||
},[id,twinId,linkedId]);
|
||||
expect(all.slice(0,2)).toEqual([{prompt:negative,serialized:negative,text:positive},{prompt:negative,serialized:negative,text:positive}]);
|
||||
expect(all[2].text).toBe(`Before all ${linkedId}`);
|
||||
expect(await page.evaluate(()=>window.senderEffects.queue-window.senderSendBaseline.queue)).toBe(0);
|
||||
await page.evaluate(async ids=>{const {app}=await import('/scripts/app.js');for(const nodeId of ids)app.graph.remove(app.graph.getNodeById(nodeId));},[twinId,linkedId]);
|
||||
// #535: the status line says why a window cannot receive.
|
||||
const find=sender.getByRole('button',{name:'Find ComfyUI windows',exact:true});
|
||||
const preview=sender.getByRole('button',{name:'Preview changes',exact:true});
|
||||
|
||||
+11
-1
@@ -125,6 +125,9 @@
|
||||
"POST /prompt_manager/images/removed/return",
|
||||
"POST /prompt_manager/images/rescope",
|
||||
"POST /prompt_manager/images/reveal",
|
||||
"POST /prompt_manager/images/thumbnails/location/adopt",
|
||||
"POST /prompt_manager/images/thumbnails/location/recreate",
|
||||
"POST /prompt_manager/images/thumbnails/plan",
|
||||
"POST /prompt_manager/jobs/autotag",
|
||||
"POST /prompt_manager/jobs/duplicates",
|
||||
"POST /prompt_manager/jobs/image-tags",
|
||||
@@ -212,6 +215,13 @@
|
||||
"thumbnail_encoders",
|
||||
"thumbnail_encoders.jpeg",
|
||||
"thumbnail_encoders.jxl",
|
||||
"thumbnail_encoders.webp"
|
||||
"thumbnail_encoders.webp",
|
||||
"thumbnail_location",
|
||||
"thumbnail_location.available",
|
||||
"thumbnail_location.cleanup_pending",
|
||||
"thumbnail_location.default_path",
|
||||
"thumbnail_location.mode",
|
||||
"thumbnail_location.path",
|
||||
"thumbnail_location.problem"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -225,3 +225,141 @@ test('acknowledgement cannot claim an extra role beyond approved changes',async(
|
||||
const {createSendSession}=await source;const s=setup(createSendSession,{apply:async()=>({targetId:'one',values:{positive:'new',negative:'also changed'}})});
|
||||
await s.session.discover();s.session.selectHost('host');s.session.selectTarget('one');await s.session.preview();await s.session.apply();assert.equal(s.session.state.phase,'error');assert.match(s.session.state.message,/unexpected acknowledgement/);s.session.dispose();
|
||||
});
|
||||
// #542: "All N eligible nodes" sends to every eligible node of one window, previewing every node first (Q542-2 C).
|
||||
function allHost({nodes=['10','9','2'],revision,preview,apply,discover}={}) {
|
||||
const calls=[];const sent=new Set();
|
||||
const targets=nodes.map(id=>pm(id,['positive'],revision?{revision}:{}));
|
||||
const client={
|
||||
discover:discover ?? (async()=>[window1(targets)]),
|
||||
preview:async(host,id,roles)=>{calls.push(['preview',id]);if(preview)return preview(id,calls,sent);return{ticket:`t-${id}-${calls.length}`,revision,target:{id,label:`Node ${id}`},fields:[{role:'positive',before:sent.has(id)?'new':'old',after:roles.positive}]};},
|
||||
apply:async(host,ticket)=>{const id=ticket.split('-')[1];calls.push(['apply',id,ticket]);if(apply)return apply(id,calls);sent.add(id);return{targetId:id,values:{positive:'new'}};},
|
||||
dispose:()=>calls.push(['dispose']),
|
||||
};
|
||||
return {client,calls,sent};
|
||||
}
|
||||
async function allSession(createSendSession,options) {
|
||||
const host=allHost(options);const session=createSendSession({client:host.client,roles:{positive:'new'},onChange:()=>{}});
|
||||
await session.discover();return {...host,session};
|
||||
}
|
||||
const order=calls=>calls.filter(call=>call[0]!=='dispose').map(call=>`${call[0]} ${call[1]}`);
|
||||
test('All is offered only with two or more eligible nodes and is never selected for you',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;
|
||||
const one=await allSession(createSendSession,{nodes:['1']});
|
||||
one.session.selectTarget(ALL_NODES);assert.equal(one.session.state.targetId,'');assert.equal(one.calls.length,0);one.session.dispose();
|
||||
const two=await allSession(createSendSession,{nodes:['1','2']});
|
||||
assert.equal(two.session.state.targetId,'');
|
||||
two.session.selectTarget(ALL_NODES);assert.equal(two.session.state.targetId,ALL_NODES);
|
||||
assert.match(two.session.state.message,/all 2 eligible nodes in Flow - ComfyUI/);
|
||||
await two.session.discover();assert.equal(two.session.state.targetId,'');two.session.dispose();
|
||||
});
|
||||
test('Apply to All previews every node first, then sends one after another in node id order',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;const s=await allSession(createSendSession);
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.deepEqual(order(s.calls),['preview 2','preview 9','preview 10','apply 2','preview 9','apply 9','preview 10','apply 10']);
|
||||
assert.equal(s.session.state.phase,'success');assert.equal(s.session.state.message,'Sent to 3 of 3 nodes in Flow - ComfyUI.');
|
||||
await s.session.apply();assert.equal(s.calls.filter(call=>call[0]==='apply').length,3);s.session.dispose();
|
||||
});
|
||||
test('a double click on Apply to All sends to each node once',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;const s=await allSession(createSendSession,{nodes:['1','2']});
|
||||
s.session.selectTarget(ALL_NODES);await Promise.all([s.session.apply(),s.session.apply()]);
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['1','2']);s.session.dispose();
|
||||
});
|
||||
test('a node refusing its preview stops All before anything is written and names the node',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;
|
||||
const s=await allSession(createSendSession,{preview:id=>{if(id==='9')throw Error('positive is connected to an upstream input.');return{ticket:`t-${id}`,target:{id},fields:[{role:'positive',before:'old',after:'new'}]};}});
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.equal(s.calls.filter(call=>call[0]==='apply').length,0);assert.equal(s.session.state.phase,'error');
|
||||
assert.match(s.session.state.message,/^#9 Node 9: positive is connected to an upstream input\. Nothing was sent\./);
|
||||
assert.equal(s.session.state.outcomeUnknown,false);s.session.dispose();
|
||||
});
|
||||
test('a refusal while sending to All is reported by node and the other nodes are still sent',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;
|
||||
const s=await allSession(createSendSession,{apply:id=>{if(id==='9')throw Error('Workflow or prompt changed. Preview again.');return{targetId:id,values:{positive:'new'}};}});
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['2','9','10']);
|
||||
assert.equal(s.session.state.phase,'error');assert.equal(s.session.state.outcomeUnknown,false);
|
||||
assert.equal(s.session.state.message,'Sent to 2 of 3 nodes in Flow - ComfyUI. #9 Node 9: Workflow or prompt changed. Preview again.');s.session.dispose();
|
||||
});
|
||||
test('a lost reply on one node of All leaves the later nodes unsent and that outcome unknown',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;
|
||||
const s=await allSession(createSendSession,{apply:id=>{if(id==='9')throw Error('Bridge request timed out');return{targetId:id,values:{positive:'new'}};}});
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['2','9']);
|
||||
assert.equal(s.session.state.outcomeUnknown,true);
|
||||
assert.match(s.session.state.message,/^Sent to 1 of 3 nodes in Flow - ComfyUI\. No confirmation received from #9 Node 9; the update may have applied\. Not sent: #10 Node 10\. Check ComfyUI, then preview again\. Bridge request timed out$/);
|
||||
await s.session.apply();assert.equal(s.calls.filter(call=>call[0]==='apply').length,2);s.session.dispose();
|
||||
});
|
||||
test('closing the dialog during All stops after the current node',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;let release;
|
||||
const s=await allSession(createSendSession,{apply:id=>id==='2'?new Promise(done=>release=()=>done({targetId:id,values:{positive:'new'}})):{targetId:id,values:{positive:'new'}}});
|
||||
s.session.selectTarget(ALL_NODES);const pending=s.session.apply();
|
||||
for(let i=0;i<50 && !release;i++)await tick();assert.ok(release,'the first node was sent');
|
||||
s.session.dispose();release();await pending;
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['2']);
|
||||
});
|
||||
test('All accepts the revision our own write moved, but not a workflow changed before the run',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;
|
||||
// The host moves the graph version on every write: later nodes preview with the new revision.
|
||||
let version=4;
|
||||
const bumped=await allSession(createSendSession,{revision:'1:4',preview:id=>({ticket:`t-${id}`,revision:`1:${version}`,target:{id},fields:[{role:'positive',before:'old',after:'new'}]}),apply:id=>{version++;return{targetId:id,values:{positive:'new'}};}});
|
||||
bumped.session.selectTarget(ALL_NODES);await bumped.session.apply();
|
||||
assert.equal(bumped.session.state.message,'Sent to 3 of 3 nodes in Flow - ComfyUI.');bumped.session.dispose();
|
||||
const moved=await allSession(createSendSession,{revision:'1:4',preview:id=>({ticket:`t-${id}`,revision:'1:5',target:{id},fields:[{role:'positive',before:'old',after:'new'}]})});
|
||||
moved.session.selectTarget(ALL_NODES);await moved.session.apply();
|
||||
assert.match(moved.session.state.message,/workflow changed since this window was found/i);assert.equal(moved.calls.filter(call=>call[0]==='apply').length,0);moved.session.dispose();
|
||||
});
|
||||
test('a node whose text changed after the preview pass is refused, not overwritten',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;let edited=false;
|
||||
const s=await allSession(createSendSession,{preview:id=>({ticket:`t-${id}`,target:{id},fields:[{role:'positive',before:id==='10'&&edited?'typed in ComfyUI':'old',after:'new'}]}),apply:id=>{edited=true;return{targetId:id,values:{positive:'new'}};}});
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['2','9']);
|
||||
assert.match(s.session.state.message,/^Sent to 2 of 3 nodes in Flow - ComfyUI\. #10 Node 10: Its prompt changed since the preview\./);s.session.dispose();
|
||||
});
|
||||
test('Preview with All shows every node in order, and Apply sends fresh tickets that match the review',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;const s=await allSession(createSendSession);
|
||||
s.session.selectTarget(ALL_NODES);await s.session.preview();
|
||||
assert.equal(s.session.state.phase,'ready');assert.equal(s.session.state.preview,null);
|
||||
assert.deepEqual(s.session.state.previews.map(item=>item.label),['#2 Node 2','#9 Node 9','#10 Node 10']);
|
||||
assert.match(s.session.state.message,/Sending to all 3 eligible nodes in Flow - ComfyUI\. Review the complete changes, then apply\./);
|
||||
const first=s.session.state.previews[0].preview.ticket;
|
||||
await s.session.apply();
|
||||
// A held review may have outlived the host's 30 s tickets, so even the first node takes a fresh one.
|
||||
assert.deepEqual(order(s.calls),['preview 2','preview 9','preview 10','preview 2','apply 2','preview 9','apply 9','preview 10','apply 10']);
|
||||
assert.notEqual(s.calls.find(call=>call[0]==='apply')[2],first);assert.equal(s.session.state.phase,'success');
|
||||
s.session.selectTarget('2');assert.deepEqual(s.session.state.previews,[]);s.session.dispose();
|
||||
});
|
||||
test('the dialog offers All for two or more nodes and reviews each node under its name',async()=>{
|
||||
const {createPromptSender,ALL_NODES}=await source;const {createScope}=await import('../../web/js/shared/toolkit/core.js');
|
||||
const dom=fakeDOM();const scope=createScope();
|
||||
const {client}=allHost({nodes:['1','2']});
|
||||
try {
|
||||
createPromptSender({scope,clientFactory:()=>client}).open({positive:'new'});await tick();
|
||||
const dialog=dom.body.children.at(-1);const target=dom.find(dialog,node=>node.id==='prompt-sender-target');
|
||||
assert.deepEqual(target.children.map(option=>[option.value,option.textContent]),[['','Choose a destination node'],['1','#1 Node 1'],['2','#2 Node 2'],[ALL_NODES,'All 2 eligible nodes']]);
|
||||
assert.equal(target.value,'');target.value=ALL_NODES;target.dispatchEvent(new Event('change'));
|
||||
dom.find(dialog,node=>node.textContent==='Preview changes').dispatchEvent(new Event('click'));for(let i=0;i<5;i++)await tick();
|
||||
const sections=[];(function walk(node){if(node.className==='prompt-sender-node')sections.push(node);node.children.forEach(walk);})(dialog);
|
||||
assert.deepEqual(sections.map(section=>section.children[0].textContent),['#1 Node 1','#2 Node 2']);
|
||||
dom.find(dialog,node=>node.textContent==='Apply to ComfyUI').dispatchEvent(new Event('click'));for(let i=0;i<10;i++)await tick();
|
||||
assert.ok(dom.find(dialog,node=>node.textContent==='Sent to 2 of 2 nodes in Flow - ComfyUI.'));
|
||||
} finally {scope.dispose();dom.restore();}
|
||||
});
|
||||
test('an expired review still sends every node, and a held review checks the discovered revision first',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;let reviewing=true;
|
||||
// The host refuses every ticket taken during the review, as after its 30 s expiry.
|
||||
const s=await allSession(createSendSession,{preview:(id,calls)=>({ticket:`t-${id}-${reviewing?'old':calls.length}`,target:{id},fields:[{role:'positive',before:'old',after:'new'}]}),apply:id=>({targetId:id,values:{positive:'new'}})});
|
||||
const realApply=s.client.apply;s.client.apply=async(host,ticket,options)=>{if(ticket.endsWith('-old'))throw Error('Preview expired or unavailable; preview again');return realApply(host,ticket,options);};
|
||||
s.session.selectTarget(ALL_NODES);await s.session.preview();reviewing=false;await s.session.apply();
|
||||
assert.equal(s.session.state.message,'Sent to 3 of 3 nodes in Flow - ComfyUI.');s.session.dispose();
|
||||
const moved=await allSession(createSendSession,{revision:'1:4',preview:(id,calls)=>({ticket:`t-${id}`,revision:calls.length>3?'1:5':'1:4',target:{id},fields:[{role:'positive',before:'old',after:'new'}]})});
|
||||
moved.session.selectTarget(ALL_NODES);await moved.session.preview();assert.equal(moved.session.state.phase,'ready');await moved.session.apply();
|
||||
assert.match(moved.session.state.message,/^Sent to 0 of 3 nodes in Flow - ComfyUI\. #2 Node 2: The workflow changed since this window was found/);
|
||||
assert.equal(moved.calls.filter(call=>call[0]==='apply').length,0);moved.session.dispose();
|
||||
});
|
||||
test('a changed prepend or append after the review refuses that node',async()=>{
|
||||
const {createSendSession,ALL_NODES}=await source;let edited=false;
|
||||
const s=await allSession(createSendSession,{preview:id=>({ticket:`t-${id}`,target:{id},fields:[{role:'positive',before:'old',after:'new'}],composition:{prepend:id==='9'&&edited?'typed':'trigger',append:'',before:'trigger old',after:'trigger new'}}),apply:id=>{edited=true;return{targetId:id,values:{positive:'new'}};}});
|
||||
s.session.selectTarget(ALL_NODES);await s.session.apply();
|
||||
assert.deepEqual(s.calls.filter(call=>call[0]==='apply').map(call=>call[1]),['2','10']);
|
||||
assert.match(s.session.state.message,/^Sent to 2 of 3 nodes in Flow - ComfyUI\. #9 Node 9: Its prompt changed since the preview\./);s.session.dispose();
|
||||
});
|
||||
|
||||
@@ -128,6 +128,24 @@ test('invalid file shows the validator message; a refusal names its job; a lost
|
||||
assert.deepEqual(await uploadRestore(file, { ApiError, fetchUrl: async () => response(200, PREVIEW) }), PREVIEW);
|
||||
});
|
||||
|
||||
// #583: WebKit reads a FormData's file part by reference, possibly after a garbage collection; the page holds the form
|
||||
// until the upload is answered, then lets it go, whether the server staged it, refused it or the request failed.
|
||||
test('a review upload keeps its form referenced until the upload is answered', async () => {
|
||||
const { uploadRestore, uploadsInFlight } = await backups;
|
||||
const file = new File(['bytes'], 'b.db');
|
||||
for (const answer of [async () => response(200, PREVIEW), async () => response(400, { success: false, error: 'x' }),
|
||||
async () => { throw new TypeError('offline'); }]) {
|
||||
let held;
|
||||
const upload = uploadRestore(file, { ApiError, fetchUrl: async (url, init) => {
|
||||
held = uploadsInFlight.has(init.body) && init.body.get('database_file').size === file.size;
|
||||
return answer();
|
||||
} }).catch(() => {});
|
||||
await upload;
|
||||
assert.equal(held, true);
|
||||
assert.equal(uploadsInFlight.size, 0);
|
||||
}
|
||||
});
|
||||
|
||||
test('Cancel discards the stage through the api client and never throws', async () => {
|
||||
const { discardRestore, PREVIEW_URL } = await backups;
|
||||
const calls = [];
|
||||
|
||||
@@ -16,12 +16,22 @@ const OVERVIEW = Object.freeze({
|
||||
|
||||
test('metrics are the server counts and Not checked before any health check', async () => {
|
||||
const { metricValues } = await overviewModule;
|
||||
assert.deepEqual(metricValues(OVERVIEW), [
|
||||
assert.deepEqual(metricValues(OVERVIEW, 24), [
|
||||
[(1234).toLocaleString(), 'Positive prompts'], ['7', 'Negative variants'], ['24', 'Images'],
|
||||
['Not checked', 'Database integrity'],
|
||||
]);
|
||||
});
|
||||
|
||||
test('#555: the Images metric is the Images page count (the sidebar badge), not the indexed rows', async () => {
|
||||
const { metricValues } = await overviewModule;
|
||||
// The demo library: 110 indexed rows, 122 files on the Images page (unlinked ones included).
|
||||
const overview = { ...OVERVIEW, images: { indexed: 110, rated: 0 } };
|
||||
assert.deepEqual(metricValues(overview, 122)[2], [(122).toLocaleString(), 'Images']);
|
||||
// Not read yet (undefined) or failed (null): a dash, as on the workspace Overview, never the row count.
|
||||
assert.deepEqual(metricValues(overview, undefined)[2], ['–', 'Images']);
|
||||
assert.deepEqual(metricValues(overview, null)[2], ['–', 'Images']);
|
||||
});
|
||||
|
||||
test('metric 4 is the last health result', async () => {
|
||||
const { integrityText } = await overviewModule;
|
||||
assert.equal(integrityText(null), 'Not checked');
|
||||
@@ -150,7 +160,7 @@ test('statistics rows are the real runs, then the library totals, without rating
|
||||
rated_images: 20,
|
||||
};
|
||||
assert.deepEqual(categoryRows(statistics), [['Landscapes', (1200).toLocaleString()], ['Uncategorized', '3']]);
|
||||
assert.deepEqual(totalsRows(statistics), [['Prompts', '12'], ['Tags', '15'], ['Images', '24']]);
|
||||
assert.deepEqual(totalsRows(statistics), [['Prompt records', '12'], ['Tags on prompts', '15'], ['Image records', '24']]);
|
||||
});
|
||||
|
||||
/** A fake api answering each `get`/`post` from a queue per URL, recording the calls. */
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
// Thumbnail location (#320, D196–D201): the row's state and status line, the move review's rows, the unavailable
|
||||
// notice's text, the Activity views of a move and a prune, the prune scope's plan line and the Move library row.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const locationModule = import('../../web/js/workspace/system-thumbnail-location.mjs');
|
||||
const kindsModule = import('../../web/js/workspace/activity-kinds.mjs');
|
||||
const reviewModule = import('../../web/js/workspace/thumbnail-review.mjs');
|
||||
const libraryModule = import('../../web/js/workspace/library-location.mjs');
|
||||
|
||||
const SOURCE = Object.freeze({ mode: 'source', path: null, default_path: '/data/thumbnails', available: true,
|
||||
problem: null, cleanup_pending: 0 });
|
||||
const FOLDER = Object.freeze({ mode: 'folder', path: 'E:\\Thumbnails', default_path: '/data/thumbnails',
|
||||
available: true, problem: null, cleanup_pending: 0 });
|
||||
|
||||
const PLAN = Object.freeze({
|
||||
from: { mode: 'source', path: null, available: true },
|
||||
to: { mode: 'folder', path: 'E:\\Thumbnails', exists: true, will_create: false, default: false },
|
||||
totals: { sources: 3, files: 8512, bytes: 1.2 * 1024 ** 3, orphans: 41, temps: 2, links: 1, left_in_place: 1,
|
||||
too_long: 32, unlisted_sources: 0, unlisted_files: 0 },
|
||||
too_long_listed: ['Archive 2025 · a_thumb.webp', 'Archive 2025 · b_thumb.webp'],
|
||||
left_listed: ['Archive 2025 · notes.txt'],
|
||||
unlisted: [],
|
||||
free_bytes: 1.6 * 1024 ** 4,
|
||||
confirm_required: true,
|
||||
token: 't',
|
||||
});
|
||||
|
||||
test('the status line says where previews are now', async () => {
|
||||
const { statusText } = await locationModule;
|
||||
assert.equal(statusText(SOURCE), 'Now: inside each source.');
|
||||
assert.equal(statusText(FOLDER), 'Now: one folder, E:\\Thumbnails.');
|
||||
assert.equal(statusText({ ...FOLDER, path: '/data/thumbnails' }),
|
||||
'Now: one folder, /data/thumbnails. Using the default folder next to your library.');
|
||||
assert.equal(statusText({ ...FOLDER, available: false, problem: 'missing' }),
|
||||
'Now: one folder, E:\\Thumbnails · unavailable. Originals are shown.');
|
||||
assert.match(statusText({ ...SOURCE, cleanup_pending: 1 }), /Old thumbnails in 1 place wait to be removed/);
|
||||
assert.equal(statusText(undefined), '');
|
||||
assert.match(statusText(SOURCE, { mode: 'folder', path: '/data/thumbnails' }),
|
||||
/^Now: inside each source\. The suggested folder is next to your library/);
|
||||
assert.equal(statusText(SOURCE, { mode: 'folder', path: '/other' }), 'Now: inside each source.');
|
||||
});
|
||||
|
||||
test('Review move… is offered only for another location', async () => {
|
||||
const { choiceChanged } = await locationModule;
|
||||
assert.equal(choiceChanged(SOURCE, { mode: 'source' }), false);
|
||||
assert.equal(choiceChanged(SOURCE, { mode: 'folder', path: ' ' }), false);
|
||||
assert.equal(choiceChanged(SOURCE, { mode: 'folder', path: '/data/thumbnails' }), true);
|
||||
assert.equal(choiceChanged(FOLDER, { mode: 'folder', path: 'E:\\Thumbnails' }), false);
|
||||
assert.equal(choiceChanged(FOLDER, { mode: 'folder', path: 'F:\\Thumbs' }), true);
|
||||
assert.equal(choiceChanged(FOLDER, { mode: 'source' }), true);
|
||||
assert.equal(choiceChanged({ ...SOURCE, problem: 'invalid' }, { mode: 'source' }), true);
|
||||
assert.equal(choiceChanged(undefined, { mode: 'source' }), false);
|
||||
});
|
||||
|
||||
test('the review lists what moves, what is removed and what stays', async () => {
|
||||
const { reviewRows } = await locationModule;
|
||||
const rows = Object.fromEntries(reviewRows(PLAN));
|
||||
assert.equal(rows.From, 'Inside each source (3 sources)');
|
||||
assert.equal(rows.To, 'E:\\Thumbnails (empty folder)');
|
||||
assert.equal(rows.Thumbnails, '8,512 in 3 sources · 1.2 GB');
|
||||
assert.equal(rows['Free space'], '1638.4 GB free');
|
||||
assert.match(rows['Too long for Windows'][0], /^32 thumbnails cannot be stored there/);
|
||||
assert.deepEqual(rows['Too long for Windows'][1], [...PLAN.too_long_listed, 'and 30 more']);
|
||||
assert.deepEqual(rows['Removed after the move'][1],
|
||||
['41 thumbnails whose original is gone', '2 unfinished temporary files']);
|
||||
assert.equal(rows['Left in place'][0], '1 file that is not a thumbnail:');
|
||||
assert.equal(rows['Links skipped'], '1 link inside a thumbnails folder is not followed');
|
||||
});
|
||||
|
||||
test('the review of a move back names the folder and the removed sources', async () => {
|
||||
const { reviewRows } = await locationModule;
|
||||
const back = reviewRows({ ...PLAN, from: { mode: 'folder', path: '/data/thumbnails' },
|
||||
to: { mode: 'source', path: null }, unlisted: [{ key: 'old-1c22e0a4', files: 240 }],
|
||||
totals: { sources: 1, files: 1, bytes: 10 }, free_bytes: undefined });
|
||||
const rows = Object.fromEntries(back);
|
||||
assert.equal(rows.From, '/data/thumbnails');
|
||||
assert.equal(rows.To, 'Inside each source (1 source)');
|
||||
assert.equal(rows['Free space'], undefined);
|
||||
assert.match(rows['Removed after the move'][0], /The folder itself is removed once empty/);
|
||||
assert.deepEqual(rows['Removed after the move'][1],
|
||||
['1 source no longer listed: 240 thumbnails are removed; they are made again if it returns']);
|
||||
const fresh = Object.fromEntries(reviewRows({ ...PLAN, to: { mode: 'folder', path: '/d', default: true, will_create: true } }));
|
||||
assert.equal(fresh.To, '/d (new, next to your library)');
|
||||
});
|
||||
|
||||
test('the notice says why the folder is unavailable', async () => {
|
||||
const { noticeText } = await locationModule;
|
||||
assert.match(noticeText({ ...FOLDER, problem: 'missing' }), /^E:\\Thumbnails is missing\. Originals are shown/);
|
||||
assert.match(noticeText({ ...FOLDER, problem: 'unreachable' }), /can’t be reached/);
|
||||
assert.match(noticeText({ ...FOLDER, problem: 'marker-mismatch' }), /holds another thumbnail folder/);
|
||||
assert.match(noticeText({ ...FOLDER, problem: 'invalid' }), /config\.json can’t be used/);
|
||||
assert.match(noticeText({ ...FOLDER, problem: null }), /is being checked/);
|
||||
});
|
||||
|
||||
test('sizes are written as the server writes them', async () => {
|
||||
const { sizeText } = await locationModule;
|
||||
assert.equal(sizeText(512), '512 B');
|
||||
assert.equal(sizeText(85.2 * 1024 * 1024), '85.2 MB');
|
||||
});
|
||||
|
||||
test('a move and a prune show their own tiles in Activity', async () => {
|
||||
const { viewOf, movePhase, finishToast } = await kindsModule;
|
||||
const move = { kind: 'thumbnails', state: 'running', cancellable: true, scope: { mode: 'move', from: 'source' },
|
||||
counters: { moved: 3, skipped: 0, left: 1, errors: 2 }, status_line: '' };
|
||||
assert.deepEqual(viewOf(move).tiles.map(([group, label]) => label), ['Moved', 'Already there', 'Left in place', 'Errors']);
|
||||
assert.equal(viewOf(move).cancelLabel, 'Cancel move');
|
||||
assert.match(movePhase(move), /^Step 1 of 3: copying\. Thumbnails are still read from inside each source/);
|
||||
assert.match(movePhase({ ...move, cancellable: false }), /can no longer be cancelled/);
|
||||
assert.match(movePhase({ ...move, scope: { mode: 'move', phase: 'remove' } }), /Removing the old copies/);
|
||||
assert.match(movePhase({ ...move, state: 'finished' }), /1 file that is not a thumbnail was left in place/);
|
||||
assert.match(movePhase({ ...move, state: 'cancelled' }), /^Nothing changed/);
|
||||
const done = { ...move, state: 'finished', status_line: 'Thumbnails moved to E:\\T: 3 moved · old copies removed.' };
|
||||
assert.equal(finishToast(done), done.status_line);
|
||||
const prune = { kind: 'thumbnails', state: 'running', cancellable: true, scope: { mode: 'prune' },
|
||||
counters: { removed: 1, left: 0, errors: 0 } };
|
||||
assert.deepEqual(viewOf(prune).tiles.map(([, label]) => label), ['Removed', 'Left in place', 'Errors']);
|
||||
const generate = { kind: 'thumbnails', state: 'running', cancellable: true, scope: { mode: 'missing' },
|
||||
counters: { generated: 0, skipped: 0, errors: 0 } };
|
||||
assert.deepEqual(viewOf(generate).tiles.map(([, label]) => label), ['Generated', 'Skipped', 'Errors']);
|
||||
});
|
||||
|
||||
test('the prune scope reads its plan line from the server', async () => {
|
||||
const { planText, PLAN_FAILED, PRUNE_OPTION } = await reviewModule;
|
||||
assert.equal(PRUNE_OPTION[1], 'Remove thumbnails of removed sources');
|
||||
assert.equal(planText({ mode: 'prune', line: '2 sources no longer listed: 1,240 thumbnails (85.2 MB) are removed.' }),
|
||||
'2 sources no longer listed: 1,240 thumbnails (85.2 MB) are removed.');
|
||||
assert.equal(planText({ mode: 'prune' }), PLAN_FAILED);
|
||||
});
|
||||
|
||||
test('Move library says where thumbnails kept in one folder stay', async () => {
|
||||
const { reviewGrid } = await libraryModule;
|
||||
const base = { path: 'p', target: 't', size_bytes: 0, config_moves: true };
|
||||
assert.equal(reviewGrid(base).some(([term]) => term === 'Thumbnails'), false);
|
||||
const rows = Object.fromEntries(reviewGrid({ ...base, thumbnails_folder: '/data/thumbnails' }));
|
||||
assert.equal(rows.Thumbnails, 'Stay in /data/thumbnails. Move them in System › Sources & scans.');
|
||||
});
|
||||
@@ -91,6 +91,8 @@ class TestLocationRoute(LibraryAPITestCase):
|
||||
"size_bytes": os.path.getsize(self.api.db.model.db_path),
|
||||
"target_source": "default",
|
||||
"config_moves": True,
|
||||
# #320 D201: previews kept inside each source (no folder).
|
||||
"thumbnails_folder": None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
"""The thumbnail location setting (#320 PR 2): parsing, validation, source
|
||||
keys, the marker, the settings payloads and reverse containment.
|
||||
|
||||
Spelling rules are tested on both path flavors (``ntpath`` and
|
||||
``posixpath``) as text; filesystem rules run on the native one.
|
||||
"""
|
||||
|
||||
import json
|
||||
import ntpath
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.modules.setdefault("server", MagicMock())
|
||||
|
||||
from py.api import preview_location as pl # noqa: E402
|
||||
from py.api import preview_store # noqa: E402
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
class TestParse(unittest.TestCase):
|
||||
def test_missing_or_source_is_inside_each_source(self):
|
||||
for value in (None, {"mode": "source"}, {}):
|
||||
with self.subTest(value=value):
|
||||
parsed = pl.parse_location(value)
|
||||
self.assertEqual((parsed.mode, parsed.problem), ("source", None))
|
||||
|
||||
def test_a_folder_location_is_kept_as_spelled(self):
|
||||
parsed = pl.parse_location(
|
||||
{"mode": "folder", "path": "D:\\Thumbs", "id": "x", "install": "i"},
|
||||
flavor=ntpath,
|
||||
)
|
||||
self.assertEqual(
|
||||
(parsed.mode, parsed.path, parsed.id, parsed.problem),
|
||||
("folder", "D:\\Thumbs", "x", None),
|
||||
)
|
||||
|
||||
def test_invalid_values_never_fall_back_to_the_sources(self):
|
||||
for value in (
|
||||
"folder",
|
||||
{"mode": "elsewhere"},
|
||||
{"mode": "folder", "path": "relative", "id": "x"},
|
||||
{"mode": "folder", "path": "/abs"},
|
||||
{"mode": "folder", "path": "/abs", "id": ""},
|
||||
{"mode": "folder", "path": 7, "id": "x"},
|
||||
):
|
||||
with self.subTest(value=value):
|
||||
parsed = pl.parse_location(value, flavor=posixpath)
|
||||
self.assertEqual((parsed.mode, parsed.problem), ("folder", "invalid"))
|
||||
|
||||
|
||||
class TestForm(unittest.TestCase):
|
||||
def test_windows_spellings(self):
|
||||
cases = {
|
||||
"D:\\Thumbs": None,
|
||||
"\\\\server\\share\\thumbs": None,
|
||||
"C:thumbs": "location-not-absolute",
|
||||
"\\thumbs": "location-not-absolute",
|
||||
"thumbs": "location-not-absolute",
|
||||
"": "location-not-absolute",
|
||||
" D:\\Thumbs": "location-not-absolute",
|
||||
"\\\\?\\D:\\Thumbs": "location-device-path",
|
||||
"\\\\.\\D:\\Thumbs": "location-device-path",
|
||||
"D:\\CON\\x": "location-invalid-name",
|
||||
"D:\\com1.txt": "location-invalid-name",
|
||||
"D:\\thumbs.": "location-invalid-name",
|
||||
"D:\\thumbs ": "location-invalid-name",
|
||||
"D:\\a|b": "location-invalid-name",
|
||||
"D:\\a?b": "location-invalid-name",
|
||||
"D:\\a\x01b": "location-invalid-name",
|
||||
}
|
||||
for path, code in cases.items():
|
||||
with self.subTest(path=path):
|
||||
self.assertEqual(pl.form_problem(path, flavor=ntpath), code)
|
||||
|
||||
def test_posix_spellings(self):
|
||||
cases = {
|
||||
"/mnt/fast/thumbs": None,
|
||||
"mnt/thumbs": "location-not-absolute",
|
||||
"/mnt/nul": "location-invalid-name",
|
||||
"/mnt/a:b": "location-invalid-name",
|
||||
"//?/x": "location-device-path",
|
||||
}
|
||||
for path, code in cases.items():
|
||||
with self.subTest(path=path):
|
||||
self.assertEqual(pl.form_problem(path, flavor=posixpath), code)
|
||||
|
||||
def test_roots_and_home_are_too_broad(self):
|
||||
self.assertTrue(pl.is_too_broad("C:\\", home="C:\\Users\\a", flavor=ntpath))
|
||||
self.assertTrue(
|
||||
pl.is_too_broad("\\\\server\\share", home="C:\\Users\\a", flavor=ntpath)
|
||||
)
|
||||
self.assertTrue(
|
||||
pl.is_too_broad("C:\\users\\A", home="C:\\Users\\a", flavor=ntpath)
|
||||
)
|
||||
self.assertTrue(pl.is_too_broad("/", home="/home/a", flavor=posixpath))
|
||||
self.assertTrue(pl.is_too_broad("/home/a/", home="/home/a", flavor=posixpath))
|
||||
self.assertFalse(
|
||||
pl.is_too_broad("/home/a/thumbs", home="/home/a", flavor=posixpath)
|
||||
)
|
||||
|
||||
def test_long_windows_paths_are_refused_without_long_paths(self):
|
||||
short = "D:\\" + "a" * 100
|
||||
long = "D:\\" + "a" * 200
|
||||
self.assertFalse(pl.too_long_for_windows(short, long_paths=False))
|
||||
self.assertTrue(pl.too_long_for_windows(long, long_paths=False))
|
||||
self.assertFalse(pl.too_long_for_windows(long, long_paths=True))
|
||||
|
||||
|
||||
class TestSourceKeys(unittest.TestCase):
|
||||
def test_slug(self):
|
||||
self.assertEqual(pl.slug("ComfyUI Output"), "comfyui-output")
|
||||
self.assertEqual(pl.slug("Ünïcödé_Fôlder"), "unicode_folder")
|
||||
self.assertEqual(pl.slug(""), "source")
|
||||
self.assertEqual(pl.slug("!!!"), "source")
|
||||
self.assertEqual(len(pl.slug("x" * 80)), 24)
|
||||
|
||||
def test_keys_are_a_slug_plus_a_hash(self):
|
||||
key = pl.source_key("D:\\AI\\Output", "d:\\ai\\output", {})
|
||||
self.assertRegex(key, r"^output-[0-9a-f]{8}$")
|
||||
self.assertRegex(pl.source_key("D:\\", "d:\\", {}), r"^source-[0-9a-f]{8}$")
|
||||
self.assertRegex(
|
||||
pl.source_key("\\\\nas\\share\\Out", "\\\\nas\\share\\out", {}),
|
||||
r"^out-[0-9a-f]{8}$",
|
||||
)
|
||||
|
||||
def test_a_taken_key_grows_its_hash(self):
|
||||
first = pl.source_key("/a/out", "/a/out", {})
|
||||
taken = {first: "/someone/else"}
|
||||
second = pl.source_key("/a/out", "/a/out", taken)
|
||||
self.assertRegex(second, r"^out-[0-9a-f]{12}$")
|
||||
taken[second] = "/another"
|
||||
self.assertRegex(
|
||||
pl.source_key("/a/out", "/a/out", taken), r"^out-[0-9a-f]{16}$"
|
||||
)
|
||||
self.assertEqual(pl.source_key("/a/out", "/a/out", {first: "/a/out"}), first)
|
||||
|
||||
def test_lookup_by_spelling_identity_then_file_id(self):
|
||||
keys = {
|
||||
"out-0123abcd": {
|
||||
"path": "D:\\Out",
|
||||
"identity": "d:\\out",
|
||||
"dev_ino": [1, 2],
|
||||
},
|
||||
"lib-0123abcd": {"path": "E:\\Lib", "identity": "e:\\lib", "dev_ino": None},
|
||||
}
|
||||
self.assertEqual(pl.find_key(keys, "D:\\Out", "x", None), "out-0123abcd")
|
||||
self.assertEqual(pl.find_key(keys, "Z:\\Out", "d:\\out", None), "out-0123abcd")
|
||||
# A letter spelling and its UNC spelling: the same file id.
|
||||
self.assertEqual(
|
||||
pl.find_key(keys, "\\\\nas\\out", "\\\\nas\\out", (1, 2)), "out-0123abcd"
|
||||
)
|
||||
self.assertIsNone(pl.find_key(keys, "F:\\New", "f:\\new", (9, 9)))
|
||||
|
||||
|
||||
class TestMarker(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.folder = self._tmp.name
|
||||
|
||||
def test_init_creates_once_and_adopts_after(self):
|
||||
first = pl.init_folder(self.folder, created_by_pm=True)
|
||||
self.assertEqual(pl.init_folder(self.folder), first)
|
||||
marker = pl.read_marker(self.folder)
|
||||
self.assertEqual(marker["id"], first)
|
||||
self.assertTrue(marker["created_by_pm"])
|
||||
self.assertEqual(marker["version"], 1)
|
||||
|
||||
def test_writes_merge_what_another_writer_added(self):
|
||||
pl.init_folder(self.folder)
|
||||
pl.write_marker(
|
||||
self.folder, lambda m: m["keys"].update({"a-0123abcd": {"install": "1"}})
|
||||
)
|
||||
pl.write_marker(
|
||||
self.folder, lambda m: m["keys"].update({"b-0123abcd": {"install": "2"}})
|
||||
)
|
||||
self.assertEqual(
|
||||
sorted(pl.read_marker(self.folder)["keys"]), ["a-0123abcd", "b-0123abcd"]
|
||||
)
|
||||
# No temp file is left beside it.
|
||||
self.assertEqual(os.listdir(self.folder), [pl.MARKER_NAME])
|
||||
|
||||
def test_a_folder_without_the_marker_is_missing(self):
|
||||
self.assertEqual(pl.folder_state(self.folder, "x")[:2], (False, "missing"))
|
||||
missing = os.path.join(self.folder, "gone")
|
||||
self.assertEqual(pl.folder_state(missing, "x")[:2], (False, "missing"))
|
||||
marker_id = pl.init_folder(self.folder)
|
||||
self.assertEqual(pl.folder_state(self.folder, marker_id)[:2], (True, None))
|
||||
self.assertEqual(
|
||||
pl.folder_state(self.folder, "other")[:2], (False, "marker-mismatch")
|
||||
)
|
||||
|
||||
@unittest.skipUnless(hasattr(os, "symlink"), "needs symlinks")
|
||||
def test_a_linked_folder_is_not_available(self):
|
||||
real = os.path.join(self.folder, "real")
|
||||
os.mkdir(real)
|
||||
marker_id = pl.init_folder(real)
|
||||
link = os.path.join(self.folder, "link")
|
||||
try:
|
||||
os.symlink(real, link, target_is_directory=True)
|
||||
except OSError:
|
||||
self.skipTest("symlinks not permitted")
|
||||
self.assertEqual(pl.folder_state(link, marker_id)[:2], (False, "invalid"))
|
||||
|
||||
|
||||
class TestValidation(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.base = os.path.realpath(self._tmp.name)
|
||||
self.source = os.path.join(self.base, "output")
|
||||
self.node = os.path.join(self.base, "node")
|
||||
os.mkdir(self.source)
|
||||
os.mkdir(self.node)
|
||||
self.context = pl.Context(
|
||||
roots=(self.source,),
|
||||
node_folder=self.node,
|
||||
default_path=os.path.join(self.node, "data", "thumbnails"),
|
||||
home=os.path.join(self.base, "home"),
|
||||
environ={},
|
||||
long_paths=True,
|
||||
folder_paths=None,
|
||||
)
|
||||
|
||||
def problem(self, path, purpose="move", **changes):
|
||||
context = self.context
|
||||
if changes:
|
||||
context = pl.Context(**{**context.__dict__, **changes})
|
||||
return pl.location_problem(path, purpose=purpose, context=context)
|
||||
|
||||
def test_an_empty_new_folder_beside_the_sources_is_accepted(self):
|
||||
self.assertIsNone(self.problem(os.path.join(self.base, "thumbs")))
|
||||
os.mkdir(os.path.join(self.base, "empty"))
|
||||
Path(self.base, "empty", "Thumbs.db").write_bytes(b"")
|
||||
self.assertIsNone(self.problem(os.path.join(self.base, "empty")))
|
||||
|
||||
def test_codes(self):
|
||||
cases = {
|
||||
"relative": "location-not-absolute",
|
||||
os.path.join(self.source, "thumbs"): "location-served",
|
||||
os.path.join(self.node, "thumbs"): "location-node-folder",
|
||||
self.base: "location-contains-source",
|
||||
os.path.join(self.base, "no", "parent"): "location-parent-missing",
|
||||
}
|
||||
for path, code in cases.items():
|
||||
with self.subTest(path=path):
|
||||
self.assertEqual(self.problem(path), code)
|
||||
|
||||
def test_the_default_may_sit_in_the_node_folder_and_make_its_parents(self):
|
||||
self.assertIsNone(self.problem(self.context.default_path))
|
||||
|
||||
def test_the_folder_in_use_is_the_same(self):
|
||||
folder = os.path.join(self.base, "thumbs")
|
||||
os.mkdir(folder)
|
||||
self.assertEqual(self.problem(folder, current_path=folder), "location-same")
|
||||
|
||||
def test_a_folder_with_other_files_is_not_empty_unless_it_is_ours(self):
|
||||
folder = os.path.join(self.base, "full")
|
||||
os.mkdir(folder)
|
||||
Path(folder, "photo.jpg").write_bytes(b"x")
|
||||
self.assertEqual(self.problem(folder), "location-not-empty")
|
||||
pl.init_folder(folder)
|
||||
self.assertIsNone(self.problem(folder))
|
||||
|
||||
@unittest.skipIf(os.name == "nt" or os.geteuid() == 0, "needs POSIX modes")
|
||||
def test_a_folder_that_refuses_writes(self):
|
||||
folder = os.path.join(self.base, "ro")
|
||||
os.mkdir(folder)
|
||||
os.chmod(folder, 0o500)
|
||||
self.addCleanup(os.chmod, folder, 0o700)
|
||||
self.assertEqual(self.problem(folder), "location-not-writable")
|
||||
|
||||
def test_a_stat_that_does_not_answer_is_unavailable(self):
|
||||
release = threading.Event()
|
||||
self.addCleanup(release.set)
|
||||
|
||||
def stuck(*_args):
|
||||
release.wait(10)
|
||||
|
||||
with patch.object(pl, "_filesystem_problem", stuck):
|
||||
code = pl.location_problem(
|
||||
os.path.join(self.base, "thumbs"),
|
||||
purpose="move",
|
||||
context=self.context,
|
||||
timeout=0.1,
|
||||
)
|
||||
self.assertEqual(code, "location-unavailable")
|
||||
|
||||
def test_a_long_windows_path(self):
|
||||
path = os.path.join(self.base, "t" * 220)
|
||||
self.assertEqual(self.problem(path, long_paths=False), "location-path-too-long")
|
||||
|
||||
def test_every_code_has_a_message_without_a_path(self):
|
||||
for code, (status, text) in pl.MESSAGES.items():
|
||||
with self.subTest(code=code):
|
||||
self.assertIn(status, (400, 409, 507))
|
||||
self.assertTrue(text.endswith("."))
|
||||
|
||||
|
||||
class TestNoLegacySaveCallers(unittest.TestCase):
|
||||
def test_save_to_file_has_no_callers(self):
|
||||
offenders = []
|
||||
for folder in ("py", "utils", "database"):
|
||||
for path in (REPO / folder).rglob("*.py"):
|
||||
for number, line in enumerate(
|
||||
path.read_text(encoding="utf-8").splitlines(), 1
|
||||
):
|
||||
code = line.split("#", 1)[0]
|
||||
if (
|
||||
re.search(r"\bsave_to_file\(", code)
|
||||
and "def save_to_file" not in code
|
||||
):
|
||||
if "PromptManagerConfig.save_to_file('" in line:
|
||||
continue # its own docstring example
|
||||
offenders.append(f"{path.relative_to(REPO)}:{number}")
|
||||
self.assertEqual(offenders, [])
|
||||
|
||||
|
||||
try:
|
||||
from tests.test_api_images import ImageAPITestCase
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_images import ImageAPITestCase
|
||||
|
||||
|
||||
class TestLocationRoutes(ImageAPITestCase):
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.addCleanup(preview_store.configure, None)
|
||||
|
||||
def use_folder(self):
|
||||
folder = self.comfy_dir / "thumbs"
|
||||
folder.mkdir()
|
||||
marker_id = pl.init_folder(str(folder), created_by_pm=True)
|
||||
preview_store.configure(
|
||||
{"mode": "folder", "path": str(folder), "id": marker_id}
|
||||
)
|
||||
return folder
|
||||
|
||||
async def test_settings_get_names_the_location_with_its_path(self):
|
||||
folder = self.use_folder()
|
||||
resp = await self.client.get("/prompt_manager/settings")
|
||||
location = (await resp.json())["thumbnail_location"]
|
||||
self.assertEqual(location["mode"], "folder")
|
||||
self.assertEqual(location["path"], str(folder))
|
||||
self.assertTrue(location["available"])
|
||||
self.assertIsNone(location["problem"])
|
||||
self.assertIn("default_path", location)
|
||||
|
||||
async def test_settings_post_never_changes_the_location(self):
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/settings",
|
||||
json={"thumbnail_location": {"mode": "folder", "path": "/x"}},
|
||||
)
|
||||
self.assertEqual(resp.status, 409)
|
||||
body = await resp.json()
|
||||
self.assertEqual(body["code"], "location-needs-move")
|
||||
self.assertNotIn("/x", body["error"])
|
||||
|
||||
async def test_status_has_the_location_but_no_path(self):
|
||||
folder = self.use_folder()
|
||||
resp = await self.client.get("/prompt_manager/images/thumbnails/status")
|
||||
body = await resp.text()
|
||||
self.assertNotIn(str(folder), body)
|
||||
self.assertEqual(
|
||||
json.loads(body)["location"],
|
||||
{"mode": "folder", "available": True, "problem": None},
|
||||
)
|
||||
|
||||
async def test_a_source_inside_or_around_the_folder_is_refused(self):
|
||||
from py.api import sources
|
||||
|
||||
folder = self.use_folder()
|
||||
inside = folder / "sub"
|
||||
inside.mkdir()
|
||||
with patch(
|
||||
"py.config.GalleryConfig.gallery_root_problem",
|
||||
classmethod(lambda cls, path: (None, "")),
|
||||
):
|
||||
for path in (inside, folder, self.comfy_dir):
|
||||
with self.subTest(path=path):
|
||||
code, reason = sources.source_problem(str(path))
|
||||
self.assertEqual(code, "holds-thumbnails")
|
||||
self.assertEqual(reason, pl.HOLDS_THUMBNAILS_REASON)
|
||||
self.assertEqual(sources.source_problem(str(self.output_dir))[0], None)
|
||||
|
||||
async def test_a_library_move_into_the_folder_is_refused(self):
|
||||
from py.api import library_location as routes
|
||||
|
||||
folder = self.use_folder()
|
||||
self.assertTrue(routes._inside_thumbnail_folder(str(folder / "lib")))
|
||||
self.assertFalse(routes._inside_thumbnail_folder(str(self.output_dir)))
|
||||
|
||||
def test_startup_makes_a_location_inside_a_source_invalid(self):
|
||||
inside = self.output_dir / "thumbs"
|
||||
inside.mkdir()
|
||||
marker_id = pl.init_folder(str(inside))
|
||||
from py.config import PromptManagerConfig
|
||||
|
||||
value = {"mode": "folder", "path": str(inside), "id": marker_id}
|
||||
with patch.object(PromptManagerConfig, "THUMBNAIL_LOCATION", value):
|
||||
location = self.api.start_preview_location()
|
||||
self.assertEqual((location.available, location.problem), (False, "invalid"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -908,20 +908,20 @@ class TestWorkspaceImages(ImageAPITestCase):
|
||||
|
||||
async def test_one_cards_parse_error_spares_the_page(self):
|
||||
prompt = self._save_prompt("a fox")
|
||||
for name, mtime in (("bad.png", 200), ("ok.png", 100)):
|
||||
# Cards are read on the image pool (#523), in no fixed order: the
|
||||
# parser fails on bad.png's own record (seed 6), not on a call count.
|
||||
for name, mtime, seed in (("bad.png", 200, 6), ("ok.png", 100, 5)):
|
||||
path = records_png(
|
||||
self.output_dir / name, seeded_records("a fox", 5), mtime
|
||||
self.output_dir / name, seeded_records("a fox", seed), mtime
|
||||
)
|
||||
self.api.db.link_image_to_prompt(prompt, str(path))
|
||||
real_parse = header_params_of_images()
|
||||
|
||||
def parsing(info):
|
||||
if parsing.calls == 0:
|
||||
parsing.calls += 1
|
||||
if '"seed": 6' in info["prompt"]:
|
||||
raise RuntimeError("unexpected parser bug")
|
||||
return real_parse(info)
|
||||
|
||||
parsing.calls = 0
|
||||
with (
|
||||
patch("py.api.images.header_params", side_effect=parsing),
|
||||
self.assertLogs(self.api.logger, "WARNING") as logged,
|
||||
|
||||
@@ -146,6 +146,18 @@ class TestPreviewQueueRefusedAtShutdown(unittest.TestCase):
|
||||
|
||||
|
||||
class TestMapParallelRefusedAtShutdown(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# The job pool is long-lived (#523): start each test without one, so
|
||||
# the patched pool class is the one created.
|
||||
self.assertTrue(parallel.shutdown_job_pool(30))
|
||||
self.addCleanup(parallel.shutdown_job_pool, 30)
|
||||
|
||||
def test_the_refused_pool_is_not_reused(self):
|
||||
with mock.patch.object(parallel, "ThreadPoolExecutor", _ShutPool):
|
||||
parallel.map_parallel(lambda n: n, [1, 2], 2)
|
||||
self.assertFalse(parallel.job_pool_started())
|
||||
self.assertEqual(parallel.map_parallel(lambda n: n + 1, [1, 2], 2), [2, 3])
|
||||
|
||||
def test_a_shut_pool_runs_the_items_on_the_calling_thread(self):
|
||||
records = _capture(self, "prompt_manager.parallel")
|
||||
with mock.patch.object(parallel, "ThreadPoolExecutor", _ShutPool):
|
||||
|
||||
@@ -564,6 +564,66 @@ class TestUndecidedGraph(unittest.TestCase):
|
||||
self.assertEqual(decision.review, "untraced")
|
||||
|
||||
|
||||
def _search_list(text="fox", side="positive"):
|
||||
"""f22: a PromptSearchList batch (no PromptManager node) feeds CLIPTextEncode."""
|
||||
graph = parity.build("f22")["prompt"]
|
||||
graph["10"]["inputs"]["text"] = text
|
||||
if side == "negative":
|
||||
# The search feeds the negative; the positive is typed text.
|
||||
graph["6"]["inputs"]["text"] = parity.POS
|
||||
graph["7"]["inputs"]["text"] = ["10", 0]
|
||||
return graph
|
||||
|
||||
|
||||
class TestSearchTextIsNeverAPrompt(unittest.TestCase):
|
||||
"""#548: the PromptSearchList query is a search, never the prompt."""
|
||||
|
||||
def test_the_search_text_is_never_linked_or_stored(self):
|
||||
for policy in Policy:
|
||||
with self.subTest(policy=policy):
|
||||
decision = _decide(
|
||||
_graph(_search_list()), policy=policy, lookup=_Lookup()
|
||||
)
|
||||
self.assertEqual(decision.kind, "never")
|
||||
self.assertIsNone(decision.text)
|
||||
self.assertEqual(decision.unlinked_status, "unresolved")
|
||||
self.assertTrue(decision.try_recorded_run)
|
||||
|
||||
def test_a_search_text_from_a_primitive_is_never_linked(self):
|
||||
graph = _search_list()
|
||||
graph["10"]["inputs"]["text"] = ["30", 0]
|
||||
graph["30"] = {"class_type": "PrimitiveString", "inputs": {"value": "fox"}}
|
||||
for policy in Policy:
|
||||
with self.subTest(policy=policy):
|
||||
decision = _decide(_graph(graph), policy=policy, lookup=_Lookup())
|
||||
self.assertEqual(decision.kind, "never")
|
||||
self.assertIsNone(decision.text)
|
||||
|
||||
def test_an_empty_search_text_is_unchanged(self):
|
||||
"""Regression guard (passes on the base): nothing to exclude."""
|
||||
for policy in Policy:
|
||||
with self.subTest(policy=policy):
|
||||
decision = _decide(
|
||||
_graph(_search_list("")), policy=policy, lookup=_Lookup()
|
||||
)
|
||||
self.assertEqual((decision.kind, decision.text), ("never", None))
|
||||
|
||||
def test_without_a_sampler_any_search_list_query_is_excluded(self):
|
||||
graph = _search_list()
|
||||
del graph["3"]
|
||||
self.assertTrue(file_prompt._search_text(graph, " fox "))
|
||||
self.assertFalse(file_prompt._search_text(graph, "a fox"))
|
||||
self.assertFalse(file_prompt._search_text(None, "fox"))
|
||||
|
||||
def test_a_search_on_the_negative_keeps_the_typed_positive(self):
|
||||
"""Regression guard (passes on the base): only the positive's source
|
||||
is excluded, so the legacy answer from typed text still links."""
|
||||
graph = _search_list(side="negative")
|
||||
decision = _decide(_graph(graph), lookup=_Lookup())
|
||||
self.assertEqual((decision.kind, decision.text), ("positive", parity.POS))
|
||||
self.assertEqual(decision.source, "legacy-parser")
|
||||
|
||||
|
||||
class TestStepsWithoutAnUndecidedGraph(unittest.TestCase):
|
||||
def test_step_9_ambiguous_parameters_only(self):
|
||||
meta = {
|
||||
|
||||
@@ -392,7 +392,10 @@ class TestStartStop(unittest.TestCase):
|
||||
self.assertLogs("prompt_manager.image_monitor", "WARNING") as logs,
|
||||
):
|
||||
self.assertFalse(self.monitor.stop_monitoring(timeout=5.0))
|
||||
self.assertEqual(fake.calls[-1], ("join", 0.0))
|
||||
# The stop runs on its own thread and gets 0 s too: it may not have
|
||||
# returned yet, and then nothing is joined at all.
|
||||
joins = [call for call in fake.calls if call[0] == "join"]
|
||||
self.assertTrue(all(call == ("join", 0.0) for call in joins), fake.calls)
|
||||
self.assertIn(
|
||||
"watchdog observer still running after 0.0 s", "\n".join(logs.output)
|
||||
)
|
||||
@@ -401,6 +404,80 @@ class TestStartStop(unittest.TestCase):
|
||||
def test_stop_monitoring_without_an_observer_returns_true(self):
|
||||
self.assertTrue(self.monitor.stop_monitoring())
|
||||
|
||||
def test_threads_the_observer_starts_are_daemons(self):
|
||||
# watchdog 7 on Windows starts a plain threading.Thread per watch in
|
||||
# observer.start(); a non-daemon one holds the interpreter's exit
|
||||
# before any atexit callback runs (#561).
|
||||
fake = ReaderObserver()
|
||||
self.addCleanup(fake.release.set)
|
||||
self._start_with(fake)
|
||||
self.assertIsNotNone(fake.reader)
|
||||
self.assertTrue(fake.reader.daemon)
|
||||
self.assertFalse(threading.current_thread().daemon)
|
||||
|
||||
def test_an_observer_that_fails_to_start_on_its_thread_is_reported(self):
|
||||
fake = FakeObserver()
|
||||
fake.start = MagicMock(side_effect=OSError("handle refused"))
|
||||
with self.assertLogs("prompt_manager.image_monitor", "ERROR") as logs:
|
||||
with patch.object(im_mod, "Observer", return_value=fake):
|
||||
self.monitor.start_monitoring([self.tmp.name])
|
||||
self.assertIn("handle refused", "\n".join(logs.output))
|
||||
self.assertFalse(self.monitor.is_monitoring)
|
||||
|
||||
def test_an_observer_stop_that_never_returns_is_bounded(self):
|
||||
# watchdog's observer.stop() joins its emitters (and, in 7.x on
|
||||
# Windows, their reader threads) without a limit (#561).
|
||||
fake = StuckStopObserver()
|
||||
self.addCleanup(fake.release.set)
|
||||
self._start_with(fake)
|
||||
with self.assertLogs("prompt_manager.image_monitor", "WARNING") as logs:
|
||||
self.assertFalse(self.monitor.stop_monitoring(timeout=0.05))
|
||||
self.assertIn("watchdog observer still running", "\n".join(logs.output))
|
||||
self.assertTrue(fake.entered.wait(30))
|
||||
self.assertFalse(self.monitor.is_monitoring)
|
||||
# A second stop waits for the stop already running; it never calls
|
||||
# observer.stop() a second time.
|
||||
fake.release.set()
|
||||
self.assertTrue(self.monitor.stop_monitoring())
|
||||
self.assertEqual(fake.stop_calls, 1)
|
||||
self.assertFalse(self.monitor._unfinished)
|
||||
|
||||
|
||||
class ReaderObserver(FakeObserver):
|
||||
"""A FakeObserver whose start() starts a plain thread, as watchdog 7's
|
||||
Windows emitter does for its ReadDirectoryChangesW reader."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.release = threading.Event()
|
||||
self.reader = None
|
||||
|
||||
def start(self):
|
||||
self.reader = threading.Thread(target=self.release.wait)
|
||||
self.reader.start()
|
||||
super().start()
|
||||
|
||||
def stop(self):
|
||||
self.release.set()
|
||||
self.reader.join(30)
|
||||
super().stop()
|
||||
|
||||
|
||||
class StuckStopObserver(FakeObserver):
|
||||
"""A FakeObserver whose stop() does not return until ``release``."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.entered = threading.Event()
|
||||
self.release = threading.Event()
|
||||
self.stop_calls = 0
|
||||
|
||||
def stop(self):
|
||||
self.stop_calls += 1
|
||||
self.entered.set()
|
||||
self.release.wait(30)
|
||||
super().stop()
|
||||
|
||||
|
||||
class FakeEvent:
|
||||
def __init__(self, path, is_directory=False):
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
"""Image work runs on PromptManager's own pools, never the default executor (#523).
|
||||
|
||||
glibc keeps each thread's malloc arena at its high-water mark, so image work
|
||||
spread over the default executor's threads (min(32, cpus + 4)) kept an
|
||||
image-sized buffer per thread. Interactive image work (card facts, inspector
|
||||
reads, view previews) runs on the fixed interactive image pool
|
||||
(``pm-image``); jobs (scan metadata, thumbnail jobs, capture previews) on the
|
||||
long-lived Worker threads pool (``pm-worker``). After a scan or thumbnail
|
||||
job, glibc's free pages are released once (``malloc_trim``).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from unittest.mock import patch
|
||||
|
||||
import py.api as api_package
|
||||
import py.api.images as images_module
|
||||
import py.api.inspection_service as inspection_service
|
||||
import py.api.thumbnail_auto as auto
|
||||
from utils import parallel
|
||||
|
||||
try:
|
||||
from tests.test_api_images import make_png
|
||||
from tests.test_thumbnail_auto import WAIT, AutoThumbnailCase, _entry
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_images import make_png
|
||||
from test_thumbnail_auto import WAIT, AutoThumbnailCase, _entry
|
||||
|
||||
IMAGE_THREAD = auto.POOL_PREFIX + "_"
|
||||
JOB_THREAD = parallel.JOB_THREAD_PREFIX + "_"
|
||||
|
||||
|
||||
class ImagePoolCase(AutoThumbnailCase):
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.assertTrue(parallel.shutdown_job_pool(WAIT))
|
||||
self.addCleanup(parallel.shutdown_job_pool, WAIT)
|
||||
self.threads = []
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def spy(self, owner, name):
|
||||
"""Record the thread every call of ``owner.name`` runs on."""
|
||||
original = getattr(owner, name)
|
||||
|
||||
def recording(*args, **kwargs):
|
||||
with self.lock:
|
||||
self.threads.append(threading.current_thread().name)
|
||||
return original(*args, **kwargs)
|
||||
|
||||
patcher = patch.object(owner, name, recording)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def assert_threads(self, prefix):
|
||||
self.assertTrue(self.threads, "the spied call never ran")
|
||||
self.assertTrue(
|
||||
all(name.startswith(prefix) for name in self.threads), self.threads
|
||||
)
|
||||
|
||||
async def run_job(self, path, body=None):
|
||||
response = await self.client.post(path, json=body or {})
|
||||
self.assertEqual(response.status, 202, await response.text())
|
||||
job = self.api._jobs.get((await response.json())["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
return job
|
||||
|
||||
|
||||
class TestInteractiveImagePool(ImagePoolCase):
|
||||
async def test_workspace_card_facts_are_read_on_the_image_pool(self):
|
||||
for index in range(5):
|
||||
make_png(self.output_dir / f"c{index}.png", size=(32, 32))
|
||||
self.spy(images_module, "_card_header")
|
||||
response = await self.client.get(
|
||||
"/prompt_manager/images/output?mode=workspace&limit=100"
|
||||
)
|
||||
self.assertEqual(response.status, 200, await response.text())
|
||||
images = (await response.json())["images"]
|
||||
self.assertEqual([image["width"] for image in images], [32] * 5)
|
||||
self.assert_threads(IMAGE_THREAD)
|
||||
self.assertEqual(len(self.threads), 5)
|
||||
|
||||
async def test_cached_card_facts_need_no_image_pool_trip(self):
|
||||
make_png(self.output_dir / "c.png", size=(32, 32))
|
||||
url = "/prompt_manager/images/output?mode=workspace&limit=100"
|
||||
await (await self.client.get(url)).read()
|
||||
self.spy(images_module, "_card_header")
|
||||
response = await self.client.get(url)
|
||||
self.assertEqual((await response.json())["images"][0]["width"], 32)
|
||||
self.assertEqual(self.threads, [])
|
||||
|
||||
async def test_the_inspector_reads_on_the_image_pool(self):
|
||||
make_png(self.output_dir / "i.png", size=(32, 32))
|
||||
self.spy(inspection_service, "read_inspection_file")
|
||||
response = await self.client.get("/prompt_manager/images/inspect/i.png?root=0")
|
||||
self.assertEqual(response.status, 200, await response.text())
|
||||
self.assert_threads(IMAGE_THREAD)
|
||||
|
||||
async def test_view_previews_and_card_reads_share_three_threads(self):
|
||||
names = [f"v{i}.png" for i in range(auto.ON_DEMAND_LIMIT + 3)]
|
||||
for name in names:
|
||||
make_png(self.output_dir / name, size=(64, 64))
|
||||
calls = []
|
||||
original = self.api._generate_one
|
||||
|
||||
def recording(*args, **kwargs):
|
||||
calls.append(threading.current_thread().name)
|
||||
return original(*args, **kwargs)
|
||||
|
||||
self.api._generate_one = recording
|
||||
for name in names:
|
||||
url = _entry(self.api, self.output_dir, name)["thumbnail_url"]
|
||||
await (await self.client.get(url)).read()
|
||||
self.assertEqual(len(calls), len(names))
|
||||
self.assertTrue(all(name.startswith(IMAGE_THREAD) for name in calls), calls)
|
||||
self.assertLessEqual(len(self.api._image_pool_threads()), auto.IMAGE_POOL_SIZE)
|
||||
|
||||
def test_map_image_keeps_order_and_runs_inline_on_a_pool_thread(self):
|
||||
self.assertEqual(self.api._map_image(lambda n: n * 2, [3, 1, 2]), [6, 2, 4])
|
||||
nested = self.api._image_pool().submit(
|
||||
self.api._map_image, lambda _: threading.current_thread().name, [1]
|
||||
)
|
||||
(name,) = nested.result(WAIT)
|
||||
self.assertTrue(name.startswith(IMAGE_THREAD), name)
|
||||
|
||||
def test_map_image_after_a_close_runs_on_the_calling_thread(self):
|
||||
pool = self.api._image_pool()
|
||||
pool.shutdown(wait=True)
|
||||
here = threading.current_thread().name
|
||||
with patch.object(self.api, "_image_pool", return_value=pool):
|
||||
names = self.api._map_image(
|
||||
lambda _: threading.current_thread().name, [1, 2]
|
||||
)
|
||||
self.assertEqual(names, [here, here])
|
||||
|
||||
async def test_after_the_final_close_no_image_thread_starts(self):
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
None, lambda: self.api.close_auto_thumbnails(final=True)
|
||||
)
|
||||
self.assertIsNone(self.api._image_pool())
|
||||
here = threading.current_thread().name
|
||||
names = self.api._map_image(lambda _: threading.current_thread().name, [1])
|
||||
self.assertEqual(names, [here])
|
||||
name = await self.api._run_image(lambda: threading.current_thread().name)
|
||||
self.assertFalse(name.startswith(IMAGE_THREAD), name)
|
||||
self.assertEqual(self.api._image_pool_threads(), [])
|
||||
|
||||
def test_map_image_runs_items_a_shutdown_cancelled_here(self):
|
||||
entered = threading.Barrier(auto.IMAGE_POOL_SIZE + 1)
|
||||
release = threading.Event()
|
||||
self.addCleanup(release.set)
|
||||
pool = self.api._image_pool()
|
||||
for _ in range(auto.IMAGE_POOL_SIZE): # occupy every pool thread
|
||||
pool.submit(lambda: (entered.wait(WAIT), release.wait(WAIT)))
|
||||
entered.wait(WAIT)
|
||||
here = threading.current_thread().name
|
||||
results, submitted = {}, threading.Event()
|
||||
|
||||
class Counting:
|
||||
calls = 0
|
||||
|
||||
def submit(self, *args):
|
||||
future = pool.submit(*args)
|
||||
Counting.calls += 1
|
||||
if Counting.calls == 2:
|
||||
submitted.set()
|
||||
return future
|
||||
|
||||
def call():
|
||||
with patch.object(self.api, "_image_pool", return_value=Counting()):
|
||||
results["names"] = self.api._map_image(
|
||||
lambda _: threading.current_thread().name, [1, 2]
|
||||
)
|
||||
|
||||
caller = threading.Thread(target=call, name="map-image-caller")
|
||||
caller.start()
|
||||
# The caller's two items are queued behind the busy threads; the
|
||||
# shutdown cancels them and the caller runs them itself.
|
||||
self.assertTrue(submitted.wait(WAIT))
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
caller.join(WAIT)
|
||||
release.set()
|
||||
self.assertFalse(caller.is_alive())
|
||||
self.assertEqual(results["names"], ["map-image-caller"] * 2)
|
||||
self.assertNotEqual(here, "map-image-caller")
|
||||
|
||||
async def test_run_image_after_a_close_uses_the_default_executor(self):
|
||||
pool = self.api._image_pool()
|
||||
pool.shutdown(wait=True)
|
||||
with patch.object(self.api, "_image_pool", return_value=pool):
|
||||
name = await self.api._run_image(lambda: threading.current_thread().name)
|
||||
self.assertFalse(name.startswith(IMAGE_THREAD), name)
|
||||
|
||||
|
||||
class TestJobPool(ImagePoolCase):
|
||||
def test_scan_metadata_is_read_on_the_job_pool(self):
|
||||
paths = [
|
||||
make_png(self.output_dir / f"s{i}.png", size=(16, 16)) for i in range(4)
|
||||
]
|
||||
self.spy(self.api, "_extract_comfyui_metadata")
|
||||
results = self.api._extract_batch_metadata_sync(paths)
|
||||
self.assertEqual([path for path, _ in results], paths)
|
||||
self.assert_threads(JOB_THREAD)
|
||||
|
||||
async def test_thumbnail_job_writes_on_the_job_pool_and_trims_after(self):
|
||||
for index in range(3):
|
||||
make_png(self.output_dir / f"t{index}.png", size=(64, 64))
|
||||
self.spy(self.api, "_generate_one")
|
||||
with patch.object(api_package, "release_free_memory") as trim:
|
||||
job = await self.run_job("/prompt_manager/jobs/thumbnails")
|
||||
self.assertEqual(job.state, "finished")
|
||||
self.assert_threads(JOB_THREAD)
|
||||
trim.assert_called_once_with()
|
||||
|
||||
async def test_scan_job_trims_after_it_finishes(self):
|
||||
make_png(self.output_dir / "s.png", size=(16, 16))
|
||||
states = []
|
||||
|
||||
def trim():
|
||||
states.append(self.api._jobs.active("scan"))
|
||||
return True
|
||||
|
||||
with patch.object(api_package, "release_free_memory", side_effect=trim):
|
||||
job = await self.run_job("/prompt_manager/jobs/scan")
|
||||
self.assertEqual(job.state, "finished")
|
||||
self.assertGreaterEqual(len(states), 1)
|
||||
self.assertIsNone(states[0], "trimmed after the scan finished")
|
||||
|
||||
async def test_a_failing_trim_never_fails_the_job(self):
|
||||
make_png(self.output_dir / "t.png", size=(64, 64))
|
||||
with patch.object(
|
||||
api_package, "release_free_memory", side_effect=RuntimeError("no")
|
||||
):
|
||||
job = await self.run_job("/prompt_manager/jobs/thumbnails")
|
||||
self.assertEqual(job.state, "finished")
|
||||
|
||||
async def test_capture_previews_write_on_the_job_pool(self):
|
||||
path = make_png(self.output_dir / "a.png", size=(64, 64))
|
||||
self.spy(self.api, "_generate_one")
|
||||
self.assertEqual(self.api.queue_capture_thumbnail(str(path)), "queued")
|
||||
await self._idle()
|
||||
self.assert_threads(JOB_THREAD)
|
||||
@@ -136,6 +136,11 @@ def cleanup_package(package, require_stopped=True):
|
||||
if registration is not None:
|
||||
assert registration.dispose(3), "package history worker leaked"
|
||||
close_every_database()
|
||||
# The package's exit actions outlive it otherwise: at interpreter exit
|
||||
# they would import from a package no longer in sys.modules (#580).
|
||||
exit_signal = getattr(package, "_exit_signal", None)
|
||||
if exit_signal is not None:
|
||||
exit_signal.reset()
|
||||
sys.modules.pop(package.__name__, None)
|
||||
|
||||
|
||||
@@ -390,7 +395,39 @@ class TestConsoleBanner(unittest.TestCase):
|
||||
|
||||
stream.flush()
|
||||
banner = stream.buffer.getvalue().decode("cp1252")
|
||||
self.assertIn("Loaded:", banner)
|
||||
lines = [
|
||||
line
|
||||
for line in banner.splitlines()
|
||||
if "PromptManager" in line or "loaded" in line
|
||||
]
|
||||
self.assertEqual(len(lines), 1, banner)
|
||||
self.assertRegex(
|
||||
lines[0], r"\[ComfyUI-PromptManager\] Version:.* \S+ - \d+ tools loaded$"
|
||||
)
|
||||
self.assertNotIn("Loaded:", banner)
|
||||
|
||||
|
||||
class TestFixtureExitActions(unittest.TestCase):
|
||||
"""A disposed fixture package runs nothing at interpreter exit (#580)."""
|
||||
|
||||
def test_a_disposed_package_leaves_no_exit_action(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.addCleanup(close_every_database)
|
||||
output_dir = os.path.join(tmp.name, "output")
|
||||
os.makedirs(output_dir)
|
||||
name = "prompt_manager_pkg_exit_actions"
|
||||
self.addCleanup(sys.modules.pop, name, None)
|
||||
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
with load_package(name, os.path.join(tmp.name, "p.db"), output_dir) as (
|
||||
package,
|
||||
_,
|
||||
):
|
||||
signal = package._exit_signal
|
||||
self.assertTrue(signal._on_exit, "the package installs exit actions")
|
||||
|
||||
self.assertEqual(signal._on_exit, [])
|
||||
|
||||
|
||||
class PriorFileHandler(logging.FileHandler):
|
||||
@@ -553,7 +590,7 @@ class TestPackageLoggerIsolation(unittest.TestCase):
|
||||
original_print = print
|
||||
|
||||
def fail_banner(*args, **kwargs):
|
||||
if any("Total:" in str(arg) for arg in args):
|
||||
if any("tools loaded" in str(arg) for arg in args):
|
||||
captured["package"] = sys.modules[name]
|
||||
captured["handlers"] = [
|
||||
h
|
||||
@@ -587,13 +624,16 @@ PACKAGE_THREAD_NAMES = ("PromptManagerPromptCleanup", "PromptManagerImageWorker"
|
||||
|
||||
def package_threads():
|
||||
"""Live threads this design owns: the tracker, the image worker, and the
|
||||
watchdog observer and emitters (which have no name of their own)."""
|
||||
watchdog observer and emitters (which have no name of their own).
|
||||
watchdog 7's InotifyFD, one daemon reader per process that no observer
|
||||
stop ends, is not the package's."""
|
||||
from watchdog.utils import BaseThread
|
||||
|
||||
return {
|
||||
thread
|
||||
for thread in threading.enumerate()
|
||||
if thread.name in PACKAGE_THREAD_NAMES or isinstance(thread, BaseThread)
|
||||
if thread.name in PACKAGE_THREAD_NAMES
|
||||
or (isinstance(thread, BaseThread) and type(thread).__name__ != "InotifyFD")
|
||||
}
|
||||
|
||||
|
||||
@@ -759,6 +799,45 @@ class TestPackageTeardown(PackageTeardownCase):
|
||||
self.assertIsNone(seen["worker"])
|
||||
self.assertTrue(all(result.values()), result)
|
||||
|
||||
def test_teardown_stops_the_job_pool_once_a_job_used_it(self):
|
||||
"""#523: the long-lived job pool is a teardown part of its own."""
|
||||
package = self.load()
|
||||
parallel = sys.modules[package.__name__ + ".utils.parallel"]
|
||||
self.assertEqual(parallel.map_parallel(lambda n: n * 2, [1, 2], 2), [2, 4])
|
||||
# This package's own pool (the suite's top-level one may also run).
|
||||
threads = list(parallel._pool[1]._threads)
|
||||
self.assertTrue(threads)
|
||||
|
||||
result = package._stop_background_work()
|
||||
|
||||
self.assertEqual(list(result), ["monitor", "thumbnails", "workers", "tracker"])
|
||||
self.assertTrue(all(result.values()), result)
|
||||
self.assertFalse(parallel.job_pool_started())
|
||||
self.assertFalse(any(t.is_alive() for t in threads))
|
||||
|
||||
def test_teardown_reports_a_job_thread_past_its_bound(self):
|
||||
package = self.load(require_stopped=False)
|
||||
parallel = sys.modules[package.__name__ + ".utils.parallel"]
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
|
||||
def hold(_):
|
||||
entered.set()
|
||||
release.wait(30)
|
||||
|
||||
caller = threading.Thread(target=parallel.map_parallel, args=(hold, [1], 1))
|
||||
caller.start()
|
||||
self.addCleanup(caller.join, 30)
|
||||
self.addCleanup(release.set)
|
||||
self.assertTrue(entered.wait(30))
|
||||
with (
|
||||
mock.patch.object(parallel, "JOB_POOL_JOIN_SECONDS", 0.05),
|
||||
self.assertLogs("prompt_manager.init", "WARNING") as logs,
|
||||
):
|
||||
result = package._stop_background_work()
|
||||
self.assertFalse(result["workers"])
|
||||
self.assertTrue(result["tracker"])
|
||||
self.assertRegex("\n".join(logs.output), r"not stopped in time .*workers")
|
||||
|
||||
def test_teardown_kills_ffmpeg_and_stops_the_preview_worker(self):
|
||||
package = self.load()
|
||||
held = self.hold_a_preview(package, advance=4.0, kill_releases=True)
|
||||
|
||||
@@ -108,6 +108,15 @@ LEGACY_WRONG = {
|
||||
"a cozy library with tall shelves",
|
||||
"legacy shows branch 1; the file name names branch 4's saver",
|
||||
),
|
||||
# #548, the one shape where nothing is read on purpose: legacy shows a
|
||||
# PromptSearchList's search text ("fox") as the positive, but that is the
|
||||
# library query, not the prompt the batch ran. The scan and the monitor
|
||||
# never store or link it either (utils.file_prompt._search_text).
|
||||
("inspection_parity", "f22"): (
|
||||
None,
|
||||
"legacy shows the PromptSearchList search text; each item is decided "
|
||||
"at run time, so no positive is read (#548)",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -149,6 +158,24 @@ class LegacyParityGateTest(unittest.TestCase):
|
||||
self.assertEqual(text, expected)
|
||||
self.assertNotEqual(text, legacy)
|
||||
|
||||
def test_the_search_list_query_is_never_stored_either(self):
|
||||
"""#548: the scan and the monitor depart from legacy for f22 too."""
|
||||
from utils.file_prompt import Policy, decide_file_prompt
|
||||
|
||||
graph, workflow = corpora()["inspection_parity"]["f22"]
|
||||
self.assertEqual(legacy_positive(graph, workflow), "fox")
|
||||
for policy in Policy:
|
||||
with self.subTest(policy=policy):
|
||||
decision = decide_file_prompt(
|
||||
{"prompt": graph, "workflow": workflow},
|
||||
file_name="f22.png",
|
||||
file_size=None,
|
||||
in_output=True,
|
||||
lookup=lambda: None,
|
||||
policy=policy,
|
||||
)
|
||||
self.assertEqual((decision.kind, decision.text), ("never", None))
|
||||
|
||||
def test_no_graph_leaves_the_legacy_agreement(self):
|
||||
recorded = json.loads(AGREEMENT_PATH.read_text(encoding="utf-8"))
|
||||
current = {name: agreeing(graphs) for name, graphs in corpora().items()}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""``release_free_memory``: glibc's malloc_trim(0), a no-op elsewhere (#523)."""
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from utils import native_memory
|
||||
|
||||
|
||||
class ReleaseFreeMemoryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# Each test looks the symbol up again.
|
||||
patcher = mock.patch.multiple(native_memory, _trim=None, _looked_up=False)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
@unittest.skipUnless(native_memory._glibc(), "glibc only")
|
||||
def test_glibc_trims_once_per_call(self):
|
||||
self.assertTrue(native_memory.release_free_memory())
|
||||
self.assertTrue(native_memory.release_free_memory())
|
||||
|
||||
def test_other_platforms_do_nothing(self):
|
||||
for platform in ("win32", "darwin", "cygwin"):
|
||||
with self.subTest(platform), mock.patch.object(sys, "platform", platform):
|
||||
native_memory._looked_up = False
|
||||
with mock.patch.object(native_memory.ctypes, "CDLL") as cdll:
|
||||
self.assertFalse(native_memory.release_free_memory())
|
||||
cdll.assert_not_called()
|
||||
|
||||
def test_musl_does_nothing(self):
|
||||
with (
|
||||
mock.patch.object(sys, "platform", "linux"),
|
||||
# Windows has no os.confstr: this test fakes Linux there too.
|
||||
mock.patch.object(
|
||||
native_memory.os,
|
||||
"confstr",
|
||||
side_effect=ValueError("unknown name"),
|
||||
create=True,
|
||||
),
|
||||
mock.patch.object(native_memory.ctypes, "CDLL") as cdll,
|
||||
):
|
||||
self.assertFalse(native_memory.release_free_memory())
|
||||
cdll.assert_not_called()
|
||||
|
||||
def test_a_missing_symbol_is_false(self):
|
||||
library = mock.Mock(spec=[]) # no malloc_trim attribute
|
||||
with (
|
||||
mock.patch.object(native_memory, "_glibc", return_value=True),
|
||||
mock.patch.object(native_memory.ctypes, "CDLL", return_value=library),
|
||||
):
|
||||
self.assertFalse(native_memory.release_free_memory())
|
||||
|
||||
def test_an_unloadable_libc_is_false(self):
|
||||
with (
|
||||
mock.patch.object(native_memory, "_glibc", return_value=True),
|
||||
mock.patch.object(native_memory.ctypes, "CDLL", side_effect=OSError),
|
||||
):
|
||||
self.assertFalse(native_memory.release_free_memory())
|
||||
|
||||
def test_a_failing_trim_never_raises(self):
|
||||
trim = mock.Mock(side_effect=OSError("boom"))
|
||||
library = mock.Mock(malloc_trim=trim)
|
||||
with (
|
||||
mock.patch.object(native_memory, "_glibc", return_value=True),
|
||||
mock.patch.object(native_memory.ctypes, "CDLL", return_value=library),
|
||||
):
|
||||
self.assertFalse(native_memory.release_free_memory())
|
||||
trim.assert_called_once_with(0)
|
||||
|
||||
def test_the_symbol_is_looked_up_once(self):
|
||||
trim = mock.Mock(return_value=1)
|
||||
library = mock.Mock(malloc_trim=trim)
|
||||
with (
|
||||
mock.patch.object(native_memory, "_glibc", return_value=True),
|
||||
mock.patch.object(
|
||||
native_memory.ctypes, "CDLL", return_value=library
|
||||
) as cdll,
|
||||
):
|
||||
self.assertTrue(native_memory.release_free_memory())
|
||||
self.assertTrue(native_memory.release_free_memory())
|
||||
cdll.assert_called_once_with(None)
|
||||
self.assertEqual(trim.call_count, 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+107
-44
@@ -31,22 +31,65 @@ CHILD = textwrap.dedent("""
|
||||
import threading
|
||||
|
||||
root, tmp = sys.argv[1], sys.argv[2]
|
||||
stuck = sys.argv[3:] == ["stuck-observer"]
|
||||
sys.path.insert(0, root)
|
||||
state = {}
|
||||
|
||||
# watchdog 7's Windows observer at its worst (#561): start() starts a
|
||||
# plain reader thread, and stop() waits for it, which never ends.
|
||||
class StuckObserver:
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._never = threading.Event()
|
||||
self._reader = None
|
||||
|
||||
def schedule(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def start(self):
|
||||
self._reader = threading.Thread(
|
||||
target=self._never.wait, name="StuckReader"
|
||||
)
|
||||
self._reader.start()
|
||||
|
||||
def stop(self):
|
||||
self._reader.join()
|
||||
|
||||
def join(self, timeout=None):
|
||||
self._reader.join(timeout)
|
||||
|
||||
def is_alive(self):
|
||||
return self._reader.is_alive()
|
||||
|
||||
if stuck:
|
||||
import watchdog.observers
|
||||
|
||||
watchdog.observers.Observer = StuckObserver
|
||||
|
||||
def report():
|
||||
from watchdog.utils import BaseThread
|
||||
|
||||
package = state.get("package")
|
||||
result = getattr(package, "_background_work_result", None)
|
||||
# The image worker, the tracker, and watchdog's observer and emitters.
|
||||
# watchdog 7's InotifyFD is one daemon reader per process that no
|
||||
# observer stop ends, by design; it is not the package's.
|
||||
alive = sorted(
|
||||
t.name
|
||||
for t in threading.enumerate()
|
||||
if t.name in ("PromptManagerImageWorker", "PromptManagerPromptCleanup")
|
||||
or isinstance(t, BaseThread)
|
||||
or (isinstance(t, BaseThread) and type(t).__name__ != "InotifyFD")
|
||||
)
|
||||
# Threads that would hold the exit before any atexit callback (#561).
|
||||
holding = sorted(
|
||||
t.name
|
||||
for t in threading.enumerate()
|
||||
if not t.daemon and t is not threading.main_thread()
|
||||
)
|
||||
print(
|
||||
"PM_EXIT_REPORT "
|
||||
+ json.dumps({"result": result, "alive": alive, "holding": holding})
|
||||
)
|
||||
print("PM_EXIT_REPORT " + json.dumps({"result": result, "alive": alive}))
|
||||
sys.stdout.flush()
|
||||
|
||||
# First registered, so it runs last: after the package's teardown and
|
||||
@@ -64,6 +107,10 @@ CHILD = textwrap.dedent("""
|
||||
fake_server_module(queue_task_done=True),
|
||||
)
|
||||
state["package"] = package
|
||||
if stuck:
|
||||
# The stop gives up after this long instead of the full 5 s.
|
||||
monitor_module = sys.modules[package.__name__ + ".utils.image_monitor"]
|
||||
monitor_module.OBSERVER_JOIN_SECONDS = 0.5
|
||||
monitor = package._global_image_monitor
|
||||
handler = monitor.handler if monitor is not None else None
|
||||
if handler is None:
|
||||
@@ -78,47 +125,52 @@ CHILD = textwrap.dedent("""
|
||||
""")
|
||||
|
||||
|
||||
def run_child(testcase, *args):
|
||||
"""Run CHILD with ``args``; returns (completed, report, detail)."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
os.makedirs(os.path.join(tmp, "output"))
|
||||
script = os.path.join(tmp, "child.py")
|
||||
with open(script, "w", encoding="utf-8") as handle:
|
||||
handle.write(CHILD)
|
||||
env = dict(os.environ)
|
||||
env.pop("PROMPT_MANAGER_CONFIG_PATH", None)
|
||||
# The child's library and log file live in its own folder, never
|
||||
# in the checkout's logs/ (which this runner may hold open) or a
|
||||
# data folder the caller set.
|
||||
env["PROMPT_MANAGER_DATA_DIR"] = tmp
|
||||
env["PYTHONIOENCODING"] = "utf-8"
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[sys.executable, script, ROOT, tmp, *args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
timeout=CHILD_CEILING_SECONDS,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
except subprocess.TimeoutExpired as hung:
|
||||
testcase.fail(
|
||||
f"the child did not exit within {CHILD_CEILING_SECONDS} s\n"
|
||||
f"stdout:\n{hung.stdout}\nstderr:\n{hung.stderr}"
|
||||
)
|
||||
# Leaving the block removes the folder: on Windows that fails
|
||||
# if the child left a handle open, which it cannot once it exited,
|
||||
# so the check here is that the child exited at all.
|
||||
detail = f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}"
|
||||
testcase.assertEqual(completed.returncode, 0, detail)
|
||||
reports = [
|
||||
line[len(REPORT_PREFIX) :]
|
||||
for line in completed.stdout.splitlines()
|
||||
if line.startswith(REPORT_PREFIX)
|
||||
]
|
||||
testcase.assertEqual(len(reports), 1, detail)
|
||||
return completed, json.loads(reports[0]), detail
|
||||
|
||||
|
||||
class TestPackageExit(unittest.TestCase):
|
||||
def test_teardown_runs_at_a_normal_exit_with_the_worker_alive(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
os.makedirs(os.path.join(tmp, "output"))
|
||||
script = os.path.join(tmp, "child.py")
|
||||
with open(script, "w", encoding="utf-8") as handle:
|
||||
handle.write(CHILD)
|
||||
env = dict(os.environ)
|
||||
env.pop("PROMPT_MANAGER_CONFIG_PATH", None)
|
||||
# The child's library and log file live in its own folder, never
|
||||
# in the checkout's logs/ (which this runner may hold open) or a
|
||||
# data folder the caller set.
|
||||
env["PROMPT_MANAGER_DATA_DIR"] = tmp
|
||||
env["PYTHONIOENCODING"] = "utf-8"
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[sys.executable, script, ROOT, tmp],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
timeout=CHILD_CEILING_SECONDS,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
except subprocess.TimeoutExpired as hung:
|
||||
self.fail(
|
||||
f"the child did not exit within {CHILD_CEILING_SECONDS} s\n"
|
||||
f"stdout:\n{hung.stdout}\nstderr:\n{hung.stderr}"
|
||||
)
|
||||
# Leaving the block removes the folder: on Windows that fails
|
||||
# if the child left a handle open, which it cannot once it exited,
|
||||
# so the check here is that the child exited at all.
|
||||
detail = f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}"
|
||||
self.assertEqual(completed.returncode, 0, detail)
|
||||
reports = [
|
||||
line[len(REPORT_PREFIX) :]
|
||||
for line in completed.stdout.splitlines()
|
||||
if line.startswith(REPORT_PREFIX)
|
||||
]
|
||||
self.assertEqual(len(reports), 1, detail)
|
||||
report = json.loads(reports[0])
|
||||
completed, report, detail = run_child(self)
|
||||
# "thumbnails" is True here because no preview worker started; its
|
||||
# close result is tested in test_thumbnail_auto and test_init_package.
|
||||
self.assertEqual(
|
||||
@@ -127,6 +179,7 @@ class TestPackageExit(unittest.TestCase):
|
||||
detail,
|
||||
)
|
||||
self.assertEqual(report["alive"], [], detail)
|
||||
self.assertEqual(report["holding"], [], detail)
|
||||
# The teardown's own warnings. Import-time warnings (the API's
|
||||
# output-folder search finds no ComfyUI here) are not about exit.
|
||||
for text in (
|
||||
@@ -137,6 +190,16 @@ class TestPackageExit(unittest.TestCase):
|
||||
):
|
||||
self.assertNotIn(text, completed.stderr, detail)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
def test_an_observer_that_never_stops_does_not_hold_the_exit(self):
|
||||
# Before #561 the reader thread kept the interpreter waiting before
|
||||
# any atexit callback ran, so the teardown never started.
|
||||
completed, report, detail = run_child(self, "stuck-observer")
|
||||
self.assertEqual(
|
||||
report["result"],
|
||||
{"monitor": False, "thumbnails": True, "tracker": True},
|
||||
detail,
|
||||
)
|
||||
# The stuck reader is still alive, but as a daemon it holds nothing.
|
||||
self.assertEqual(report["holding"], [], detail)
|
||||
self.assertIn("watchdog observer still running", completed.stderr, detail)
|
||||
self.assertIn("Background work not stopped in time", completed.stderr, detail)
|
||||
|
||||
@@ -90,6 +90,18 @@ class TestPairScan(AdminAPITestCase):
|
||||
self.assertEqual(self.api.db.list_prompt_pairs()["total"], 0)
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_a_prompt_search_list_query_is_never_stored_as_a_prompt(self):
|
||||
# #548: the search text was stored and every batch image linked to it.
|
||||
g = sampled_graph()
|
||||
g["10"] = {
|
||||
"class_type": "PromptSearchList",
|
||||
"inputs": {"text": "fox", "category": "", "tags": "", "limit": 50},
|
||||
}
|
||||
g["1"]["inputs"]["text"] = ["10", 0]
|
||||
self.assertEqual(self._scanned("search.png", g), "unresolved")
|
||||
self.assertEqual(self._texts(), [])
|
||||
self.assertEqual(self.api.db.list_prompt_pairs()["total"], 0)
|
||||
|
||||
def test_numeric_widgets_are_never_stored_as_a_prompt(self):
|
||||
# The pre-redesign scan stored "2048" from a resolution widget here.
|
||||
g = {
|
||||
|
||||
@@ -0,0 +1,455 @@
|
||||
"""PNG text without a pixel decode (#523), against the legacy reader floor.
|
||||
|
||||
Pillow's ``PngImageFile.text`` loads (decodes) every pixel to reach text
|
||||
chunks written after IDAT. ``utils.png_text.png_text`` walks the remaining
|
||||
chunks instead. The legacy reader (``Image.open(path).text``, every failure
|
||||
read as ``{}``) is the floor: a file it read must read the same, and the
|
||||
curated corpus below must read identically. The documented difference is
|
||||
pixel data, which the walk never decodes: undecodable or cut-short pixel data
|
||||
made the legacy reader fail, the walk still reads the text before it.
|
||||
"""
|
||||
|
||||
import io
|
||||
import random
|
||||
import struct
|
||||
import tempfile
|
||||
import unittest
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from PIL import Image, ImageFile, PngImagePlugin
|
||||
|
||||
from utils.metadata_extractor import ComfyUIMetadataExtractor
|
||||
from utils.png_text import png_text
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
||||
IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp"}
|
||||
SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
FUZZ_SEED = 523
|
||||
FUZZ_CASES = 400
|
||||
|
||||
|
||||
def legacy_text(path):
|
||||
"""The legacy scan reader: Pillow's ``.text`` (a full decode)."""
|
||||
try:
|
||||
with Image.open(path) as image:
|
||||
return dict(getattr(image, "text", None) or {})
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def walked_text(path):
|
||||
"""The new reader under the same failure rule."""
|
||||
try:
|
||||
with Image.open(path) as image:
|
||||
return png_text(image)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def chunk(kind, data):
|
||||
return (
|
||||
struct.pack(">I", len(data))
|
||||
+ kind
|
||||
+ data
|
||||
+ struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
)
|
||||
|
||||
|
||||
def chunks_of(raw):
|
||||
"""``[(kind, data)]`` of a well-formed PNG."""
|
||||
found, offset = [], 8
|
||||
while offset < len(raw):
|
||||
length, kind = struct.unpack(">I4s", raw[offset : offset + 8])
|
||||
found.append((kind, raw[offset + 8 : offset + 8 + length]))
|
||||
offset += 12 + length
|
||||
return found
|
||||
|
||||
|
||||
def assemble(parts):
|
||||
return SIGNATURE + b"".join(
|
||||
part if isinstance(part, bytes) else chunk(*part) for part in parts
|
||||
)
|
||||
|
||||
|
||||
def base_png(mode="RGB", size=(64, 64), text=None, noise=False):
|
||||
"""A PNG from Pillow (text before IDAT), as ``[(kind, data)]``."""
|
||||
image = Image.new(mode, size)
|
||||
if noise:
|
||||
rng = random.Random(7)
|
||||
image = Image.frombytes(
|
||||
"RGB", size, bytes(rng.randrange(256) for _ in range(size[0] * size[1] * 3))
|
||||
)
|
||||
info = PngImagePlugin.PngInfo()
|
||||
for key, value in (text or {}).items():
|
||||
info.add_text(key, value)
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="PNG", pnginfo=info)
|
||||
return chunks_of(buffer.getvalue())
|
||||
|
||||
|
||||
def split_idat(parts, pieces=3):
|
||||
"""Spread the IDAT payload over ``pieces`` consecutive IDAT chunks."""
|
||||
data = b"".join(d for k, d in parts if k == b"IDAT")
|
||||
step = max(1, len(data) // pieces)
|
||||
idats = [(b"IDAT", data[i : i + step]) for i in range(0, len(data), step)]
|
||||
first = next(i for i, (k, _) in enumerate(parts) if k == b"IDAT")
|
||||
rest = [p for p in parts if p[0] != b"IDAT"]
|
||||
return rest[:first] + idats + rest[first:]
|
||||
|
||||
|
||||
def before_iend(parts, *extra):
|
||||
return parts[:-1] + list(extra) + parts[-1:]
|
||||
|
||||
|
||||
def text_chunk(key, value):
|
||||
return (b"tEXt", key.encode("latin-1") + b"\0" + value.encode("latin-1"))
|
||||
|
||||
|
||||
def ztxt_chunk(key, value):
|
||||
return (b"zTXt", key.encode() + b"\0\0" + zlib.compress(value.encode("latin-1")))
|
||||
|
||||
|
||||
def itxt_chunk(key, value, compressed=False):
|
||||
body = value.encode("utf-8")
|
||||
if compressed:
|
||||
body = zlib.compress(body)
|
||||
flag = b"\1\0" if compressed else b"\0\0"
|
||||
return (b"iTXt", key.encode() + b"\0" + flag + b"en\0" + b"Title\0" + body)
|
||||
|
||||
|
||||
def bad_crc(kind, data):
|
||||
good = chunk(kind, data)
|
||||
return good[:-4] + bytes(b ^ 0xFF for b in good[-4:])
|
||||
|
||||
|
||||
PROMPT = '{"3": {"class_type": "KSampler", "inputs": {"seed": 1}}}'
|
||||
WORKFLOW = '{"nodes": [], "links": []}'
|
||||
|
||||
|
||||
def curated():
|
||||
"""Name to PNG bytes; each must read the same with both readers."""
|
||||
plain = base_png(text={"prompt": PROMPT})
|
||||
trailing = before_iend(
|
||||
base_png(), text_chunk("prompt", PROMPT), text_chunk("workflow", WORKFLOW)
|
||||
)
|
||||
cases = {
|
||||
"no-text": assemble(base_png()),
|
||||
"text-before-pixels": assemble(plain),
|
||||
"text-after-pixels": assemble(trailing),
|
||||
"ztxt-after-pixels": assemble(
|
||||
before_iend(base_png(), ztxt_chunk("prompt", PROMPT))
|
||||
),
|
||||
"itxt-plain-and-compressed": assemble(
|
||||
before_iend(
|
||||
base_png(),
|
||||
itxt_chunk("prompt", "café " + PROMPT),
|
||||
itxt_chunk("workflow", WORKFLOW, compressed=True),
|
||||
)
|
||||
),
|
||||
"xmp-after-pixels": assemble(
|
||||
before_iend(
|
||||
base_png(),
|
||||
itxt_chunk("XML:com.adobe.xmp", "<x:xmpmeta/>"),
|
||||
text_chunk("prompt", PROMPT),
|
||||
)
|
||||
),
|
||||
"itxt-bad-utf8": assemble(
|
||||
before_iend(
|
||||
base_png(),
|
||||
(b"iTXt", b"prompt\0\0\0\0\0\xff\xfe"),
|
||||
text_chunk("workflow", WORKFLOW),
|
||||
)
|
||||
),
|
||||
"later-copy-wins": assemble(
|
||||
before_iend(plain, text_chunk("prompt", '{"later": 1}'))
|
||||
),
|
||||
"empty-key-and-no-separator": assemble(
|
||||
before_iend(base_png(), (b"tEXt", b"\0value"), (b"tEXt", b"lonely"))
|
||||
),
|
||||
"split-pixels-text-between": assemble(
|
||||
split_idat(before_iend(base_png(noise=True), text_chunk("prompt", PROMPT)))
|
||||
),
|
||||
"palette": assemble(
|
||||
before_iend(base_png(mode="P"), text_chunk("prompt", PROMPT))
|
||||
),
|
||||
"sixteen-bit": assemble(
|
||||
before_iend(base_png(mode="I;16"), text_chunk("prompt", PROMPT))
|
||||
),
|
||||
"bad-crc-before-pixels": assemble(
|
||||
[base_png()[0], bad_crc(*text_chunk("prompt", PROMPT))] + base_png()[1:]
|
||||
),
|
||||
"bad-crc-after-pixels": assemble(
|
||||
base_png()[:-1] + [bad_crc(*text_chunk("prompt", PROMPT))] + base_png()[-1:]
|
||||
),
|
||||
"unknown-chunk-after-pixels": assemble(
|
||||
before_iend(base_png(), (b"prVt", b"x" * 40), text_chunk("prompt", PROMPT))
|
||||
),
|
||||
"garbage-chunk-id-after-pixels": assemble(
|
||||
base_png()[:-1]
|
||||
+ [b"\0\0\0\4\x00\x01\x02\x03abcd\0\0\0\0"]
|
||||
+ [chunk(*text_chunk("prompt", PROMPT))]
|
||||
+ base_png()[-1:]
|
||||
),
|
||||
"oversized-ztxt-after-pixels": assemble(
|
||||
before_iend(
|
||||
base_png(),
|
||||
(
|
||||
b"zTXt",
|
||||
b"prompt\0\0"
|
||||
+ zlib.compress(b"a" * (PngImagePlugin.MAX_TEXT_CHUNK + 10)),
|
||||
),
|
||||
)
|
||||
),
|
||||
"pixels-length-past-end": assemble(
|
||||
[p for p in base_png() if p[0] != b"IDAT" and p[0] != b"IEND"]
|
||||
)
|
||||
+ b"\xff\xff\x00\x00IDAT"
|
||||
+ b"\0" * 64,
|
||||
}
|
||||
full = assemble(trailing)
|
||||
idat_at = full.index(b"IDAT")
|
||||
cases["truncated-in-pixels"] = full[: idat_at + 20]
|
||||
cases["no-end-after-pixels"] = full[: full.rindex(b"IEND") - 4]
|
||||
cases["truncated-in-trailing-text"] = full[: full.rindex(b"workflow") + 4]
|
||||
cases["unknown-chunk-past-end"] = (
|
||||
assemble(before_iend(base_png(), ztxt_chunk("prompt", PROMPT))[:-1])
|
||||
+ chunk(b"prVt", b"x" * 40)[:-20]
|
||||
)
|
||||
cases["not-a-png"] = b"nope, not an image at all"
|
||||
cases["signature-only"] = SIGNATURE
|
||||
return cases
|
||||
|
||||
|
||||
def corrupt_pixels():
|
||||
"""Complete IDAT chunks with valid CRCs over data zlib cannot inflate."""
|
||||
parts = before_iend(base_png(), text_chunk("prompt", PROMPT))
|
||||
return assemble(
|
||||
[(k, b"\x78\x9c" + b"\xff" * 30) if k == b"IDAT" else (k, d) for k, d in parts]
|
||||
)
|
||||
|
||||
|
||||
def apng():
|
||||
frames = [Image.new("RGB", (16, 16), colour) for colour in ("red", "blue")]
|
||||
info = PngImagePlugin.PngInfo()
|
||||
info.add_text("prompt", PROMPT)
|
||||
buffer = io.BytesIO()
|
||||
frames[0].save(
|
||||
buffer, format="PNG", save_all=True, append_images=frames[1:], pnginfo=info
|
||||
)
|
||||
return assemble(
|
||||
before_iend(chunks_of(buffer.getvalue()), text_chunk("workflow", WORKFLOW))
|
||||
)
|
||||
|
||||
|
||||
def fixture_images():
|
||||
return sorted(p for p in FIXTURES.rglob("*") if p.suffix.lower() in IMAGE_SUFFIXES)
|
||||
|
||||
|
||||
class _TempFiles(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
|
||||
def write(self, name, raw):
|
||||
path = Path(self._tmp.name) / name
|
||||
path.write_bytes(raw)
|
||||
return path
|
||||
|
||||
|
||||
class PngTextParityTests(_TempFiles):
|
||||
def test_every_fixture_image_reads_like_the_legacy_reader(self):
|
||||
images = fixture_images()
|
||||
self.assertGreaterEqual(len(images), 25)
|
||||
for path in images:
|
||||
with self.subTest(path=path.relative_to(FIXTURES)):
|
||||
self.assertEqual(walked_text(path), legacy_text(path))
|
||||
|
||||
def test_curated_corpus_reads_like_the_legacy_reader(self):
|
||||
for name, raw in curated().items():
|
||||
with self.subTest(name):
|
||||
path = self.write(name + ".png", raw)
|
||||
self.assertEqual(walked_text(path), legacy_text(path))
|
||||
|
||||
def test_curated_corpus_exercises_trailing_text(self):
|
||||
"""The corpus is not vacuous: trailing text exists and is read."""
|
||||
cases = curated()
|
||||
path = self.write("t.png", cases["text-after-pixels"])
|
||||
self.assertEqual(walked_text(path), {"prompt": PROMPT, "workflow": WORKFLOW})
|
||||
with Image.open(path) as image:
|
||||
self.assertNotIn("prompt", image.info)
|
||||
self.assertEqual(
|
||||
walked_text(self.write("x.png", cases["truncated-in-pixels"])), {}
|
||||
)
|
||||
|
||||
def test_itxt_values_keep_their_language_and_type(self):
|
||||
path = self.write("i.png", curated()["itxt-plain-and-compressed"])
|
||||
new, old = walked_text(path), legacy_text(path)
|
||||
self.assertIsInstance(new["prompt"], PngImagePlugin.iTXt)
|
||||
self.assertEqual(new["prompt"].lang, old["prompt"].lang)
|
||||
self.assertEqual(new["prompt"].tkey, old["prompt"].tkey)
|
||||
|
||||
def test_animated_png_falls_back_to_the_legacy_read(self):
|
||||
path = self.write("a.png", apng())
|
||||
self.assertEqual(walked_text(path), legacy_text(path))
|
||||
self.assertEqual(walked_text(path)["workflow"], WORKFLOW)
|
||||
|
||||
def test_truncated_pixels_read_alike_when_truncated_images_load(self):
|
||||
"""With LOAD_TRUNCATED_IMAGES set (a node pack may set it) both readers
|
||||
accept a short file alike."""
|
||||
path = self.write("t.png", curated()["truncated-in-pixels"])
|
||||
with patch.object(ImageFile, "LOAD_TRUNCATED_IMAGES", True):
|
||||
self.assertEqual(walked_text(path), legacy_text(path))
|
||||
|
||||
def test_a_text_decode_error_ends_the_walk_like_legacy(self):
|
||||
"""Pillow's ``load_end`` stops at a chunk whose handler raises
|
||||
``UnicodeDecodeError`` and keeps the text read so far."""
|
||||
path = self.write(
|
||||
"u.png",
|
||||
assemble(
|
||||
before_iend(
|
||||
base_png(),
|
||||
ztxt_chunk("prompt", PROMPT),
|
||||
text_chunk("workflow", WORKFLOW),
|
||||
ztxt_chunk("parameters", "steps 20"),
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
def undecodable(stream, position, length):
|
||||
raise UnicodeDecodeError("latin-1", b"", 0, 1, "test")
|
||||
|
||||
with patch.object(PngImagePlugin.PngStream, "chunk_tEXt", undecodable):
|
||||
old = legacy_text(path)
|
||||
self.assertEqual(walked_text(path), old)
|
||||
self.assertEqual(old, {"prompt": PROMPT})
|
||||
|
||||
def test_the_text_memory_budget_counts_text_before_the_pixels(self):
|
||||
"""Pillow's MAX_TEXT_MEMORY spans the whole file: text before and
|
||||
after the pixels, each under it, fail together in both readers."""
|
||||
path = self.write(
|
||||
"m.png",
|
||||
assemble(
|
||||
before_iend(
|
||||
base_png(text={"prompt": PROMPT}), text_chunk("workflow", WORKFLOW)
|
||||
)
|
||||
),
|
||||
)
|
||||
budget = len(PROMPT) + len(WORKFLOW) - 1
|
||||
with patch.object(PngImagePlugin, "MAX_TEXT_MEMORY", budget):
|
||||
self.assertEqual(legacy_text(path), {})
|
||||
self.assertEqual(walked_text(path), {})
|
||||
with patch.object(PngImagePlugin, "MAX_TEXT_MEMORY", budget + 1):
|
||||
self.assertEqual(walked_text(path), legacy_text(path))
|
||||
self.assertEqual(len(walked_text(path)), 2)
|
||||
|
||||
def test_complete_but_undecodable_pixels_read_more_than_legacy(self):
|
||||
path = self.write("c.png", corrupt_pixels())
|
||||
self.assertEqual(legacy_text(path), {})
|
||||
self.assertEqual(walked_text(path), {"prompt": PROMPT})
|
||||
|
||||
def test_cut_short_pixels_read_the_text_before_them(self):
|
||||
full = assemble(base_png(text={"prompt": PROMPT}))
|
||||
path = self.write("s.png", full[: full.index(b"IDAT") + 12])
|
||||
self.assertEqual(legacy_text(path), {})
|
||||
self.assertEqual(walked_text(path), {"prompt": PROMPT})
|
||||
|
||||
def test_mutated_files_never_read_less_than_legacy(self):
|
||||
"""Seeded fuzz: truncations and byte flips of a file with text before,
|
||||
between and after its pixel chunks. Wherever the legacy reader found
|
||||
text, the walk finds exactly the same."""
|
||||
base = assemble(
|
||||
split_idat(
|
||||
before_iend(
|
||||
base_png(text={"workflow": WORKFLOW}),
|
||||
ztxt_chunk("prompt", PROMPT),
|
||||
itxt_chunk("parameters", "steps 20"),
|
||||
)
|
||||
)
|
||||
)
|
||||
rng = random.Random(FUZZ_SEED)
|
||||
compared = 0
|
||||
for case in range(FUZZ_CASES):
|
||||
raw = bytearray(base)
|
||||
if case % 2:
|
||||
del raw[rng.randrange(8, len(raw)) :]
|
||||
else:
|
||||
for _ in range(rng.randrange(1, 4)):
|
||||
raw[rng.randrange(8, len(raw))] = rng.randrange(256)
|
||||
path = self.write(f"fuzz{case}.png", bytes(raw))
|
||||
old = legacy_text(path)
|
||||
if old:
|
||||
compared += 1
|
||||
with self.subTest(case=case):
|
||||
self.assertEqual(walked_text(path), old)
|
||||
self.assertGreater(compared, FUZZ_CASES // 5)
|
||||
|
||||
def test_non_png_images_have_no_text(self):
|
||||
for name in ("tiny.jpg", "tiny.webp"):
|
||||
with Image.open(FIXTURES / "local-inspect" / name) as image:
|
||||
self.assertEqual(png_text(image), {})
|
||||
|
||||
|
||||
class NoPixelDecodeTests(_TempFiles):
|
||||
"""Metadata reads never decode pixels (the #523 memory finding)."""
|
||||
|
||||
def png_fixtures(self):
|
||||
return sorted((FIXTURES / "inspection_parity").glob("*.png"))
|
||||
|
||||
def test_png_text_does_not_load_pixels_and_leaves_the_image_usable(self):
|
||||
path = self.write("t.png", curated()["text-after-pixels"])
|
||||
with Image.open(path) as image:
|
||||
with patch.object(ImageFile.ImageFile, "load", autospec=True) as load:
|
||||
self.assertEqual(png_text(image)["workflow"], WORKFLOW)
|
||||
load.assert_not_called()
|
||||
image.load()
|
||||
self.assertEqual(image.size, (64, 64))
|
||||
self.assertEqual(image.text["workflow"], WORKFLOW)
|
||||
|
||||
def test_scan_reader_does_not_decode_pixels(self):
|
||||
from py.api import PromptManagerAPI
|
||||
|
||||
api = PromptManagerAPI.__new__(PromptManagerAPI)
|
||||
fixtures = self.png_fixtures() + [
|
||||
self.write("t.png", curated()["text-after-pixels"])
|
||||
]
|
||||
with patch.object(ImageFile.ImageFile, "load", autospec=True) as load:
|
||||
found = [api._extract_comfyui_metadata(str(p)) for p in fixtures]
|
||||
load.assert_not_called()
|
||||
self.assertEqual(found, [legacy_text(p) for p in fixtures])
|
||||
self.assertTrue(all(found))
|
||||
|
||||
def test_monitor_extractor_does_not_decode_pixels(self):
|
||||
extractor = ComfyUIMetadataExtractor()
|
||||
fixtures = self.png_fixtures()
|
||||
with patch.object(ImageFile.ImageFile, "load", autospec=True) as load:
|
||||
found = [extractor.extract_metadata(str(p)) for p in fixtures]
|
||||
load.assert_not_called()
|
||||
self.assertTrue(all(metadata and len(metadata) > 1 for metadata in found))
|
||||
|
||||
|
||||
class MonitorExtractorParityTests(_TempFiles):
|
||||
"""``extract_metadata`` answers exactly as with the legacy ``.text``."""
|
||||
|
||||
def legacy_extract(self, path):
|
||||
def old_text(image):
|
||||
return dict(getattr(image, "text", None) or {})
|
||||
|
||||
with patch("utils.metadata_extractor.png_text", old_text):
|
||||
return ComfyUIMetadataExtractor().extract_metadata(str(path))
|
||||
|
||||
def test_every_fixture_and_curated_file_extracts_like_legacy(self):
|
||||
paths = fixture_images() + [
|
||||
self.write(name + ".png", raw) for name, raw in curated().items()
|
||||
]
|
||||
extractor = ComfyUIMetadataExtractor()
|
||||
for path in paths:
|
||||
with self.subTest(path=path.name):
|
||||
self.assertEqual(
|
||||
extractor.extract_metadata(str(path)), self.legacy_extract(path)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,457 @@
|
||||
"""One resolver for where previews live on disk (#320 PR 1).
|
||||
|
||||
``py/api/preview_store.py`` owns every mapping between an image source, a
|
||||
media file and its preview on disk. In source mode (the only mode so far)
|
||||
it returns exactly the paths the code built by hand before
|
||||
(``<source>/thumbnails/<rel>_thumb<ext>``), so nothing changes on disk. The
|
||||
structural test keeps new ``thumbnails`` path joins out of the rest of the
|
||||
backend.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
import unittest.mock
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
sys.modules.setdefault("server", MagicMock())
|
||||
|
||||
from py.api import preview_store, thumbnail_settings # noqa: E402
|
||||
from py.api.thumbnail_encode import ( # noqa: E402
|
||||
PreviewWriteHeld,
|
||||
replace_unless_held,
|
||||
)
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
WAIT = 10
|
||||
|
||||
|
||||
class SourceModeCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.root = Path(self._tmp.name) / "output"
|
||||
self.root.mkdir()
|
||||
|
||||
|
||||
class TestSourceModePaths(SourceModeCase):
|
||||
def test_default_location_is_inside_each_source(self):
|
||||
location = preview_store.current()
|
||||
self.assertEqual(location.mode, preview_store.SOURCE_MODE)
|
||||
self.assertTrue(location.available)
|
||||
self.assertIsNone(location.problem)
|
||||
|
||||
def test_store_of_a_source_is_its_thumbnails_folder(self):
|
||||
self.assertEqual(preview_store.store_dir(self.root), self.root / "thumbnails")
|
||||
|
||||
def test_preview_path_matches_the_old_join(self):
|
||||
rel = Path("sub", "deep", "a.png")
|
||||
self.assertEqual(
|
||||
preview_store.preview_path(self.root, rel, ".webp"),
|
||||
self.root / "thumbnails/sub/deep/a_thumb.webp",
|
||||
)
|
||||
self.assertEqual(
|
||||
preview_store.preview_path(self.root, "a.mp4", ".jpg"),
|
||||
self.root / "thumbnails" / "a_thumb.jpg",
|
||||
)
|
||||
|
||||
def test_preview_path_defaults_to_the_written_extension(self):
|
||||
from py.api.thumbnail_encode import writer_settings
|
||||
|
||||
self.assertEqual(
|
||||
preview_store.preview_path(self.root, "a.png").suffix,
|
||||
writer_settings().extension,
|
||||
)
|
||||
|
||||
def test_request_path_maps_url_parts_to_the_old_join(self):
|
||||
parts = ("thumbnails", "sub", "a_thumb.webp")
|
||||
self.assertEqual(
|
||||
preview_store.request_path(self.root, parts), self.root.joinpath(*parts)
|
||||
)
|
||||
|
||||
def test_request_path_is_none_for_anything_but_preview_parts(self):
|
||||
self.assertIsNone(preview_store.request_path(self.root, ("sub", "a.png")))
|
||||
self.assertIsNone(preview_store.request_path(self.root, ("Thumbnails", "a")))
|
||||
self.assertIsNone(preview_store.request_path(self.root, ("thumbnails",)))
|
||||
|
||||
def test_ensure_store_is_the_store(self):
|
||||
self.assertEqual(
|
||||
preview_store.ensure_store(self.root), self.root / "thumbnails"
|
||||
)
|
||||
|
||||
def test_owns_a_preview_inside_the_store(self):
|
||||
inside = self.root / "thumbnails" / "sub" / "a_thumb.webp"
|
||||
self.assertTrue(preview_store.owns(self.root, inside))
|
||||
self.assertFalse(preview_store.owns(self.root, self.root / "a.png"))
|
||||
self.assertFalse(
|
||||
preview_store.owns(self.root, self.root.parent / "elsewhere.webp")
|
||||
)
|
||||
|
||||
@unittest.skipUnless(hasattr(os, "symlink"), "needs symlinks")
|
||||
def test_owns_refuses_a_store_linked_out_of_the_source(self):
|
||||
outside = Path(self._tmp.name) / "outside"
|
||||
outside.mkdir()
|
||||
try:
|
||||
(self.root / "thumbnails").symlink_to(outside, target_is_directory=True)
|
||||
except OSError:
|
||||
self.skipTest("symlinks not permitted")
|
||||
self.assertFalse(
|
||||
preview_store.owns(self.root, self.root / "thumbnails" / "a_thumb.webp")
|
||||
)
|
||||
|
||||
@unittest.skipUnless(hasattr(os, "symlink"), "needs symlinks")
|
||||
def test_owns_refuses_a_preview_folder_linked_out_of_the_store(self):
|
||||
outside = Path(self._tmp.name) / "outside"
|
||||
outside.mkdir()
|
||||
(self.root / "thumbnails").mkdir()
|
||||
try:
|
||||
(self.root / "thumbnails" / "sub").symlink_to(
|
||||
outside, target_is_directory=True
|
||||
)
|
||||
except OSError:
|
||||
self.skipTest("symlinks not permitted")
|
||||
self.assertFalse(
|
||||
preview_store.owns(
|
||||
self.root, self.root / "thumbnails" / "sub" / "a_thumb.webp"
|
||||
)
|
||||
)
|
||||
|
||||
def test_store_container_is_the_source(self):
|
||||
self.assertEqual(preview_store.store_container(self.root), self.root)
|
||||
|
||||
def test_given_store_argument_is_kept_unless_it_is_the_default(self):
|
||||
default = self.root / "thumbnails"
|
||||
other = Path(self._tmp.name) / "custom"
|
||||
self.assertEqual(preview_store.given_store(self.root, default), default)
|
||||
self.assertEqual(preview_store.given_store(self.root, other), other)
|
||||
|
||||
|
||||
class TestPreviewText(unittest.TestCase):
|
||||
def test_preview_text_is_an_exact_thumbnails_prefix_and_preview_name(self):
|
||||
for text in (
|
||||
"thumbnails/a_thumb.webp",
|
||||
"thumbnails/sub/a_thumb.jxl",
|
||||
"thumbnails/a_thumb.JPG",
|
||||
"thumbnails/a_thumb.Png",
|
||||
):
|
||||
with self.subTest(text=text):
|
||||
self.assertTrue(preview_store.is_preview_text(text))
|
||||
|
||||
def test_other_text_is_not_preview_text(self):
|
||||
for text in (
|
||||
"Thumbnails/a_thumb.webp",
|
||||
"thumbnails/a.png",
|
||||
"thumbnails/_thumb.webp",
|
||||
"thumbnails/a_thumb.txt",
|
||||
"sub/thumbnails/a_thumb.webp",
|
||||
"thumbnails/../a_thumb.webp",
|
||||
"thumbnails//a_thumb.webp",
|
||||
"thumbnails/./a_thumb.webp",
|
||||
"thumbnails\\a_thumb.webp",
|
||||
"thumbnails/c:a_thumb.webp",
|
||||
"thumbnails/sub/thumbnails/a_thumb.webp",
|
||||
"thumbnails",
|
||||
"",
|
||||
):
|
||||
with self.subTest(text=text):
|
||||
self.assertFalse(preview_store.is_preview_text(text))
|
||||
|
||||
|
||||
class TestWritesAndTheSwitch(SourceModeCase):
|
||||
def _temp_and_dst(self):
|
||||
folder = self.root / "thumbnails"
|
||||
folder.mkdir()
|
||||
tmp = folder / ".a_thumb.part-1-1.webp"
|
||||
tmp.write_bytes(b"new")
|
||||
return tmp, folder / "a_thumb.webp"
|
||||
|
||||
def test_a_write_from_the_current_generation_lands(self):
|
||||
tmp, dst = self._temp_and_dst()
|
||||
replace_unless_held(tmp, dst, preview_store.write_guard())
|
||||
self.assertEqual(dst.read_bytes(), b"new")
|
||||
|
||||
def test_a_write_whose_generation_changed_is_dropped(self):
|
||||
tmp, dst = self._temp_and_dst()
|
||||
guard = preview_store.write_guard()
|
||||
preview_store.bump_generation()
|
||||
with self.assertRaises(PreviewWriteHeld):
|
||||
replace_unless_held(tmp, dst, guard)
|
||||
self.assertFalse(dst.exists())
|
||||
|
||||
def test_the_hold_check_still_applies(self):
|
||||
tmp, dst = self._temp_and_dst()
|
||||
with self.assertRaises(PreviewWriteHeld):
|
||||
replace_unless_held(tmp, dst, preview_store.write_guard(lambda: False))
|
||||
self.assertFalse(dst.exists())
|
||||
|
||||
def test_the_check_and_the_rename_run_under_the_write_lock(self):
|
||||
"""The hold check and the rename hold ``write_lock``, the lock the
|
||||
generation bump takes: a switch cannot fall between them."""
|
||||
tmp, dst = self._temp_and_dst()
|
||||
in_check = threading.Event()
|
||||
release = threading.Event()
|
||||
seen = {}
|
||||
|
||||
def slow_hold():
|
||||
seen["locked"] = preview_store.write_lock.locked()
|
||||
in_check.set()
|
||||
release.wait(WAIT)
|
||||
return True
|
||||
|
||||
guard = preview_store.write_guard(slow_hold)
|
||||
writer = threading.Thread(target=replace_unless_held, args=(tmp, dst, guard))
|
||||
writer.start()
|
||||
try:
|
||||
self.assertTrue(in_check.wait(WAIT))
|
||||
self.assertTrue(seen["locked"])
|
||||
self.assertTrue(preview_store.write_lock.locked())
|
||||
finally:
|
||||
release.set()
|
||||
writer.join(WAIT)
|
||||
self.assertFalse(preview_store.write_lock.locked())
|
||||
self.assertEqual(dst.read_bytes(), b"new")
|
||||
bumped = preview_store.bump_generation()
|
||||
self.assertEqual(preview_store.generation(), bumped)
|
||||
|
||||
|
||||
class TestOnlyTheResolverBuildsStorePaths(unittest.TestCase):
|
||||
"""No backend module joins ``thumbnails`` onto a path by hand any more."""
|
||||
|
||||
ALLOWED = {
|
||||
Path("py/api/preview_store.py"),
|
||||
Path("py/api/thumbnail_settings.py"), # URL text
|
||||
}
|
||||
# A path join with a literal "thumbnails" operand.
|
||||
JOIN = re.compile(
|
||||
r"""/\s*["']thumbnails["']"""
|
||||
r"""|["']thumbnails["']\s*/"""
|
||||
r"""|joinpath\([^)]*["']thumbnails["']"""
|
||||
r"""|os\.path\.join\([^)]*["']thumbnails["']"""
|
||||
r"""|["']thumbnails/"""
|
||||
r"""|/\s*PREVIEW_FOLDER|joinpath\([^)]*PREVIEW_FOLDER"""
|
||||
r"""|Path\([^)]*,\s*["']thumbnails["']"""
|
||||
)
|
||||
|
||||
def test_no_thumbnails_join_outside_the_resolver(self):
|
||||
offenders = []
|
||||
files = sorted(REPO.glob("*.py"))
|
||||
for folder in ("py", "utils", "database"):
|
||||
files += sorted((REPO / folder).rglob("*.py"))
|
||||
for path in files:
|
||||
rel = path.relative_to(REPO)
|
||||
if rel in self.ALLOWED:
|
||||
continue
|
||||
for number, line in enumerate(
|
||||
path.read_text(encoding="utf-8").splitlines(), 1
|
||||
):
|
||||
code = line.split("#", 1)[0]
|
||||
if self.JOIN.search(code):
|
||||
offenders.append(f"{rel}:{number}: {line.strip()}")
|
||||
self.assertEqual(offenders, [])
|
||||
|
||||
|
||||
class TestSettingsLookupsFollowTheStore(SourceModeCase):
|
||||
def test_owned_thumbnails_reads_the_store(self):
|
||||
folder = self.root / "thumbnails" / "sub"
|
||||
folder.mkdir(parents=True)
|
||||
(folder / "a_thumb.webp").write_bytes(b"x")
|
||||
self.assertEqual(
|
||||
thumbnail_settings.owned_thumbnails(self.root, Path("sub", "a.png")),
|
||||
[folder / "a_thumb.webp"],
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
from tests.test_thumbnail_auto import AutoThumbnailCase, _entry
|
||||
from tests.test_api_images import make_png
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_thumbnail_auto import AutoThumbnailCase, _entry
|
||||
from test_api_images import make_png
|
||||
|
||||
|
||||
class RedirectedStoreCase(AutoThumbnailCase):
|
||||
"""Every caller follows the resolver: with the store of the output root
|
||||
moved to ``<tmp>/elsewhere/output-key`` (a stand-in for a later mode),
|
||||
previews are read, written, listed and removed there and only there."""
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.container = self.comfy_dir / "elsewhere"
|
||||
self.store = self.container / "output-key"
|
||||
self.store.mkdir(parents=True)
|
||||
self.old_store = self.output_dir / "thumbnails"
|
||||
root = self.output_dir
|
||||
real_store = preview_store.store_dir
|
||||
real_container = preview_store.store_container
|
||||
|
||||
def store_dir(r, location=None):
|
||||
return self.store if Path(r) == root else real_store(r, location)
|
||||
|
||||
def store_container(r, location=None):
|
||||
if Path(r) == root:
|
||||
return self.container
|
||||
return real_container(r, location)
|
||||
|
||||
for name, fake in (
|
||||
("store_dir", store_dir),
|
||||
("store_container", store_container),
|
||||
):
|
||||
patcher = unittest.mock.patch.object(preview_store, name, fake)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def _preview(self, rel, ext=".webp"):
|
||||
return self.store / (thumbnail_settings.preview_stem(rel) + ext)
|
||||
|
||||
|
||||
class TestEveryCallerFollowsTheStore(RedirectedStoreCase):
|
||||
async def test_an_existing_preview_is_served_from_the_store(self):
|
||||
make_png(self.output_dir / "sub" / "a.png", size=(64, 64))
|
||||
preview = self._preview("sub/a.png")
|
||||
make_png(preview, size=(32, 32))
|
||||
url = _entry(self.api, self.output_dir, "sub/a.png")["thumbnail_url"]
|
||||
self.assertIn("v=", url) # the listing found it in the store
|
||||
resp = await self.client.get(url)
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(await resp.read(), preview.read_bytes())
|
||||
|
||||
async def test_a_legacy_upper_case_preview_is_served_from_the_store(self):
|
||||
make_png(self.output_dir / "k.JPG", size=(64, 64))
|
||||
legacy = self.store / "k_thumb.JPG"
|
||||
legacy.write_bytes(b"legacy preview")
|
||||
resp = await self.client.get(
|
||||
"/prompt_manager/images/serve/thumbnails/k_thumb.JPG?root=0"
|
||||
)
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(await resp.read(), b"legacy preview")
|
||||
|
||||
async def test_a_missing_preview_is_made_in_the_store_on_first_view(self):
|
||||
make_png(self.output_dir / "sub" / "b.png", size=(640, 480))
|
||||
url = _entry(self.api, self.output_dir, "sub/b.png")["thumbnail_url"]
|
||||
resp = await self.client.get(url)
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertTrue(self._preview("sub/b.png").is_file())
|
||||
self.assertFalse(self.old_store.exists())
|
||||
|
||||
async def test_a_leftover_in_the_source_folder_is_not_served(self):
|
||||
make_png(self.output_dir / "c.png", size=(640, 480))
|
||||
leftover = self.old_store / "c_thumb.webp"
|
||||
leftover.parent.mkdir()
|
||||
leftover.write_bytes(b"old preview bytes")
|
||||
resp = await self.client.get(
|
||||
"/prompt_manager/images/serve/thumbnails/c_thumb.webp?root=0"
|
||||
)
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertNotEqual(await resp.read(), b"old preview bytes")
|
||||
self.assertTrue(self._preview("c.png").is_file())
|
||||
|
||||
async def test_capture_writes_into_the_store(self):
|
||||
path = make_png(self.output_dir / "run" / "d.png", size=(64, 64))
|
||||
self.assertEqual(self.api.queue_capture_thumbnail(str(path)), "queued")
|
||||
await self._idle()
|
||||
self.assertTrue(self._preview("run/d.png").is_file())
|
||||
self.assertFalse(self.old_store.exists())
|
||||
|
||||
async def test_generate_and_clear_follow_the_store(self):
|
||||
make_png(self.output_dir / "e.png", size=(64, 64))
|
||||
payload = await asyncio.get_running_loop().run_in_executor(
|
||||
None, self.api._generate_gallery_thumbnails_sync, [self.output_dir]
|
||||
)
|
||||
self.assertTrue(payload["success"])
|
||||
self.assertTrue(self._preview("e.png").is_file())
|
||||
self.assertFalse(self.old_store.exists())
|
||||
self.assertTrue(payload["thumbnails_path"].endswith("output-key"))
|
||||
cleared, _size = self.api._clear_gallery_thumbnails_sync([self.output_dir])
|
||||
self.assertEqual(cleared, 1)
|
||||
self.assertFalse(self._preview("e.png").exists())
|
||||
self.assertTrue(self.store.is_dir()) # the store itself stays
|
||||
|
||||
async def test_removing_a_file_removes_its_preview_in_the_store(self):
|
||||
from py.api.admin import _remove_thumbnail
|
||||
|
||||
make_png(self.output_dir / "f.png", size=(64, 64))
|
||||
preview = self._preview("f.png")
|
||||
preview.write_bytes(b"x")
|
||||
self.assertTrue(_remove_thumbnail(self.output_dir, Path("f.png")))
|
||||
self.assertFalse(preview.exists())
|
||||
|
||||
async def test_the_temp_sweep_walks_the_store(self):
|
||||
stray = self.store / ".g_thumb.part-1-1.webp"
|
||||
stray.write_bytes(b"x")
|
||||
old = os.path.getmtime(stray) - 10 * 24 * 3600
|
||||
os.utime(stray, (old, old))
|
||||
self.assertEqual(self.api.sweep_thumbnail_temps(), 1)
|
||||
self.assertFalse(stray.exists())
|
||||
|
||||
async def test_history_cover_writes_into_the_store(self):
|
||||
from py.api import history_current
|
||||
|
||||
self.assertTrue(
|
||||
history_current._preview_writable(self.output_dir, self._preview("h.png"))
|
||||
)
|
||||
self.assertFalse(
|
||||
history_current._preview_writable(
|
||||
self.output_dir, self.old_store / "h_thumb.webp"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class TestCaptureResolvesAtWriteTime(RedirectedStoreCase):
|
||||
def test_a_queued_capture_follows_a_store_that_moved_before_its_write(self):
|
||||
from py.api import thumbnail_auto as auto
|
||||
|
||||
written = []
|
||||
thumbnailer = auto.AutoThumbnailer(
|
||||
lambda src, dst, *a, **k: written.append(dst) or {"action": "generated"},
|
||||
lambda: True,
|
||||
self.api.logger,
|
||||
start_worker=False,
|
||||
)
|
||||
self.addCleanup(thumbnailer.close)
|
||||
src = make_png(self.output_dir / "i.png", size=(32, 32))
|
||||
deferred = auto.DeferredPreview(self.output_dir, Path("i.png"))
|
||||
self.assertTrue(thumbnailer.enqueue(src, deferred, False))
|
||||
moved = self.container / "moved"
|
||||
moved.mkdir()
|
||||
self.store = moved # the location changes before the worker runs
|
||||
thumbnailer._write_queued(thumbnailer._queue.popleft())
|
||||
self.assertEqual(written, [moved / "i_thumb.webp"])
|
||||
|
||||
def test_the_after_scan_queue_maps_each_target_to_its_own_root(self):
|
||||
from py.api import thumbnail_auto as auto
|
||||
|
||||
nested = self.output_dir / "inner"
|
||||
nested.mkdir()
|
||||
src = nested / "j.png"
|
||||
outer = auto._deferred(
|
||||
[self.output_dir, nested], src, self._preview("inner/j.png")
|
||||
)
|
||||
inner = auto._deferred(
|
||||
[self.output_dir, nested], src, nested / "thumbnails" / "j_thumb.webp"
|
||||
)
|
||||
self.assertEqual(
|
||||
(outer.root, outer.rel), (self.output_dir, Path("inner/j.png"))
|
||||
)
|
||||
self.assertEqual((inner.root, inner.rel), (nested, Path("j.png")))
|
||||
stray = self.comfy_dir / "x_thumb.webp"
|
||||
self.assertEqual(auto._deferred([self.output_dir], src, stray), stray)
|
||||
|
||||
def test_serve_and_history_share_one_flight_key(self):
|
||||
from py.api import thumbnail_auto as auto
|
||||
|
||||
parts = ("thumbnails", "sub", "a_thumb.webp")
|
||||
self.assertEqual(
|
||||
auto._request_key(self.output_dir, parts),
|
||||
auto._dst_key(self._preview("sub/a.png")),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,539 @@
|
||||
"""One folder mode (#320 PR 2): previews in ``<folder>/<source key>``.
|
||||
|
||||
Folder mode is reachable from config.json and tests only (D4B). These tests
|
||||
set it on the resolver directly (``preview_store.configure``) and drive
|
||||
every route that reads, writes, lists or removes previews: they must touch
|
||||
only ``<folder>/<key>``, never ``<source>/thumbnails``, never through a link,
|
||||
and nothing at all while the folder is unavailable. Waits are events; no
|
||||
sleeps.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
try:
|
||||
from tests.test_thumbnail_auto import AutoThumbnailCase, _entry, WAIT
|
||||
from tests.test_api_images import make_png
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_thumbnail_auto import AutoThumbnailCase, _entry, WAIT
|
||||
from test_api_images import make_png
|
||||
|
||||
from py.api import preview_location, preview_store, thumbnail_settings # noqa: E402
|
||||
|
||||
SERVE = "/prompt_manager/images/serve/"
|
||||
|
||||
|
||||
class FolderModeCase(AutoThumbnailCase):
|
||||
"""The output root's previews live in ``<comfy>/thumbs/<key>``."""
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.folder = self.use_folder(self.comfy_dir / "thumbs")
|
||||
self.old_store = self.output_dir / "thumbnails"
|
||||
|
||||
def use_folder(self, folder, install="install-a"):
|
||||
folder.mkdir(exist_ok=True)
|
||||
marker_id = preview_location.init_folder(str(folder), created_by_pm=True)
|
||||
self.addCleanup(preview_store.configure, None)
|
||||
preview_store.configure(
|
||||
{"mode": "folder", "path": str(folder), "id": marker_id, "install": install}
|
||||
)
|
||||
self.assertTrue(preview_store.current().available)
|
||||
return folder
|
||||
|
||||
def key_folder(self):
|
||||
store = preview_store.ensure_store(self.output_dir, create=True)
|
||||
self.assertIsNotNone(store)
|
||||
return store
|
||||
|
||||
def preview(self, rel, ext=".webp"):
|
||||
return self.key_folder() / (thumbnail_settings.preview_stem(rel) + ext)
|
||||
|
||||
def marker(self):
|
||||
return json.loads(
|
||||
(self.folder / preview_location.MARKER_NAME).read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
|
||||
class TestFolderModeRoutes(FolderModeCase):
|
||||
async def test_the_key_is_a_readable_name_plus_a_hash_in_the_marker(self):
|
||||
key = self.key_folder().name
|
||||
self.assertRegex(key, r"^output-[0-9a-f]{8}$")
|
||||
entry = self.marker()["keys"][key]
|
||||
self.assertEqual(entry["path"], str(self.output_dir))
|
||||
self.assertEqual(entry["install"], "install-a")
|
||||
|
||||
async def test_another_case_of_the_source_finds_the_same_key(self):
|
||||
key = self.key_folder()
|
||||
other = Path(str(self.output_dir).swapcase())
|
||||
if not other.is_dir():
|
||||
self.skipTest("a case-sensitive volume")
|
||||
self.assertEqual(preview_store.store_dir(other), key)
|
||||
|
||||
@unittest.skipUnless(hasattr(os, "symlink"), "needs symlinks")
|
||||
async def test_a_linked_spelling_of_the_source_finds_the_same_key(self):
|
||||
key = self.key_folder()
|
||||
link = self.comfy_dir / "output-link"
|
||||
try:
|
||||
link.symlink_to(self.output_dir, target_is_directory=True)
|
||||
except OSError:
|
||||
self.skipTest("symlinks not permitted")
|
||||
self.assertEqual(preview_store.store_dir(link), key)
|
||||
|
||||
async def test_an_existing_preview_is_listed_and_served_from_the_key_folder(self):
|
||||
make_png(self.output_dir / "sub" / "a.png", size=(64, 64))
|
||||
preview = self.preview("sub/a.png")
|
||||
make_png(preview, size=(32, 32))
|
||||
url = _entry(self.api, self.output_dir, "sub/a.png")["thumbnail_url"]
|
||||
self.assertIn("v=", url)
|
||||
self.assertTrue(url.startswith(SERVE + "thumbnails/sub/a_thumb.webp"))
|
||||
resp = await self.client.get(url)
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(await resp.read(), preview.read_bytes())
|
||||
|
||||
async def test_a_missing_preview_is_made_in_the_key_folder(self):
|
||||
make_png(self.output_dir / "b.png", size=(640, 480))
|
||||
url = _entry(self.api, self.output_dir, "b.png")["thumbnail_url"]
|
||||
resp = await self.client.get(url)
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertTrue(self.preview("b.png").is_file())
|
||||
self.assertFalse(self.old_store.exists())
|
||||
|
||||
async def test_a_leftover_in_the_source_folder_is_never_served(self):
|
||||
make_png(self.output_dir / "c.png", size=(640, 480))
|
||||
self.old_store.mkdir()
|
||||
(self.old_store / "c_thumb.webp").write_bytes(b"leftover")
|
||||
resp = await self.client.get(SERVE + "thumbnails/c_thumb.webp?root=0")
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertNotEqual(await resp.read(), b"leftover")
|
||||
|
||||
async def test_other_text_under_thumbnails_is_refused(self):
|
||||
self.old_store.mkdir()
|
||||
make_png(self.old_store / "notes.png", size=(8, 8))
|
||||
resp = await self.client.get(SERVE + "thumbnails/notes.png?root=0")
|
||||
self.assertEqual(resp.status, 403)
|
||||
|
||||
async def test_traversal_through_thumbnails_is_refused(self):
|
||||
for text in (
|
||||
"thumbnails/../../thumbs/x_thumb.webp",
|
||||
"thumbnails/..%2F..%2Fx_thumb.webp",
|
||||
"thumbnails/%2e%2e/x_thumb.webp",
|
||||
):
|
||||
with self.subTest(text=text):
|
||||
resp = await self.client.get(SERVE + text + "?root=0")
|
||||
self.assertIn(resp.status, (403, 404))
|
||||
|
||||
@unittest.skipUnless(hasattr(os, "symlink"), "needs symlinks")
|
||||
async def test_a_link_out_of_the_key_folder_is_never_served_or_written(self):
|
||||
outside = self.comfy_dir / "outside"
|
||||
outside.mkdir()
|
||||
make_png(outside / "d_thumb.webp", size=(8, 8))
|
||||
make_png(self.output_dir / "sub" / "d.png", size=(64, 64))
|
||||
try:
|
||||
(self.key_folder() / "sub").symlink_to(outside, target_is_directory=True)
|
||||
except OSError:
|
||||
self.skipTest("symlinks not permitted")
|
||||
resp = await self.client.get(SERVE + "thumbnails/sub/d_thumb.webp?root=0")
|
||||
self.assertEqual(resp.status, 403)
|
||||
self.assertFalse(preview_store.owns(self.output_dir, outside / "d_thumb.webp"))
|
||||
self.assertFalse(
|
||||
preview_store.owns(self.output_dir, self.key_folder() / "sub" / "x.webp")
|
||||
)
|
||||
|
||||
@unittest.skipUnless(os.name == "nt", "junctions are Windows only")
|
||||
async def test_a_junction_inside_a_store_is_refused(self):
|
||||
import _winapi
|
||||
|
||||
outside = self.comfy_dir / "outside"
|
||||
outside.mkdir()
|
||||
(outside / "e_thumb.webp").write_bytes(b"x")
|
||||
_winapi.CreateJunction(str(outside), str(self.key_folder() / "sub"))
|
||||
self.assertTrue(preview_location.is_link_like(self.key_folder() / "sub"))
|
||||
resp = await self.client.get(SERVE + "thumbnails/sub/e_thumb.webp?root=0")
|
||||
self.assertEqual(resp.status, 403)
|
||||
|
||||
async def test_capture_writes_into_the_key_folder(self):
|
||||
path = make_png(self.output_dir / "run" / "f.png", size=(64, 64))
|
||||
self.assertEqual(self.api.queue_capture_thumbnail(str(path)), "queued")
|
||||
await self._idle()
|
||||
self.assertTrue(self.preview("run/f.png").is_file())
|
||||
self.assertFalse(self.old_store.exists())
|
||||
|
||||
async def test_clear_removes_previews_only(self):
|
||||
make_png(self.output_dir / "g.png", size=(64, 64))
|
||||
payload = await asyncio.get_running_loop().run_in_executor(
|
||||
None, self.api._generate_gallery_thumbnails_sync, [self.output_dir]
|
||||
)
|
||||
self.assertTrue(payload["success"])
|
||||
keep = self.key_folder() / "readme.txt"
|
||||
keep.write_text("not a preview", encoding="utf-8")
|
||||
resp = await self.client.post("/prompt_manager/images/clear-thumbnails")
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertFalse(self.preview("g.png").exists())
|
||||
self.assertTrue(keep.exists())
|
||||
self.assertTrue((self.folder / preview_location.MARKER_NAME).is_file())
|
||||
self.assertTrue(self.output_dir.joinpath("g.png").is_file())
|
||||
|
||||
async def test_removing_a_file_removes_its_preview_in_the_key_folder(self):
|
||||
from py.api.admin import _remove_thumbnail
|
||||
|
||||
make_png(self.output_dir / "h.png", size=(64, 64))
|
||||
preview = self.preview("h.png")
|
||||
preview.write_bytes(b"x")
|
||||
self.assertTrue(_remove_thumbnail(self.output_dir, Path("h.png")))
|
||||
self.assertFalse(preview.exists())
|
||||
|
||||
async def test_history_cover_is_authorized_by_the_store(self):
|
||||
from py.api import history_current
|
||||
|
||||
preview = self.preview("i.png")
|
||||
preview.write_bytes(b"x")
|
||||
target = history_current._canonical(preview)
|
||||
self.assertTrue(
|
||||
history_current._target_authorized(
|
||||
target, self.output_dir, None, self.output_dir
|
||||
)
|
||||
)
|
||||
stray = self.comfy_dir / "stray_thumb.webp"
|
||||
stray.write_bytes(b"x")
|
||||
self.assertFalse(
|
||||
history_current._target_authorized(
|
||||
history_current._canonical(stray),
|
||||
self.output_dir,
|
||||
None,
|
||||
self.output_dir,
|
||||
)
|
||||
)
|
||||
|
||||
async def test_the_thumbnail_folder_is_never_recreated_by_a_write(self):
|
||||
make_png(self.output_dir / "j.png", size=(64, 64))
|
||||
self.key_folder()
|
||||
for child in sorted(self.folder.rglob("*"), reverse=True):
|
||||
child.rmdir() if child.is_dir() else child.unlink()
|
||||
self.folder.rmdir()
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
preview_store.make_parent(self.folder / "output-x" / "sub" / "a.webp")
|
||||
self.assertFalse(self.folder.exists())
|
||||
|
||||
|
||||
class TestUnavailableFolder(FolderModeCase):
|
||||
def go_missing(self):
|
||||
(self.folder / preview_location.MARKER_NAME).unlink()
|
||||
self.assertFalse(preview_store.refresh())
|
||||
location = preview_store.current()
|
||||
self.assertEqual((location.available, location.problem), (False, "missing"))
|
||||
|
||||
async def test_a_folder_without_its_marker_is_missing_and_writes_nothing(self):
|
||||
make_png(self.output_dir / "k.png", size=(640, 480))
|
||||
self.go_missing()
|
||||
entry = _entry(self.api, self.output_dir, "k.png")
|
||||
self.assertFalse(entry.get("thumbnail_ready", False))
|
||||
resp = await self.client.get(SERVE + "thumbnails/k_thumb.webp?root=0")
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(resp.headers["Cache-Control"], "no-store")
|
||||
self.assertEqual(
|
||||
await resp.read(), (self.output_dir / "k.png").read_bytes()
|
||||
) # the original
|
||||
self.assertEqual(
|
||||
sorted(p.name for p in self.folder.rglob("*") if p.is_file()), []
|
||||
)
|
||||
self.assertFalse(self.old_store.exists())
|
||||
|
||||
async def test_thumbnail_work_is_refused_while_unavailable(self):
|
||||
self.go_missing()
|
||||
for method, url, body in (
|
||||
("POST", "/prompt_manager/images/generate-thumbnails", {}),
|
||||
("POST", "/prompt_manager/images/generate-thumbnails/progress", None),
|
||||
("POST", "/prompt_manager/images/clear-thumbnails", None),
|
||||
("POST", "/prompt_manager/jobs/thumbnails", {"mode": "missing"}),
|
||||
):
|
||||
with self.subTest(url=url):
|
||||
resp = await self.client.request(method, url, json=body)
|
||||
self.assertEqual(resp.status, 409)
|
||||
self.assertEqual((await resp.json())["code"], "location-unavailable")
|
||||
|
||||
async def test_status_reports_the_location_without_a_path(self):
|
||||
self.go_missing()
|
||||
resp = await self.client.get("/prompt_manager/images/thumbnails/status")
|
||||
location = (await resp.json())["location"]
|
||||
self.assertEqual(
|
||||
location, {"mode": "folder", "available": False, "problem": "missing"}
|
||||
)
|
||||
|
||||
async def test_another_folders_marker_is_a_mismatch(self):
|
||||
marker = self.marker()
|
||||
marker["id"] = "someone-else"
|
||||
(self.folder / preview_location.MARKER_NAME).write_text(
|
||||
json.dumps(marker), encoding="utf-8"
|
||||
)
|
||||
self.assertFalse(preview_store.refresh())
|
||||
self.assertEqual(preview_store.current().problem, "marker-mismatch")
|
||||
|
||||
async def test_the_temp_sweep_skips_an_unavailable_folder(self):
|
||||
stray = self.key_folder() / ".l_thumb.part-1-1.webp"
|
||||
stray.write_bytes(b"x")
|
||||
old = os.path.getmtime(stray) - 10 * 24 * 3600
|
||||
os.utime(stray, (old, old))
|
||||
self.go_missing()
|
||||
self.assertEqual(self.api.sweep_thumbnail_temps(), 0)
|
||||
self.assertTrue(stray.exists())
|
||||
|
||||
async def test_a_stalled_store_call_marks_it_unreachable_and_stops_io(self):
|
||||
make_png(self.output_dir / "m.png", size=(64, 64))
|
||||
self.preview("m.png").write_bytes(b"x")
|
||||
ticks = iter([0.0, preview_store.IO_BUDGET + 1.0])
|
||||
with patch.object(preview_store, "_clock", lambda: next(ticks, 99.0)):
|
||||
_entry(self.api, self.output_dir, "m.png") # one slow lookup
|
||||
self.assertEqual(preview_store.current().problem, "unreachable")
|
||||
calls = []
|
||||
real = thumbnail_settings.owned_in_folder
|
||||
with patch.object(
|
||||
thumbnail_settings,
|
||||
"owned_in_folder",
|
||||
lambda *a, **k: calls.append(a) or real(*a, **k),
|
||||
):
|
||||
entry = _entry(self.api, self.output_dir, "m.png")
|
||||
self.assertEqual(calls, []) # no store I/O while unreachable
|
||||
self.assertNotIn("v=", entry["thumbnail_url"])
|
||||
self.assertTrue(preview_store.refresh()) # the probe finds it again
|
||||
|
||||
async def test_a_check_that_blocks_is_unreachable_within_the_budget(self):
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
self.addCleanup(release.set)
|
||||
|
||||
def blocking(path, expected_id):
|
||||
entered.set()
|
||||
release.wait(WAIT)
|
||||
return True, None, None
|
||||
|
||||
with (
|
||||
patch.object(preview_store, "IO_BUDGET", 0.2),
|
||||
patch.object(preview_location, "folder_state", blocking),
|
||||
):
|
||||
self.assertFalse(preview_store.refresh())
|
||||
self.assertTrue(entered.is_set())
|
||||
self.assertEqual(preview_store.current().problem, "unreachable")
|
||||
release.set()
|
||||
|
||||
async def test_an_invalid_config_value_writes_nothing_anywhere(self):
|
||||
preview_store.configure({"mode": "folder", "path": "relative/thumbs"})
|
||||
location = preview_store.current()
|
||||
self.assertEqual((location.available, location.problem), (False, "invalid"))
|
||||
make_png(self.output_dir / "n.png", size=(640, 480))
|
||||
resp = await self.client.get(SERVE + "thumbnails/n_thumb.webp?root=0")
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertFalse(self.old_store.exists())
|
||||
|
||||
|
||||
class TestSharedFolders(FolderModeCase):
|
||||
async def test_two_installs_keep_each_others_keys(self):
|
||||
self.key_folder()
|
||||
other_root = self.comfy_dir / "other"
|
||||
other_root.mkdir()
|
||||
setting = preview_store.setting()
|
||||
preview_store.configure(
|
||||
{
|
||||
"mode": "folder",
|
||||
"path": setting.path,
|
||||
"id": setting.id,
|
||||
"install": "install-b",
|
||||
}
|
||||
)
|
||||
self.assertIsNotNone(preview_store.ensure_store(other_root))
|
||||
keys = self.marker()["keys"]
|
||||
installs = sorted(entry["install"] for entry in keys.values())
|
||||
self.assertEqual(installs, ["install-a", "install-b"])
|
||||
|
||||
async def test_init_folder_adopts_an_existing_marker(self):
|
||||
marker_id = self.marker()["id"]
|
||||
self.assertEqual(preview_location.init_folder(str(self.folder)), marker_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestHostileMarker(FolderModeCase):
|
||||
"""A hand-edited or foreign marker never takes a write or a delete out of
|
||||
the thumbnail folder (shared and UNC folders can be edited by others)."""
|
||||
|
||||
def plant(self, keys):
|
||||
marker = self.marker()
|
||||
marker["keys"] = keys
|
||||
(self.folder / preview_location.MARKER_NAME).write_text(
|
||||
json.dumps(marker), encoding="utf-8"
|
||||
)
|
||||
self.assertTrue(preview_store.refresh())
|
||||
|
||||
def outside_files(self):
|
||||
return sorted(
|
||||
p.relative_to(self.comfy_dir).as_posix()
|
||||
for p in self.comfy_dir.rglob("*")
|
||||
if p.is_file()
|
||||
and not p.is_relative_to(self.folder)
|
||||
and not p.is_relative_to(self.output_dir)
|
||||
and p.suffix in (".webp", ".jpg", ".png")
|
||||
)
|
||||
|
||||
async def test_keys_that_are_not_one_folder_name_are_dropped(self):
|
||||
entry = {"path": str(self.output_dir), "identity": "x", "install": "i"}
|
||||
self.plant(
|
||||
{
|
||||
"../escape-0123abcd": entry,
|
||||
str(self.comfy_dir / "abs-0123abcd"): entry,
|
||||
"a/b-0123abcd": entry,
|
||||
"a\\b-0123abcd": entry,
|
||||
"Upper-0123abcd": entry,
|
||||
"nohash": entry,
|
||||
"fine-0123abcd": "not a dict",
|
||||
}
|
||||
)
|
||||
self.assertEqual(preview_location.read_marker(str(self.folder))["keys"], {})
|
||||
store = self.key_folder()
|
||||
self.assertEqual(store.parent, self.folder)
|
||||
make_png(self.output_dir / "p.png", size=(640, 480))
|
||||
resp = await self.client.get(SERVE + "thumbnails/p_thumb.webp?root=0")
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertTrue((store / "p_thumb.webp").is_file())
|
||||
self.assertEqual(self.outside_files(), [])
|
||||
|
||||
@unittest.skipUnless(hasattr(os, "symlink"), "needs symlinks")
|
||||
async def test_a_key_folder_that_is_a_link_is_never_written_or_cleared(self):
|
||||
outside = self.comfy_dir / "outside"
|
||||
outside.mkdir()
|
||||
victim = outside / "q_thumb.webp"
|
||||
victim.write_bytes(b"keep me")
|
||||
store = self.key_folder()
|
||||
store.rmdir()
|
||||
try:
|
||||
store.symlink_to(outside, target_is_directory=True)
|
||||
except OSError:
|
||||
self.skipTest("symlinks not permitted")
|
||||
make_png(self.output_dir / "q.png", size=(640, 480))
|
||||
from py.api.admin import _remove_thumbnail
|
||||
|
||||
self.assertFalse(_remove_thumbnail(self.output_dir, Path("q.png")))
|
||||
resp = await self.client.post("/prompt_manager/images/clear-thumbnails")
|
||||
self.assertIn(resp.status, (200, 400))
|
||||
resp = await self.client.get(SERVE + "thumbnails/q_thumb.webp?root=0")
|
||||
self.assertIn(resp.status, (200, 403))
|
||||
self.assertEqual(victim.read_bytes(), b"keep me")
|
||||
self.assertEqual(sorted(p.name for p in outside.iterdir()), ["q_thumb.webp"])
|
||||
|
||||
@unittest.skipUnless(hasattr(os, "symlink"), "needs symlinks")
|
||||
async def test_a_linked_folder_below_a_key_is_never_deleted_through(self):
|
||||
outside = self.comfy_dir / "outside"
|
||||
outside.mkdir()
|
||||
victim = outside / "r_thumb.webp"
|
||||
victim.write_bytes(b"keep me")
|
||||
try:
|
||||
(self.key_folder() / "sub").symlink_to(outside, target_is_directory=True)
|
||||
except OSError:
|
||||
self.skipTest("symlinks not permitted")
|
||||
make_png(self.output_dir / "sub" / "r.png", size=(64, 64))
|
||||
from py.api.admin import _remove_thumbnail
|
||||
|
||||
self.assertFalse(_remove_thumbnail(self.output_dir, Path("sub/r.png")))
|
||||
resp = await self.client.post("/prompt_manager/images/clear-thumbnails")
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(victim.read_bytes(), b"keep me")
|
||||
|
||||
@unittest.skipUnless(os.name == "nt", "junctions are Windows only")
|
||||
async def test_a_key_folder_that_is_a_junction_is_never_written_or_cleared(self):
|
||||
import _winapi
|
||||
|
||||
outside = self.comfy_dir / "outside"
|
||||
outside.mkdir()
|
||||
victim = outside / "v_thumb.webp"
|
||||
victim.write_bytes(b"keep me")
|
||||
store = self.key_folder()
|
||||
store.rmdir()
|
||||
_winapi.CreateJunction(str(outside), str(store))
|
||||
make_png(self.output_dir / "v.png", size=(640, 480))
|
||||
from py.api.admin import _remove_thumbnail
|
||||
|
||||
self.assertFalse(_remove_thumbnail(self.output_dir, Path("v.png")))
|
||||
resp = await self.client.post("/prompt_manager/images/clear-thumbnails")
|
||||
self.assertIn(resp.status, (200, 400))
|
||||
resp = await self.client.get(SERVE + "thumbnails/v_thumb.webp?root=0")
|
||||
self.assertIn(resp.status, (200, 403))
|
||||
self.assertEqual(victim.read_bytes(), b"keep me")
|
||||
self.assertEqual(sorted(p.name for p in outside.iterdir()), ["v_thumb.webp"])
|
||||
|
||||
async def test_a_write_outside_the_folder_is_refused(self):
|
||||
with self.assertRaises(OSError):
|
||||
preview_store.make_parent(self.comfy_dir / "elsewhere" / "a_thumb.webp")
|
||||
self.assertFalse((self.comfy_dir / "elsewhere").exists())
|
||||
|
||||
async def test_an_unmounted_folder_takes_no_write_before_the_next_check(self):
|
||||
"""An unmounted mount point is an empty folder: the flag may still
|
||||
say available, but without the marker nothing is made there."""
|
||||
store = self.key_folder()
|
||||
store.rmdir()
|
||||
(self.folder / preview_location.MARKER_NAME).unlink()
|
||||
self.assertTrue(preview_store.current().available) # not checked yet
|
||||
with self.assertRaises(OSError):
|
||||
preview_store.make_parent(store / "sub" / "s_thumb.webp")
|
||||
self.assertEqual(list(self.folder.iterdir()), [])
|
||||
|
||||
|
||||
class TestStartupNeverFallsBackToTheSources(AutoThumbnailCase):
|
||||
async def test_a_folder_setting_is_in_force_before_its_check(self):
|
||||
self.addCleanup(preview_store.configure, None)
|
||||
folder = self.comfy_dir / "thumbs"
|
||||
value = {"mode": "folder", "path": str(folder), "id": "x"}
|
||||
preview_store.configure(value, check=False)
|
||||
location = preview_store.current()
|
||||
self.assertEqual((location.mode, location.available), ("folder", False))
|
||||
make_png(self.output_dir / "t.png", size=(640, 480))
|
||||
resp = await self.client.get(SERVE + "thumbnails/t_thumb.webp?root=0")
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertFalse((self.output_dir / "thumbnails").exists())
|
||||
|
||||
async def test_a_failed_startup_check_leaves_the_folder_invalid(self):
|
||||
from py.config import PromptManagerConfig
|
||||
|
||||
self.addCleanup(preview_store.configure, None)
|
||||
value = {"mode": "folder", "path": str(self.comfy_dir / "thumbs"), "id": "x"}
|
||||
with (
|
||||
patch.object(PromptManagerConfig, "THUMBNAIL_LOCATION", value),
|
||||
patch.object(
|
||||
type(self.api),
|
||||
"_thumbnail_location_context",
|
||||
side_effect=RuntimeError("boom"),
|
||||
),
|
||||
):
|
||||
self.api.start_preview_location()
|
||||
location = preview_store.current()
|
||||
self.assertEqual(
|
||||
(location.mode, location.available, location.problem),
|
||||
("folder", False, "invalid"),
|
||||
)
|
||||
|
||||
|
||||
class TestNoStoreIoOnTheEventLoop(FolderModeCase):
|
||||
async def test_legacy_listings_do_not_stat_the_folder(self):
|
||||
make_png(self.output_dir / "u.png", size=(64, 64))
|
||||
self.preview("u.png").write_bytes(b"x")
|
||||
calls = []
|
||||
real = thumbnail_settings.owned_in_folder
|
||||
with patch.object(
|
||||
thumbnail_settings,
|
||||
"owned_in_folder",
|
||||
lambda *a, **k: calls.append(threading.current_thread().name)
|
||||
or real(*a, **k),
|
||||
):
|
||||
images = self.api._enrich_images(
|
||||
[{"image_path": str(self.output_dir / "u.png")}]
|
||||
)
|
||||
self.assertEqual(calls, [])
|
||||
self.assertEqual(images[0]["thumbnail_url"], SERVE + "thumbnails/u_thumb.webp")
|
||||
@@ -291,7 +291,7 @@ class TestThumbnailOnFirstView(AutoThumbnailCase):
|
||||
await self._writers(state, auto.ON_DEMAND_LIMIT)
|
||||
await self._waiting(len(names))
|
||||
self.assertEqual(default_calls, [], "no view waits on a default thread")
|
||||
self.assertTrue(all(name.startswith("pm-preview") for name in state["threads"]))
|
||||
self.assertTrue(all(name.startswith("pm-image") for name in state["threads"]))
|
||||
status = await self.client.get("/prompt_manager/images/thumbnails/status")
|
||||
self.assertEqual(status.status, 200, "another route answers meanwhile")
|
||||
await status.read()
|
||||
@@ -678,7 +678,7 @@ class TestShutdown(AutoThumbnailCase):
|
||||
_entry(self.api, self.output_dir, "viewed.png")["thumbnail_url"]
|
||||
)
|
||||
await resp.read()
|
||||
pool_threads = list(self.api._preview_pool_threads())
|
||||
pool_threads = list(self.api._image_pool_threads())
|
||||
self.assertTrue(pool_threads)
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(
|
||||
@@ -904,7 +904,8 @@ class TestThumbnailOnCapture(AutoThumbnailCase):
|
||||
self.api.queue_capture_thumbnail(str(path))
|
||||
release.set()
|
||||
await self._idle()
|
||||
self.assertEqual(batches, [(1, 1), (2, 2), (2, 2)])
|
||||
# The job pool's size is the setting; each batch is at most that (#523).
|
||||
self.assertEqual(batches, [(1, 2), (2, 2), (2, 2)])
|
||||
self.assertLessEqual(state["peak"], 2)
|
||||
for i in range(5):
|
||||
self.assertTrue(
|
||||
|
||||
@@ -0,0 +1,414 @@
|
||||
"""Move thumbnails (#320 PR 3): the reviewed job in every direction, its
|
||||
failure, cancel and restart paths, Recreate and the removed-sources scope.
|
||||
|
||||
Each test runs against a temp config.json (``AdminAPITestCase``) and a temp
|
||||
journal; jobs are awaited through their task, never a timer.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
try:
|
||||
from tests.test_api_admin import AdminAPITestCase
|
||||
from tests.test_api_images import make_png
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_admin import AdminAPITestCase
|
||||
from test_api_images import make_png
|
||||
|
||||
from py.api import preview_location, preview_store # noqa: E402
|
||||
from py.api import thumbnail_move_files as files # noqa: E402
|
||||
from py.api import thumbnail_move_plan as move_plan # noqa: E402
|
||||
from py.config import PromptManagerConfig # noqa: E402
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
PLAN = "/prompt_manager/images/thumbnails/plan"
|
||||
START = "/prompt_manager/jobs/thumbnails"
|
||||
|
||||
|
||||
class MoveCase(AdminAPITestCase):
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.api._get_all_output_dirs = lambda: [self.output_dir]
|
||||
self.api.recover_thumbnail_move()
|
||||
self.journal = Path(self.tmpdir) / "data" / "thumbnail-move.json"
|
||||
for target in (move_plan,):
|
||||
patcher = patch.object(target, "journal_path", lambda: str(self.journal))
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
default = patch(
|
||||
"py.api.thumbnail_move_plan.default_path",
|
||||
lambda: str(Path(self.tmpdir) / "default-thumbs"),
|
||||
)
|
||||
default.start()
|
||||
self.addCleanup(default.stop)
|
||||
self.addCleanup(preview_store.configure, None)
|
||||
self.addCleanup(setattr, PromptManagerConfig, "THUMBNAIL_LOCATION", None)
|
||||
self.old_store = self.output_dir / "thumbnails"
|
||||
self.folder = Path(self.tmpdir) / "thumbs"
|
||||
# Two media files with previews, an orphan, a temp and a note.
|
||||
make_png(self.output_dir / "a.png", size=(8, 8))
|
||||
make_png(self.output_dir / "sub" / "b.png", size=(8, 8))
|
||||
make_png(self.old_store / "a_thumb.webp", size=(4, 4))
|
||||
make_png(self.old_store / "sub" / "b_thumb.webp", size=(4, 4))
|
||||
make_png(self.old_store / "gone_thumb.webp", size=(4, 4))
|
||||
(self.old_store / ".a_thumb.part-1-1.webp").write_bytes(b"t")
|
||||
(self.old_store / "notes.txt").write_text("mine", encoding="utf-8")
|
||||
|
||||
async def plan(self, location):
|
||||
resp = await self.client.post(PLAN, json={"mode": "move", "location": location})
|
||||
return resp.status, await resp.json()
|
||||
|
||||
async def move(self, location, confirmed=True):
|
||||
status, plan = await self.plan(location)
|
||||
self.assertEqual(status, 200, plan)
|
||||
resp = await self.client.post(
|
||||
START, json={"mode": "move", "token": plan["token"], "confirmed": confirmed}
|
||||
)
|
||||
body = await resp.json()
|
||||
self.assertEqual(resp.status, 202, body)
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await job.task
|
||||
return plan, job
|
||||
|
||||
def config(self):
|
||||
with open(self.config_path, encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
def key_files(self, folder=None):
|
||||
folder = folder or self.folder
|
||||
return sorted(
|
||||
p.relative_to(folder).as_posix()
|
||||
for p in folder.rglob("*")
|
||||
if p.is_file() and p.name != preview_location.MARKER_NAME
|
||||
)
|
||||
|
||||
|
||||
class TestMoveToAFolder(MoveCase):
|
||||
async def test_review_counts_what_moves_is_removed_and_stays(self):
|
||||
status, plan = await self.plan({"mode": "folder", "path": str(self.folder)})
|
||||
self.assertEqual(status, 200, plan)
|
||||
totals = plan["totals"]
|
||||
self.assertEqual(
|
||||
(
|
||||
totals["files"],
|
||||
totals["orphans"],
|
||||
totals["temps"],
|
||||
totals["left_in_place"],
|
||||
),
|
||||
(2, 1, 1, 1),
|
||||
)
|
||||
self.assertTrue(plan["confirm_required"])
|
||||
self.assertTrue(plan["to"]["will_create"])
|
||||
self.assertFalse(self.folder.exists()) # the review writes nothing
|
||||
|
||||
async def test_a_move_copies_switches_and_removes_old_copies(self):
|
||||
before = (self.output_dir / "a.png").read_bytes()
|
||||
_plan, job = await self.move({"mode": "folder", "path": str(self.folder)})
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertEqual(job.counters["moved"], 2)
|
||||
keys = self.key_files()
|
||||
self.assertEqual(len(keys), 2)
|
||||
self.assertTrue(all(re.match(r"^output-[0-9a-f]{8}/", k) for k in keys))
|
||||
location = self.config()["thumbnails"]["location"]
|
||||
self.assertEqual(
|
||||
(location["mode"], location["path"]), ("folder", str(self.folder))
|
||||
)
|
||||
self.assertEqual(location["cleanup"], [])
|
||||
self.assertEqual(preview_store.current().mode, "folder")
|
||||
# Old copies, the orphan and the temp went; the note stays, so its
|
||||
# folder stays; originals are untouched.
|
||||
self.assertEqual(
|
||||
sorted(p.name for p in self.old_store.rglob("*")), ["notes.txt"]
|
||||
)
|
||||
self.assertEqual((self.output_dir / "a.png").read_bytes(), before)
|
||||
self.assertFalse(self.journal.exists())
|
||||
copied = next(self.folder.rglob("a_thumb.webp"))
|
||||
self.assertEqual(copied.stat().st_mtime_ns % files.MTIME_STEP_NS, 0)
|
||||
|
||||
async def test_the_move_back_removes_the_folder_it_made(self):
|
||||
await self.move({"mode": "folder", "path": str(self.folder)})
|
||||
_plan, job = await self.move({"mode": "source"})
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertTrue((self.old_store / "a_thumb.webp").is_file())
|
||||
self.assertTrue((self.old_store / "sub" / "b_thumb.webp").is_file())
|
||||
self.assertFalse(self.folder.exists())
|
||||
self.assertEqual(self.config()["thumbnails"]["location"]["mode"], "source")
|
||||
|
||||
async def test_folder_to_folder_keeps_the_key_names(self):
|
||||
await self.move({"mode": "folder", "path": str(self.folder)})
|
||||
first = self.key_files()
|
||||
other = Path(self.tmpdir) / "thumbs2"
|
||||
_plan, job = await self.move({"mode": "folder", "path": str(other)})
|
||||
self.assertEqual(job.state, "finished", job.log)
|
||||
self.assertEqual(self.key_files(other), first)
|
||||
self.assertFalse(self.folder.exists())
|
||||
|
||||
async def test_previews_of_sources_no_longer_listed_go_when_leaving(self):
|
||||
await self.move({"mode": "folder", "path": str(self.folder)})
|
||||
setting = preview_store.setting()
|
||||
|
||||
def add(marker):
|
||||
marker["keys"]["old-0123abcd"] = {
|
||||
"path": "/gone",
|
||||
"identity": "/gone",
|
||||
"install": setting.install,
|
||||
}
|
||||
|
||||
preview_location.write_marker(str(self.folder), add)
|
||||
make_png(self.folder / "old-0123abcd" / "x_thumb.webp", size=(4, 4))
|
||||
preview_store.refresh()
|
||||
status, plan = await self.plan({"mode": "source"})
|
||||
self.assertEqual(plan["totals"]["unlisted_sources"], 1)
|
||||
_plan, job = await self.move({"mode": "source"})
|
||||
self.assertFalse(self.folder.exists())
|
||||
|
||||
async def test_confirmation_and_a_stale_token(self):
|
||||
status, plan = await self.plan({"mode": "folder", "path": str(self.folder)})
|
||||
resp = await self.client.post(
|
||||
START, json={"mode": "move", "token": plan["token"]}
|
||||
)
|
||||
self.assertEqual((await resp.json())["code"], "confirmation-required")
|
||||
resp = await self.client.post(
|
||||
START, json={"mode": "move", "token": "nope", "confirmed": True}
|
||||
)
|
||||
self.assertEqual(resp.status, 409)
|
||||
self.assertEqual((await resp.json())["code"], "plan-stale")
|
||||
|
||||
async def test_a_folder_that_is_too_small_is_refused(self):
|
||||
with patch.object(move_plan, "_free", lambda _path: 1):
|
||||
status, body = await self.plan({"mode": "folder", "path": str(self.folder)})
|
||||
self.assertEqual(status, 507)
|
||||
self.assertEqual(body["code"], "insufficient-space")
|
||||
|
||||
async def test_validation_refusals_come_back_in_place(self):
|
||||
status, body = await self.plan(
|
||||
{"mode": "folder", "path": str(self.output_dir / "thumbs")}
|
||||
)
|
||||
self.assertEqual((status, body["code"]), (400, "location-served"))
|
||||
status, body = await self.plan({"mode": "source"})
|
||||
self.assertEqual(body["code"], "location-same")
|
||||
|
||||
|
||||
class TestFailures(MoveCase):
|
||||
async def test_a_per_file_error_still_switches_and_removes_its_old_copy(self):
|
||||
real = files.copy_one
|
||||
|
||||
def failing(source, destination, *args, **kwargs):
|
||||
if Path(source).name == "a_thumb.webp":
|
||||
raise files.CopyFailed("Could not be read; made again on view")
|
||||
return real(source, destination, *args, **kwargs)
|
||||
|
||||
with patch.object(files, "copy_one", failing):
|
||||
_plan, job = await self.move({"mode": "folder", "path": str(self.folder)})
|
||||
self.assertEqual(job.state, "finished")
|
||||
self.assertEqual((job.counters["moved"], job.counters["errors"]), (1, 1))
|
||||
self.assertFalse((self.old_store / "a_thumb.webp").exists())
|
||||
|
||||
async def test_a_folder_wide_error_changes_nothing(self):
|
||||
def gone(*_args, **_kwargs):
|
||||
raise files.FolderGone("disk full")
|
||||
|
||||
with patch.object(files, "copy_one", gone):
|
||||
_plan, job = await self.move({"mode": "folder", "path": str(self.folder)})
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertIn("nothing changed", job.status_line)
|
||||
self.assertFalse(self.folder.exists())
|
||||
self.assertTrue((self.old_store / "a_thumb.webp").is_file())
|
||||
self.assertEqual(preview_store.current().mode, "source")
|
||||
self.assertFalse(self.journal.exists())
|
||||
|
||||
async def test_cancel_before_the_switch_changes_nothing(self):
|
||||
status, plan = await self.plan({"mode": "folder", "path": str(self.folder)})
|
||||
real = self.api._move_copy_one
|
||||
|
||||
def cancel_first(pair, **kwargs):
|
||||
job = self.api._jobs.active("thumbnails")
|
||||
job.cancelled = True
|
||||
return real(pair, **kwargs)
|
||||
|
||||
with patch.object(self.api, "_move_copy_one", cancel_first):
|
||||
resp = await self.client.post(
|
||||
START, json={"mode": "move", "token": plan["token"], "confirmed": True}
|
||||
)
|
||||
job = self.api._jobs.get((await resp.json())["job"]["id"])
|
||||
await job.task
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertFalse(self.folder.exists())
|
||||
self.assertEqual(preview_store.current().mode, "source")
|
||||
|
||||
async def test_an_exit_during_the_copy_leaves_the_journal_for_the_next_start(self):
|
||||
status, plan = await self.plan({"mode": "folder", "path": str(self.folder)})
|
||||
real = self.api._move_copy_one
|
||||
exiting = {"now": False}
|
||||
|
||||
def exit_after(pair, **kwargs):
|
||||
result = real(pair, **kwargs)
|
||||
exiting["now"] = True # the interpreter starts exiting
|
||||
return result
|
||||
|
||||
from py.api import thumbnail_move
|
||||
|
||||
with (
|
||||
patch.object(self.api, "_move_copy_one", exit_after),
|
||||
patch.object(
|
||||
thumbnail_move.exit_signal, "requested", lambda: exiting["now"]
|
||||
),
|
||||
):
|
||||
resp = await self.client.post(
|
||||
START, json={"mode": "move", "token": plan["token"], "confirmed": True}
|
||||
)
|
||||
job = self.api._jobs.get((await resp.json())["job"]["id"])
|
||||
await job.task
|
||||
self.assertTrue(self.journal.exists())
|
||||
self.assertTrue(self.key_files()) # partial copies were left
|
||||
# The next start undoes it: copies and the created folder go.
|
||||
self.api._move_recovered = False
|
||||
self.api.recover_thumbnail_move()
|
||||
self.assertFalse(self.journal.exists())
|
||||
self.assertFalse(self.folder.exists())
|
||||
self.assertTrue((self.old_store / "a_thumb.webp").is_file())
|
||||
|
||||
|
||||
class TestRestart(MoveCase):
|
||||
async def test_a_journal_left_after_the_switch_is_forgotten(self):
|
||||
await self.move({"mode": "folder", "path": str(self.folder)})
|
||||
self.journal.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.journal.write_text(
|
||||
json.dumps(
|
||||
{"state": "copying", "to": {"mode": "folder", "path": str(self.folder)}}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
self.api._move_recovered = False
|
||||
self.api.recover_thumbnail_move()
|
||||
self.assertFalse(self.journal.exists())
|
||||
self.assertTrue(self.key_files())
|
||||
|
||||
async def test_old_copies_that_waited_are_removed_by_the_finishing_job(self):
|
||||
with patch.object(files, "remove_file", lambda path, **_k: False):
|
||||
_plan, job = await self.move({"mode": "folder", "path": str(self.folder)})
|
||||
self.assertTrue(preview_store.setting().cleanup)
|
||||
self.assertTrue((self.old_store / "a_thumb.webp").exists())
|
||||
job = await self.api.finish_thumbnail_move()
|
||||
await job.task
|
||||
self.assertEqual(job.title, "Move thumbnails (finishing)")
|
||||
self.assertFalse((self.old_store / "a_thumb.webp").exists())
|
||||
self.assertEqual(self.config()["thumbnails"]["location"]["cleanup"], [])
|
||||
|
||||
|
||||
class TestAdmission(MoveCase):
|
||||
async def test_automatic_previews_and_setting_changes_wait_for_the_move(self):
|
||||
status, plan = await self.plan({"mode": "folder", "path": str(self.folder)})
|
||||
seen = {}
|
||||
real = self.api._move_copy_one
|
||||
|
||||
def observe(pair, **kwargs):
|
||||
seen["blocker"] = self.api._auto_thumbnail_blocker()
|
||||
return real(pair, **kwargs)
|
||||
|
||||
with patch.object(self.api, "_move_copy_one", observe):
|
||||
resp = await self.client.post(
|
||||
START, json={"mode": "move", "token": plan["token"], "confirmed": True}
|
||||
)
|
||||
job = self.api._jobs.get((await resp.json())["job"]["id"])
|
||||
refused = self.api._thumbnail_change_refused({"THUMBNAIL_QUALITY": 5})
|
||||
self.assertEqual(refused.status, 409)
|
||||
await job.task
|
||||
self.assertIs(seen["blocker"], job)
|
||||
|
||||
def test_the_move_never_trims_memory_and_copies_on_the_job_pool(self):
|
||||
text = "".join(
|
||||
(REPO / "py" / "api" / name).read_text(encoding="utf-8")
|
||||
for name in ("thumbnail_move.py", "thumbnail_prune.py")
|
||||
)
|
||||
self.assertNotIn("_release_job_memory", text)
|
||||
self.assertIn("map_parallel", (REPO / "py/api/thumbnail_move.py").read_text())
|
||||
|
||||
|
||||
class TestPrune(MoveCase):
|
||||
async def test_only_this_installs_removed_sources_go(self):
|
||||
await self.move({"mode": "folder", "path": str(self.folder)})
|
||||
install = preview_store.setting().install
|
||||
|
||||
def add(marker):
|
||||
marker["keys"]["old-0123abcd"] = {"identity": "/gone", "install": install}
|
||||
marker["keys"]["theirs-0123abcd"] = {"identity": "/x", "install": "other"}
|
||||
|
||||
preview_location.write_marker(str(self.folder), add)
|
||||
make_png(self.folder / "old-0123abcd" / "x_thumb.webp", size=(4, 4))
|
||||
make_png(self.folder / "theirs-0123abcd" / "y_thumb.webp", size=(4, 4))
|
||||
preview_store.refresh()
|
||||
resp = await self.client.get(PLAN + "?mode=prune")
|
||||
body = await resp.json()
|
||||
self.assertEqual([s["key"] for s in body["sources"]], ["old-0123abcd"])
|
||||
self.assertIn("1 source no longer listed", body["line"])
|
||||
resp = await self.client.post(START, json={"mode": "prune"})
|
||||
job = self.api._jobs.get((await resp.json())["job"]["id"])
|
||||
await job.task
|
||||
self.assertFalse((self.folder / "old-0123abcd").exists())
|
||||
self.assertTrue((self.folder / "theirs-0123abcd" / "y_thumb.webp").exists())
|
||||
keys = preview_location.read_marker(str(self.folder))["keys"]
|
||||
self.assertNotIn("old-0123abcd", keys)
|
||||
|
||||
|
||||
class TestRecreate(MoveCase):
|
||||
async def test_recreate_and_its_refusals(self):
|
||||
await self.move({"mode": "folder", "path": str(self.folder)})
|
||||
url = "/prompt_manager/images/thumbnails/location/recreate"
|
||||
resp = await self.client.post(url)
|
||||
self.assertEqual((await resp.json())["code"], "location-has-marker")
|
||||
for child in sorted(self.folder.rglob("*"), reverse=True):
|
||||
child.rmdir() if child.is_dir() else child.unlink()
|
||||
self.folder.rmdir()
|
||||
self.assertFalse(preview_store.refresh())
|
||||
resp = await self.client.post(url)
|
||||
self.assertEqual(resp.status, 200, await resp.text())
|
||||
self.assertTrue(preview_store.current().available)
|
||||
self.assertNotEqual(self.config()["thumbnails"]["location"]["id"], None)
|
||||
|
||||
async def test_use_this_folder_adopts_another_marker(self):
|
||||
await self.move({"mode": "folder", "path": str(self.folder)})
|
||||
marker = preview_location.read_marker(str(self.folder))
|
||||
marker["id"] = "someone-else"
|
||||
(self.folder / preview_location.MARKER_NAME).write_text(
|
||||
json.dumps(marker), encoding="utf-8"
|
||||
)
|
||||
self.assertFalse(preview_store.refresh())
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/images/thumbnails/location/adopt"
|
||||
)
|
||||
self.assertEqual(resp.status, 200, await resp.text())
|
||||
self.assertTrue(preview_store.current().available)
|
||||
self.assertEqual(self.config()["thumbnails"]["location"]["id"], "someone-else")
|
||||
|
||||
|
||||
class TestFiles(unittest.TestCase):
|
||||
def test_mtime_rounds_up_to_two_seconds(self):
|
||||
self.assertEqual(files.rounded_mtime_ns(0), 0)
|
||||
self.assertEqual(files.rounded_mtime_ns(1), files.MTIME_STEP_NS)
|
||||
self.assertEqual(
|
||||
files.rounded_mtime_ns(files.MTIME_STEP_NS), files.MTIME_STEP_NS
|
||||
)
|
||||
|
||||
def test_long_windows_destinations_are_per_file_errors(self):
|
||||
self.assertTrue(files.too_long("D:\\" + "x" * 240, long_paths=False))
|
||||
self.assertFalse(files.too_long("D:\\x", long_paths=False))
|
||||
|
||||
def test_a_held_open_file_is_retried_then_left(self):
|
||||
waits = []
|
||||
error = OSError("in use")
|
||||
error.winerror = 32
|
||||
with patch.object(files.os, "unlink", side_effect=error):
|
||||
self.assertFalse(files.remove_file("x", sleep=waits.append))
|
||||
self.assertEqual(waits, [0.5, 1.0, 2.0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,4 +1,8 @@
|
||||
"""Tests for utils/parallel.py: bounded thread-pool map used by long jobs."""
|
||||
"""Tests for utils/parallel.py: bounded thread-pool map used by long jobs.
|
||||
|
||||
The jobs share one long-lived pool of Worker threads (#523): its threads (and
|
||||
their malloc arenas) are reused from job to job instead of a new set per call.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
@@ -8,10 +12,22 @@ import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from utils.parallel import map_parallel # noqa: E402
|
||||
from utils import parallel # noqa: E402
|
||||
from utils.parallel import JOB_THREAD_PREFIX, map_parallel # noqa: E402
|
||||
|
||||
WAIT = 30
|
||||
|
||||
|
||||
def _job_threads():
|
||||
return {
|
||||
t for t in threading.enumerate() if t.name.startswith(JOB_THREAD_PREFIX + "_")
|
||||
}
|
||||
|
||||
|
||||
class TestMapParallel(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.assertTrue(parallel.shutdown_job_pool(WAIT))
|
||||
self.addCleanup(parallel.shutdown_job_pool, WAIT)
|
||||
|
||||
def test_results_keep_input_order(self):
|
||||
def slow_square(n):
|
||||
@@ -22,12 +38,102 @@ class TestMapParallel(unittest.TestCase):
|
||||
map_parallel(slow_square, [1, 2, 3, 4], workers=4), [1, 4, 9, 16]
|
||||
)
|
||||
|
||||
def test_one_worker_runs_on_the_calling_thread(self):
|
||||
def test_one_worker_runs_on_the_one_job_thread(self):
|
||||
names = []
|
||||
map_parallel(
|
||||
lambda _: names.append(threading.current_thread().name), [1, 2, 3], 1
|
||||
)
|
||||
self.assertEqual(set(names), {threading.current_thread().name})
|
||||
self.assertEqual(len(set(names)), 1)
|
||||
self.assertTrue(names[0].startswith(JOB_THREAD_PREFIX + "_"), names)
|
||||
|
||||
def test_jobs_reuse_the_same_long_lived_threads(self):
|
||||
"""Arenas are reused instead of a new set of threads per job."""
|
||||
first, second = set(), set()
|
||||
map_parallel(lambda _: first.add(threading.current_thread()), range(8), 2)
|
||||
map_parallel(lambda _: second.add(threading.current_thread()), range(8), 2)
|
||||
self.assertTrue(second <= first, (first, second))
|
||||
self.assertLessEqual(len(_job_threads()), 2)
|
||||
self.assertTrue(all(t.is_alive() for t in first | second))
|
||||
|
||||
def test_a_new_worker_count_replaces_the_pool(self):
|
||||
map_parallel(lambda n: n, range(4), 2)
|
||||
old = _job_threads()
|
||||
map_parallel(lambda n: n, range(4), 3)
|
||||
for thread in old:
|
||||
thread.join(WAIT)
|
||||
self.assertFalse(thread.is_alive(), thread.name)
|
||||
self.assertLessEqual(len(_job_threads()), 3)
|
||||
|
||||
def test_shutdown_also_joins_a_pool_a_resize_retired(self):
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
|
||||
def hold(_):
|
||||
entered.set()
|
||||
release.wait(WAIT)
|
||||
|
||||
caller = threading.Thread(target=map_parallel, args=(hold, [1], 1))
|
||||
caller.start()
|
||||
self.addCleanup(caller.join, WAIT)
|
||||
self.addCleanup(release.set)
|
||||
self.assertTrue(entered.wait(WAIT))
|
||||
map_parallel(lambda n: n, [1, 2], 2) # a new size retires the busy pool
|
||||
self.assertFalse(parallel.shutdown_job_pool(0.05))
|
||||
release.set()
|
||||
caller.join(WAIT)
|
||||
self.assertTrue(parallel.shutdown_job_pool(WAIT))
|
||||
self.assertFalse(parallel.job_pool_started())
|
||||
|
||||
def test_an_item_run_from_a_job_thread_runs_inline(self):
|
||||
"""A nested map never waits for the threads it is running on."""
|
||||
|
||||
def outer(n):
|
||||
return map_parallel(lambda m: (m, threading.current_thread()), [n], 1)
|
||||
|
||||
results = map_parallel(outer, [1, 2], 1)
|
||||
self.assertEqual([r[0][0] for r in results], [1, 2])
|
||||
|
||||
def test_shutdown_stops_the_threads_and_a_later_job_starts_fresh(self):
|
||||
map_parallel(lambda n: n, range(4), 2)
|
||||
threads = _job_threads()
|
||||
self.assertTrue(threads)
|
||||
self.assertTrue(parallel.job_pool_started())
|
||||
self.assertTrue(parallel.shutdown_job_pool(WAIT))
|
||||
self.assertFalse(parallel.job_pool_started())
|
||||
self.assertFalse(any(t.is_alive() for t in threads))
|
||||
self.assertEqual(map_parallel(lambda n: n + 1, [1], 2), [2])
|
||||
|
||||
def test_shutdown_reports_a_thread_still_running(self):
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
|
||||
def hold(_):
|
||||
entered.set()
|
||||
release.wait(WAIT)
|
||||
|
||||
caller = threading.Thread(target=map_parallel, args=(hold, [1], 1))
|
||||
caller.start()
|
||||
self.assertTrue(entered.wait(WAIT))
|
||||
self.assertFalse(parallel.shutdown_job_pool(0.05))
|
||||
release.set()
|
||||
caller.join(WAIT)
|
||||
self.assertFalse(caller.is_alive())
|
||||
|
||||
def test_a_failing_item_waits_for_the_running_ones(self):
|
||||
"""The caller sees the error only after its other items stopped."""
|
||||
running = threading.Event()
|
||||
finished = []
|
||||
|
||||
def item(n):
|
||||
if n == 0:
|
||||
self.assertTrue(running.wait(WAIT))
|
||||
raise ValueError("zero")
|
||||
running.set()
|
||||
time.sleep(0.05)
|
||||
finished.append(n)
|
||||
return n
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
map_parallel(item, [0, 1], 2)
|
||||
self.assertEqual(finished, [1])
|
||||
|
||||
def test_several_workers_use_several_threads(self):
|
||||
names = set()
|
||||
|
||||
@@ -77,6 +77,7 @@ ROUTE_AUDIT = {
|
||||
"POST /prompt_manager/jobs/autotag",
|
||||
"POST /prompt_manager/jobs/thumbnails",
|
||||
"GET /prompt_manager/images/thumbnails/plan",
|
||||
"POST /prompt_manager/images/thumbnails/plan",
|
||||
"GET /prompt_manager/images/thumbnails/status",
|
||||
"POST /prompt_manager/images/generate-thumbnails",
|
||||
"POST /prompt_manager/images/generate-thumbnails/progress",
|
||||
@@ -171,6 +172,8 @@ ROUTE_AUDIT = {
|
||||
"POST /prompt_manager/cleanup",
|
||||
"POST /prompt_manager/diagnostics/test-link",
|
||||
"POST /prompt_manager/images/clear-thumbnails",
|
||||
"POST /prompt_manager/images/thumbnails/location/adopt",
|
||||
"POST /prompt_manager/images/thumbnails/location/recreate",
|
||||
"POST /prompt_manager/jobs/model-download",
|
||||
"POST /prompt_manager/jobs/{job_id}/cancel",
|
||||
"POST /prompt_manager/library",
|
||||
|
||||
Vendored
+9
-22
@@ -129,11 +129,11 @@ test('Firefox/WebKit page workflows run only outside pull requests; coverage run
|
||||
}
|
||||
});
|
||||
|
||||
test('coverage runs as three shards that each upload their raw coverage', () => {
|
||||
test('coverage runs as four shards that each upload their raw coverage', () => {
|
||||
const coverage = allJobs.get('frontend-coverage');
|
||||
assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3']);
|
||||
assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/3\)$/m);
|
||||
assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/3/);
|
||||
assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3', '4']);
|
||||
assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/4\)$/m);
|
||||
assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/4/);
|
||||
const upload = steps(coverage).find(step => /name: coverage-raw-\$\{\{ matrix\.shard \}\}/.test(step));
|
||||
assert.ok(upload, 'each shard uploads coverage-raw-N');
|
||||
assert.match(upload, /^ {10}path: coverage\/js\/raw\/?$/m);
|
||||
@@ -331,7 +331,7 @@ const testJobs = () => jobs(readWorkflow('test.yml'));
|
||||
|
||||
test('the Python matrix is Linux 3.12 on a pull request and Linux 3.12-3.14 on every other event', () => {
|
||||
const body = testJobs().get('test');
|
||||
assert.deepEqual(matrixList(body, 'os'), ['ubuntu-latest'], 'Windows and macOS never gate a merge');
|
||||
assert.deepEqual(matrixList(body, 'os'), ['ubuntu-latest'], 'Windows never gates a merge');
|
||||
const match = /^ {8}python-version: \$\{\{ github\.event_name == 'pull_request' && fromJSON\('(\[[^\]]*\])'\) \|\| fromJSON\('(\[[^\]]*\])'\) \}\}$/m.exec(body);
|
||||
assert.ok(match, 'python-version is chosen by event');
|
||||
assert.deepEqual({ pr: JSON.parse(match[1]), other: JSON.parse(match[2]) }, { pr: ['3.12'], other: ['3.12', '3.13', '3.14'] });
|
||||
@@ -339,34 +339,21 @@ test('the Python matrix is Linux 3.12 on a pull request and Linux 3.12-3.14 on e
|
||||
assert.match(body, /^ {4}name: py\$\{\{ matrix\.python-version \}\} on \$\{\{ matrix\.os \}\}$/m);
|
||||
});
|
||||
|
||||
test('Windows and macOS run 3.12-3.14 on main, nightly and on demand, never on a PR or in the queue', () => {
|
||||
test('Windows runs 3.12-3.14 on main, nightly and on demand, never on a PR or in the queue', () => {
|
||||
const text = readWorkflow('portability.yml');
|
||||
const triggers = text.slice(text.indexOf('\non:'), text.indexOf('\nconcurrency:'));
|
||||
for (const event of ['push', 'schedule', 'workflow_dispatch']) assert.match(triggers, new RegExp(`^ {2}${event}:`, 'm'));
|
||||
for (const event of ['pull_request', 'merge_group']) assert.doesNotMatch(triggers, new RegExp(`^ {2}${event}`, 'm'));
|
||||
assert.match(triggers, /^ {4}branches: \[main\]$/m, 'push runs only for main');
|
||||
const body = jobs(text).get('test');
|
||||
assert.deepEqual(matrixList(body, 'os'), ['windows-latest', 'macos-latest']);
|
||||
assert.deepEqual(matrixList(body, 'os'), ['windows-latest'], 'macOS is not tested in CI (#571)');
|
||||
assert.doesNotMatch(text, /macos-/, 'no macOS runner is used');
|
||||
assert.deepEqual(matrixList(body, 'python-version'), ['"3.12"', '"3.13"', '"3.14"']);
|
||||
assert.match(body, /^ {6}fail-fast: false$/m);
|
||||
assert.match(body, /^ {8}run: python -m tests\.run_suite -v$/m);
|
||||
assert.doesNotMatch(text, /cancel-in-progress: true|cancel-in-progress: \$/, 'no portability run is cancelled');
|
||||
});
|
||||
|
||||
// #499: the native trash proof runs on both macOS CPUs, apart from the suite so a red suite never skips it.
|
||||
test('portability runs the native trash proof on arm64 and Intel macOS', () => {
|
||||
const body = jobs(readWorkflow('portability.yml')).get('native-trash');
|
||||
assert.ok(body, 'portability.yml has a native-trash job');
|
||||
assert.deepEqual(matrixList(body, 'os'), ['macos-latest', 'macos-15-intel']);
|
||||
assert.match(body, /^ {6}fail-fast: false$/m);
|
||||
assert.doesNotMatch(body, /^ {4}needs:/m, 'it does not wait for the suite job');
|
||||
assert.match(body, /^ {10}PM_NATIVE_TRASH: macos$/m);
|
||||
assert.match(body, /^ {8}run: python -m unittest tests\.test_trash_macos_native tests\.test_trash_macos -v$/m);
|
||||
const jobMinutes = Number(/^ {4}timeout-minutes: (\d+)$/m.exec(body)[1]);
|
||||
const stepMinutes = Number(/^ {8}timeout-minutes: (\d+)$/m.exec(body)?.[1]);
|
||||
assert.ok(stepMinutes > 0 && jobMinutes - stepMinutes >= 2, 'the proof step stops before the job limit');
|
||||
});
|
||||
|
||||
// #493: the portability limit follows the rule in its comment: ceil(1.5 x the slowest of three
|
||||
// green main runs) + 3 minutes, never under 5, with those runs and times written above jobs:.
|
||||
// The times are whole-job wall times. That the runs were green and on main is the author's word.
|
||||
@@ -427,5 +414,5 @@ test('every required check reports on a pull request and the gates carry the ful
|
||||
].sort());
|
||||
assert.equal(contexts.filter(context => /^py/.test(context)).length, 1,
|
||||
'a dropped matrix leg never reports on a pull request, so it cannot be required');
|
||||
assert.ok(!contexts.some(context => /windows|macos/.test(context)), 'Windows and macOS never gate a merge');
|
||||
assert.ok(!contexts.some(context => /windows|macos/.test(context)), 'Windows never gates a merge');
|
||||
});
|
||||
|
||||
@@ -0,0 +1,395 @@
|
||||
"""Measure the server's resident memory under a busy Images/Library session (#523).
|
||||
|
||||
Manual only, Linux only (it reads ``/proc`` and glibc's ``malloc_info``); never
|
||||
run in CI. It starts the production API inside the disposable browser test
|
||||
server (``tests/browser/server.py``: its own temporary library, database and
|
||||
output folder; never a live ComfyUI) and drives it for ``--minutes``:
|
||||
|
||||
- ``--clients`` threads loop over: a workspace grid page of 100 images, 40 of
|
||||
its thumbnails, 6 inspector reads and 6 full-file reads, the Library list
|
||||
with 3 entries' history and images, and the recent prompt and image lists;
|
||||
- every minute, ``--batch`` new PNGs (832 to 1536 px, ComfyUI-shaped
|
||||
metadata) are copied into the output folder, then a scan job and a
|
||||
thumbnail job run to completion.
|
||||
|
||||
Every 5 s the server samples its RSS, Python heap (tracemalloc), threads and
|
||||
glibc arena count. The test server pins glibc's mmap threshold (#516); this
|
||||
driver turns that off so the production allocator behaviour is measured,
|
||||
unless ``--mallopt`` asks for the pinned comparison run.
|
||||
|
||||
python tools/perf/memory_drive.py --label after --minutes 12
|
||||
python tools/perf/memory_drive.py --repo /path/to/other/checkout --label before
|
||||
|
||||
The checkout under ``--repo`` (default: this one) is the code measured; the
|
||||
driver itself only needs Pillow. The last line printed is a Markdown table row.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import subprocess # nosec B404 - starts this script's own server child
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import tracemalloc
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
REPO = os.path.dirname(os.path.dirname(HERE))
|
||||
PORT_ANNOUNCEMENT = "PM_TEST_SERVER_PORT="
|
||||
OUTPUT_ANNOUNCEMENT = "output="
|
||||
SAMPLE_SECONDS = 5
|
||||
MEDIA_FILES = 120
|
||||
MEDIA_SIDES = (832, 1024, 1216, 1536)
|
||||
REQUEST_TIMEOUT = 120
|
||||
JOB_POLL_LIMIT = 600
|
||||
FINAL_STATES = ("finished", "cancelled", "failed")
|
||||
M_MMAP_THRESHOLD = -3
|
||||
|
||||
|
||||
# ── the server child ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _arena_count(libc):
|
||||
"""glibc's arena count (``<heap nr=`` entries of ``malloc_info``)."""
|
||||
buffer, size = ctypes.c_char_p(), ctypes.c_size_t()
|
||||
stream = libc.open_memstream(ctypes.byref(buffer), ctypes.byref(size))
|
||||
libc.malloc_info(0, stream)
|
||||
libc.fclose(stream)
|
||||
text = ctypes.string_at(buffer, size.value).decode()
|
||||
libc.free(ctypes.cast(buffer, ctypes.c_void_p))
|
||||
return text.count("<heap nr=")
|
||||
|
||||
|
||||
def _status():
|
||||
with open("/proc/self/status", encoding="ascii") as status:
|
||||
return {
|
||||
line.split(":")[0]: int(line.split()[1])
|
||||
for line in status
|
||||
if line.startswith(("VmRSS", "RssAnon", "Threads"))
|
||||
}
|
||||
|
||||
|
||||
def _sample_forever(libc, samples_path):
|
||||
start = time.monotonic()
|
||||
announced = False
|
||||
with open(samples_path, "a", buffering=1, encoding="utf-8") as out:
|
||||
while True:
|
||||
folder_paths = sys.modules.get("folder_paths")
|
||||
if not announced and hasattr(folder_paths, "get_output_directory"):
|
||||
out.write(OUTPUT_ANNOUNCEMENT + folder_paths.get_output_directory())
|
||||
out.write("\n")
|
||||
announced = True
|
||||
status = _status()
|
||||
out.write(
|
||||
json.dumps(
|
||||
{
|
||||
"t": round(time.monotonic() - start),
|
||||
"rss": status["VmRSS"] >> 10,
|
||||
"anon": status["RssAnon"] >> 10,
|
||||
"heap": tracemalloc.get_traced_memory()[0] >> 20,
|
||||
"threads": status["Threads"],
|
||||
"arenas": _arena_count(libc),
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
time.sleep(SAMPLE_SECONDS if announced else 0.5)
|
||||
|
||||
|
||||
def serve(repo, samples_path, mallopt):
|
||||
"""Run the browser test server of *repo* in this process, sampled."""
|
||||
sys.path.insert(0, repo)
|
||||
os.chdir(repo)
|
||||
tracemalloc.start(1)
|
||||
libc = ctypes.CDLL(None)
|
||||
libc.open_memstream.restype = ctypes.c_void_p
|
||||
libc.open_memstream.argtypes = [
|
||||
ctypes.POINTER(ctypes.c_char_p),
|
||||
ctypes.POINTER(ctypes.c_size_t),
|
||||
]
|
||||
libc.fclose.argtypes = [ctypes.c_void_p]
|
||||
libc.malloc_info.argtypes = [ctypes.c_int, ctypes.c_void_p]
|
||||
libc.free.argtypes = [ctypes.c_void_p]
|
||||
if mallopt:
|
||||
libc.mallopt(M_MMAP_THRESHOLD, 128 * 1024)
|
||||
threading.Thread(
|
||||
target=_sample_forever, args=(libc, samples_path), daemon=True
|
||||
).start()
|
||||
from tests.browser import server
|
||||
|
||||
if not mallopt:
|
||||
server.pin_mmap_threshold = lambda: False # production allocator
|
||||
server.main(["--port", "0", "--exit-on-stdin-eof"])
|
||||
|
||||
|
||||
# ── the driver ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def make_media(folder, count=MEDIA_FILES, seed=523):
|
||||
"""Synthetic ComfyUI-shaped PNGs (smooth noise, prompt and workflow)."""
|
||||
from PIL import Image, PngImagePlugin
|
||||
|
||||
rng = random.Random(seed)
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
for index in range(count):
|
||||
side = MEDIA_SIDES[index % len(MEDIA_SIDES)]
|
||||
small = Image.frombytes(
|
||||
"RGB", (64, 64), bytes(rng.randrange(256) for _ in range(64 * 64 * 3))
|
||||
)
|
||||
image = small.resize((side, side), Image.BICUBIC)
|
||||
text = f"synthetic scene {index}, soft light, detailed"
|
||||
prompt = {
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"seed": index,
|
||||
"steps": 20,
|
||||
"cfg": 7.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "normal",
|
||||
"denoise": 1.0,
|
||||
"model": ["4", 0],
|
||||
"positive": ["6", 0],
|
||||
"negative": ["7", 0],
|
||||
"latent_image": ["5", 0],
|
||||
},
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CheckpointLoaderSimple",
|
||||
"inputs": {"ckpt_name": "model.safetensors"},
|
||||
},
|
||||
"5": {
|
||||
"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": side, "height": side, "batch_size": 1},
|
||||
},
|
||||
"6": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": text, "clip": ["4", 1]},
|
||||
},
|
||||
"7": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "blurry", "clip": ["4", 1]},
|
||||
},
|
||||
"9": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"filename_prefix": "img", "images": ["8", 0]},
|
||||
},
|
||||
}
|
||||
workflow = {
|
||||
"nodes": [
|
||||
{"id": int(k), "type": v["class_type"]} for k, v in prompt.items()
|
||||
]
|
||||
}
|
||||
info = PngImagePlugin.PngInfo()
|
||||
info.add_text("prompt", json.dumps(prompt))
|
||||
info.add_text("workflow", json.dumps(workflow))
|
||||
image.save(
|
||||
os.path.join(folder, f"m{index:03d}.png"), pnginfo=info, compress_level=4
|
||||
)
|
||||
|
||||
|
||||
class Driver:
|
||||
def __init__(self, base, output, media, batch):
|
||||
self.base, self.output, self.media, self.batch = base, output, media, batch
|
||||
self.errors, self.batches = [], 0
|
||||
self.names = sorted(os.listdir(media))
|
||||
|
||||
def call(self, method, path, body=None, raw=False):
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
request = urllib.request.Request( # nosec B310 - loopback test server
|
||||
self.base + path,
|
||||
data=data,
|
||||
method=method,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen( # nosec B310
|
||||
request, timeout=REQUEST_TIMEOUT
|
||||
) as response:
|
||||
payload = response.read()
|
||||
except Exception as error: # keep driving; errors are counted
|
||||
self.errors.append(f"{method} {path.split('?')[0]}: {error}")
|
||||
return None
|
||||
return payload if raw else json.loads(payload or b"{}")
|
||||
|
||||
def add_batch(self):
|
||||
self.batches += 1
|
||||
folder = os.path.join(self.output, f"batch{self.batches:03d}")
|
||||
os.makedirs(folder)
|
||||
for index in range(self.batch):
|
||||
name = self.names[(self.batches * self.batch + index) % len(self.names)]
|
||||
shutil.copyfile(
|
||||
os.path.join(self.media, name),
|
||||
os.path.join(folder, f"img_{self.batches:03d}_{index:03d}.png"),
|
||||
)
|
||||
|
||||
def wait_job(self, response):
|
||||
job = ((response or {}).get("job") or {}).get("id")
|
||||
for _ in range(JOB_POLL_LIMIT if job else 0):
|
||||
state = self.call("GET", f"/prompt_manager/jobs/{job}")
|
||||
if not state or (state.get("job") or {}).get("state") in FINAL_STATES:
|
||||
return
|
||||
time.sleep(1)
|
||||
|
||||
def scan_and_thumbnails(self):
|
||||
self.add_batch()
|
||||
self.wait_job(self.call("POST", "/prompt_manager/jobs/scan", {}))
|
||||
self.wait_job(self.call("POST", "/prompt_manager/jobs/thumbnails", {}))
|
||||
|
||||
def browse(self, _index=None):
|
||||
offset = random.randint(0, 3) * 100 # nosec B311 - load shape only
|
||||
page = self.call(
|
||||
"GET",
|
||||
f"/prompt_manager/images/output?mode=workspace&limit=100&offset={offset}",
|
||||
)
|
||||
images = [i for i in (page or {}).get("images") or [] if isinstance(i, dict)]
|
||||
urls = [i.get("thumbnail_url") or i.get("url") for i in images]
|
||||
urls = [u for u in urls if u]
|
||||
for url in random.sample(urls, min(40, len(urls))):
|
||||
self.call("GET", url, raw=True)
|
||||
for image in random.sample(images, min(6, len(images))):
|
||||
self.inspect(image)
|
||||
if image.get("url"):
|
||||
self.call("GET", image["url"], raw=True)
|
||||
library = self.call("GET", "/prompt_manager/library?limit=100") or {}
|
||||
entries = [e for e in library.get("items") or [] if isinstance(e, dict)]
|
||||
for entry in random.sample(entries, min(3, len(entries))):
|
||||
prompt_id = entry.get("id") or entry.get("positive_id")
|
||||
if prompt_id is not None:
|
||||
self.call(
|
||||
"GET", f"/prompt_manager/library/{prompt_id}/history", raw=True
|
||||
)
|
||||
self.call(
|
||||
"GET", f"/prompt_manager/library/{prompt_id}/images", raw=True
|
||||
)
|
||||
self.call("GET", "/prompt_manager/recent?limit=50", raw=True)
|
||||
self.call("GET", "/prompt_manager/images/recent?limit=50", raw=True)
|
||||
|
||||
def inspect(self, image):
|
||||
relative, index = image.get("path"), image.get("root_index")
|
||||
if not relative or not isinstance(index, int):
|
||||
return
|
||||
query = {"root": str(index)}
|
||||
if image.get("root_scope"):
|
||||
query["root_scope"] = image["root_scope"]
|
||||
self.call(
|
||||
"GET",
|
||||
"/prompt_manager/images/inspect/"
|
||||
+ urllib.parse.quote(relative)
|
||||
+ "?"
|
||||
+ urllib.parse.urlencode(query),
|
||||
raw=True,
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_output(samples_path, process):
|
||||
while process.poll() is None:
|
||||
with open(samples_path, encoding="utf-8") as samples:
|
||||
for line in samples:
|
||||
if line.startswith(OUTPUT_ANNOUNCEMENT):
|
||||
return line[len(OUTPUT_ANNOUNCEMENT) :].strip()
|
||||
time.sleep(0.2)
|
||||
raise RuntimeError("the server exited before it started")
|
||||
|
||||
|
||||
def summarize(samples_path, label, batches, errors):
|
||||
rows = []
|
||||
with open(samples_path, encoding="utf-8") as samples:
|
||||
for line in samples:
|
||||
if line.startswith("{"):
|
||||
rows.append(json.loads(line))
|
||||
if not rows:
|
||||
raise RuntimeError("no samples were taken")
|
||||
rss = [row["rss"] for row in rows]
|
||||
late = [row for row in rows if row["t"] >= 120] or rows
|
||||
end = rows[-1]
|
||||
return (
|
||||
f"| {label} | {rss[0]} → {end['rss']} | {min(r['rss'] for r in late)}–"
|
||||
f"{max(r['rss'] for r in late)} | {max(rss)} | {rows[0]['heap']} → "
|
||||
f"{end['heap']} | {max(r['threads'] for r in rows)} | "
|
||||
f"{max(r['arenas'] for r in rows)} | {batches} | {len(errors)} |"
|
||||
)
|
||||
|
||||
|
||||
def drive(args):
|
||||
work = args.out or tempfile.mkdtemp(prefix="pm-memory-")
|
||||
media = os.path.join(work, "media")
|
||||
if not os.path.isdir(media):
|
||||
print(f"Writing {MEDIA_FILES} PNGs to {media}", flush=True)
|
||||
make_media(media)
|
||||
samples_path = os.path.join(work, f"{args.label}.samples")
|
||||
open(samples_path, "w", encoding="utf-8").close()
|
||||
command = [sys.executable, os.path.abspath(__file__), "--serve", samples_path]
|
||||
command += ["--repo", os.path.abspath(args.repo)]
|
||||
command += ["--mallopt"] if args.mallopt else []
|
||||
with open(os.path.join(work, f"{args.label}.stderr"), "w") as stderr:
|
||||
process = subprocess.Popen( # nosec B603
|
||||
command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr
|
||||
)
|
||||
try:
|
||||
port = None
|
||||
for line in process.stdout:
|
||||
text = line.decode(errors="replace")
|
||||
if text.startswith(PORT_ANNOUNCEMENT):
|
||||
port = int(text[len(PORT_ANNOUNCEMENT) :])
|
||||
break
|
||||
if port is None:
|
||||
raise RuntimeError("the server announced no port")
|
||||
output = _wait_for_output(samples_path, process)
|
||||
driver = Driver(f"http://127.0.0.1:{port}", output, media, args.batch)
|
||||
run(driver, args)
|
||||
finally:
|
||||
process.stdin.close()
|
||||
process.wait(60)
|
||||
print(f"Samples: {samples_path}")
|
||||
print(
|
||||
"| run | RSS start → end (MB) | after 2 min | max | heap (MB) | threads "
|
||||
"| arenas | batches | errors |"
|
||||
)
|
||||
print(summarize(samples_path, args.label, driver.batches, driver.errors))
|
||||
|
||||
|
||||
def run(driver, args):
|
||||
driver.call("POST", "/__test__/reset", {})
|
||||
driver.scan_and_thumbnails()
|
||||
deadline = time.monotonic() + args.minutes * 60
|
||||
next_batch = time.monotonic() + 60
|
||||
with ThreadPoolExecutor(args.clients) as pool:
|
||||
while time.monotonic() < deadline:
|
||||
list(pool.map(driver.browse, range(args.clients)))
|
||||
if time.monotonic() > next_batch:
|
||||
driver.scan_and_thumbnails()
|
||||
next_batch = time.monotonic() + 60
|
||||
time.sleep(SAMPLE_SECONDS + 1) # one more sample after the load stops
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--repo", default=REPO, help="checkout to measure")
|
||||
parser.add_argument("--label", default="run")
|
||||
parser.add_argument("--minutes", type=float, default=12)
|
||||
parser.add_argument("--clients", type=int, default=6)
|
||||
parser.add_argument("--batch", type=int, default=60, help="new PNGs a minute")
|
||||
parser.add_argument("--out", help="work folder (media, samples); default temp")
|
||||
parser.add_argument(
|
||||
"--mallopt", action="store_true", help="pin the mmap threshold (comparison)"
|
||||
)
|
||||
parser.add_argument("--serve", metavar="SAMPLES", help=argparse.SUPPRESS)
|
||||
args = parser.parse_args(argv)
|
||||
if not sys.platform.startswith("linux"):
|
||||
parser.error("Linux only: it reads /proc and glibc's malloc_info")
|
||||
if args.serve:
|
||||
serve(os.path.abspath(args.repo), args.serve, args.mallopt)
|
||||
else:
|
||||
drive(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -641,6 +641,12 @@
|
||||
"functions": 95,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/system-thumbnail-location.mjs": {
|
||||
"lines": 80,
|
||||
"branches": 65,
|
||||
"functions": 75,
|
||||
"statements": 80
|
||||
},
|
||||
"workspace/system-thumbnails.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 95,
|
||||
|
||||
+48
-5
@@ -230,6 +230,9 @@ def _undecided_graph(found: _File) -> Decision:
|
||||
if switched is not None:
|
||||
# The legacy answer is a switched-off node's own text (#90)
|
||||
return Decision(switched, canonical=found.canonical)
|
||||
if text is not None and _search_text(found.canonical["prompt"], text):
|
||||
# The query of a Search List batch is a search, never a prompt (#548)
|
||||
return _never(found, _unlinked(found), try_recorded_run=True)
|
||||
if text is not None and _joined_text(found.canonical["prompt"], text):
|
||||
# The legacy parser joins prepend + text + append: the text a
|
||||
# PromptManager node stores cannot be told from it (#343).
|
||||
@@ -311,6 +314,8 @@ _NOT_TEXT_INPUTS = frozenset(
|
||||
}
|
||||
)
|
||||
_MAX_SWITCH_VISITS = 5000
|
||||
# The batch node whose ``text`` input is a library search query (#548).
|
||||
SEARCH_LIST_TYPE = "PromptSearchList"
|
||||
|
||||
|
||||
def _negative_name(key: Any) -> bool:
|
||||
@@ -328,10 +333,28 @@ def _switched_off_upstream(graph: Any) -> Optional[str]:
|
||||
"""
|
||||
if not isinstance(graph, dict):
|
||||
return None
|
||||
roots = _positive_roots(graph)
|
||||
if not roots:
|
||||
upstream = _positive_upstream(graph)
|
||||
if upstream is None:
|
||||
ids = _prompt_manager_ids(graph)
|
||||
return _switch_kind(save_setting(graph, node_id) for node_id in ids)
|
||||
found = [
|
||||
node_id
|
||||
for node_id in upstream
|
||||
if graph[node_id].get("class_type") in PROMPT_MANAGER_TYPES
|
||||
]
|
||||
return _switch_kind(save_setting(graph, node_id) for node_id in found)
|
||||
|
||||
|
||||
def _positive_upstream(graph: dict) -> Optional[set]:
|
||||
"""The ids of the nodes upstream of a sampler's positive, or None when
|
||||
the graph has no recognisable sampler.
|
||||
|
||||
Follows every input except negative and non-text ones (#90's walk follows
|
||||
only prompt-like names).
|
||||
"""
|
||||
roots = _positive_roots(graph)
|
||||
if not roots:
|
||||
return None
|
||||
seen, found = set(), set()
|
||||
stack = [str(link[0]) for link in roots]
|
||||
while stack and len(seen) < _MAX_SWITCH_VISITS:
|
||||
@@ -342,8 +365,7 @@ def _switched_off_upstream(graph: Any) -> Optional[str]:
|
||||
node = graph.get(node_id)
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
if node.get("class_type") in PROMPT_MANAGER_TYPES:
|
||||
found.add(node_id)
|
||||
found.add(node_id)
|
||||
inputs = node.get("inputs")
|
||||
for key, value in (inputs.items() if isinstance(inputs, dict) else ()):
|
||||
if key in _NOT_TEXT_INPUTS or _negative_name(key):
|
||||
@@ -351,7 +373,28 @@ def _switched_off_upstream(graph: Any) -> Optional[str]:
|
||||
if isinstance(value, (list, tuple)) and len(value) == 2:
|
||||
if isinstance(value[0], (str, int)) and isinstance(value[1], int):
|
||||
stack.append(str(value[0]))
|
||||
return _switch_kind(save_setting(graph, node_id) for node_id in found)
|
||||
return found
|
||||
|
||||
|
||||
def _search_text(graph: Any, text: str) -> bool:
|
||||
"""Whether ``text`` is the search query of a PromptSearchList node that
|
||||
feeds a positive (any such node when no sampler is recognised).
|
||||
|
||||
The legacy parser traces a positive wired from a Search List to the
|
||||
node's ``text`` input, which is the database search, never the prompt
|
||||
the batch ran (#548). Each item is counted by the node at run time.
|
||||
"""
|
||||
if not isinstance(graph, dict):
|
||||
return False
|
||||
upstream = _positive_upstream(graph)
|
||||
ids = graph if upstream is None else upstream
|
||||
wanted = text.strip()
|
||||
return any(
|
||||
isinstance(graph[node_id], dict)
|
||||
and graph[node_id].get("class_type") == SEARCH_LIST_TYPE
|
||||
and resolve_text(graph, node_id) == wanted
|
||||
for node_id in ids
|
||||
)
|
||||
|
||||
|
||||
def _off_text(graph: Any, text: str) -> Optional[str]:
|
||||
|
||||
+87
-11
@@ -96,6 +96,65 @@ WORKER_JOIN_SECONDS = 5.0
|
||||
OBSERVER_JOIN_SECONDS = 5.0
|
||||
|
||||
|
||||
def _start_daemon_observer(observer) -> None:
|
||||
"""``observer.start()`` on a short-lived daemon thread (#561).
|
||||
|
||||
A thread is a daemon when the thread that creates it is one. watchdog
|
||||
7 on Windows starts a plain ``threading.Thread`` per watch inside
|
||||
``start()``; created from ComfyUI's main thread it is not a daemon, and
|
||||
the interpreter waits for every such thread before it runs any
|
||||
``atexit`` callback, so the teardown that would stop it never runs.
|
||||
Started from here, every thread the observer creates is a daemon.
|
||||
Waits for the start without a limit (start-up, not exit); re-raises
|
||||
what ``start()`` raised.
|
||||
"""
|
||||
raised: List[BaseException] = []
|
||||
|
||||
def start() -> None:
|
||||
try:
|
||||
observer.start()
|
||||
except BaseException as e: # re-raised on the caller's thread
|
||||
raised.append(e)
|
||||
|
||||
starter = threading.Thread(
|
||||
target=start, name="PromptManagerObserverStart", daemon=True
|
||||
)
|
||||
starter.start()
|
||||
starter.join()
|
||||
if raised:
|
||||
raise raised[0]
|
||||
|
||||
|
||||
class _ObserverStop:
|
||||
"""One ``observer.stop()`` running on a daemon thread (#561).
|
||||
|
||||
watchdog's ``stop()`` joins its emitters (and, in 7.x on Windows, their
|
||||
reader threads) without a limit. Run here, a stop that never returns
|
||||
holds neither the caller past its deadline nor the interpreter's exit.
|
||||
"""
|
||||
|
||||
def __init__(self, observer) -> None:
|
||||
self.error: Optional[Exception] = None
|
||||
self._thread = threading.Thread(
|
||||
target=self._run,
|
||||
args=(observer,),
|
||||
name="PromptManagerObserverStop",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def _run(self, observer) -> None:
|
||||
try:
|
||||
observer.stop()
|
||||
except Exception as e: # reported by the stop that waits for it
|
||||
self.error = e
|
||||
|
||||
def wait(self, timeout: float) -> bool:
|
||||
"""True once ``stop()`` has returned or raised."""
|
||||
self._thread.join(timeout)
|
||||
return not self._thread.is_alive()
|
||||
|
||||
|
||||
class ImageGenerationHandler(FileSystemEventHandler):
|
||||
"""Filesystem event handler for detecting new image generation.
|
||||
|
||||
@@ -1170,6 +1229,9 @@ class ImageMonitor:
|
||||
# (observer, handler) pairs a stop left running: the next stop waits
|
||||
# for them again instead of reporting True.
|
||||
self._unfinished: List[Tuple[Any, Any]] = []
|
||||
# Observer stops still running on their thread, by observer: a later
|
||||
# stop waits for the same call instead of starting another (#561).
|
||||
self._observer_stops: Dict[Any, _ObserverStop] = {}
|
||||
self.logger = get_logger("prompt_manager.image_monitor")
|
||||
self.review_recorder: Optional[Callable[[str], str]] = None
|
||||
self.thumbnail_queue: Optional[Callable[[str], str]] = None
|
||||
@@ -1273,7 +1335,7 @@ class ImageMonitor:
|
||||
self.logger.warning("No directory could be watched")
|
||||
return
|
||||
try:
|
||||
observer.start()
|
||||
_start_daemon_observer(observer)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Image monitoring could not start: {e}")
|
||||
return
|
||||
@@ -1303,12 +1365,12 @@ class ImageMonitor:
|
||||
|
||||
Closes the handler first (its worker is joined and its dispatch
|
||||
returns at once), then stops the watchdog observer and joins it. The
|
||||
worker and the observer share the one deadline: the observer's join
|
||||
gets what the worker left, possibly 0 s.
|
||||
watchdog's ``observer.stop()`` joins its emitters without a limit;
|
||||
it counts against the deadline but is not bounded here. A part an
|
||||
earlier call left running is waited for again. Safe to call when
|
||||
monitoring never started and more than once.
|
||||
worker and the observer share the one deadline: the observer's stop
|
||||
and join get what the worker left, possibly 0 s. watchdog's
|
||||
``observer.stop()`` joins its emitters without a limit, so it runs on
|
||||
a daemon thread and is waited for only until the deadline (#561). A
|
||||
part an earlier call left running is waited for again. Safe to call
|
||||
when monitoring never started and more than once.
|
||||
|
||||
Returns:
|
||||
True when every worker and observer it stopped has ended.
|
||||
@@ -1349,13 +1411,27 @@ class ImageMonitor:
|
||||
return False
|
||||
|
||||
def _stop_observer(self, observer, timeout: float) -> bool:
|
||||
"""``observer.stop()`` (safe to repeat), then a join of ``timeout``
|
||||
seconds; a join that times out is logged with the time it got."""
|
||||
"""``observer.stop()`` on its own daemon thread, then a join, both
|
||||
within ``timeout`` seconds; a stop or join still running then is
|
||||
logged with the time it got. A stop still running from an earlier
|
||||
call is waited for, never started again."""
|
||||
if observer is None:
|
||||
return True
|
||||
deadline = time.monotonic() + timeout
|
||||
stop = self._observer_stops.get(observer)
|
||||
if stop is None:
|
||||
stop = self._observer_stops[observer] = _ObserverStop(observer)
|
||||
if not stop.wait(timeout):
|
||||
self.logger.warning(
|
||||
f"watchdog observer still running after {timeout:.1f} s"
|
||||
)
|
||||
return False
|
||||
del self._observer_stops[observer]
|
||||
if stop.error is not None:
|
||||
self.logger.warning(f"Error while stopping image monitoring: {stop.error}")
|
||||
return False
|
||||
try:
|
||||
observer.stop()
|
||||
observer.join(timeout)
|
||||
observer.join(max(0.0, deadline - time.monotonic()))
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Error while stopping image monitoring: {e}")
|
||||
return False
|
||||
|
||||
@@ -37,6 +37,7 @@ from typing import Optional, Dict, Any
|
||||
from PIL import Image
|
||||
|
||||
from .logging_config import get_logger
|
||||
from .png_text import png_text
|
||||
|
||||
|
||||
class ComfyUIMetadataExtractor:
|
||||
@@ -88,12 +89,14 @@ class ComfyUIMetadataExtractor:
|
||||
# Add basic file information
|
||||
metadata["file_info"] = self.get_file_info(image_path, image)
|
||||
|
||||
# Extract ComfyUI-specific metadata from PNG text chunks
|
||||
if hasattr(image, "text") and image.text:
|
||||
# Extract ComfyUI-specific metadata from PNG text chunks,
|
||||
# read without decoding the pixels (#523)
|
||||
text = png_text(image)
|
||||
if text:
|
||||
# Look for ComfyUI workflow data
|
||||
if "workflow" in image.text:
|
||||
if "workflow" in text:
|
||||
try:
|
||||
workflow_data = json.loads(image.text["workflow"])
|
||||
workflow_data = json.loads(text["workflow"])
|
||||
metadata["workflow"] = workflow_data
|
||||
|
||||
# Extract text encoder nodes from workflow
|
||||
@@ -107,9 +110,9 @@ class ComfyUIMetadataExtractor:
|
||||
self.logger.warning(f"Failed to parse workflow JSON: {e}")
|
||||
|
||||
# Look for prompt data
|
||||
if "prompt" in image.text:
|
||||
if "prompt" in text:
|
||||
try:
|
||||
prompt_data = json.loads(image.text["prompt"])
|
||||
prompt_data = json.loads(text["prompt"])
|
||||
metadata["prompt"] = prompt_data
|
||||
except json.JSONDecodeError as e:
|
||||
self.logger.warning(f"Failed to parse prompt JSON: {e}")
|
||||
@@ -129,13 +132,13 @@ class ComfyUIMetadataExtractor:
|
||||
]
|
||||
|
||||
for field in metadata_fields:
|
||||
if field in image.text:
|
||||
if field in text:
|
||||
try:
|
||||
# Try to parse as JSON first
|
||||
metadata[field] = json.loads(image.text[field])
|
||||
metadata[field] = json.loads(text[field])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
# Store as string if not valid JSON
|
||||
metadata[field] = image.text[field]
|
||||
metadata[field] = text[field]
|
||||
|
||||
return (
|
||||
metadata
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Give glibc's free heap pages back to the OS after a long image job (#523).
|
||||
|
||||
Image work (decode, resize, encode) leaves image-sized free blocks in the
|
||||
malloc arenas of the threads that did it; glibc keeps those pages resident.
|
||||
``release_free_memory`` calls ``malloc_trim(0)`` once, which returns the free
|
||||
pages of every arena and changes no allocator setting (no ``mallopt``, no
|
||||
``MALLOC_ARENA_MAX``: ComfyUI owns this process's allocator).
|
||||
|
||||
glibc only: on Windows, macOS and musl (Alpine) it does nothing. It never
|
||||
raises; a missing or failing ``malloc_trim`` is a quiet ``False``.
|
||||
"""
|
||||
|
||||
import ctypes
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
|
||||
_lock = threading.Lock()
|
||||
_trim = None
|
||||
_looked_up = False
|
||||
|
||||
|
||||
def _glibc() -> bool:
|
||||
"""Whether this process runs on glibc (musl has no CS_GNU_LIBC_VERSION)."""
|
||||
if not sys.platform.startswith("linux"):
|
||||
return False
|
||||
try:
|
||||
version = os.confstr("CS_GNU_LIBC_VERSION")
|
||||
except (AttributeError, ValueError, OSError):
|
||||
return False
|
||||
return bool(version) and version.startswith("glibc")
|
||||
|
||||
|
||||
def _malloc_trim():
|
||||
"""glibc's ``malloc_trim``, looked up once; None where there is none."""
|
||||
global _trim, _looked_up
|
||||
with _lock:
|
||||
if not _looked_up:
|
||||
_looked_up = True
|
||||
_trim = _lookup()
|
||||
return _trim
|
||||
|
||||
|
||||
def _lookup():
|
||||
if not _glibc():
|
||||
return None
|
||||
try:
|
||||
trim = ctypes.CDLL(None).malloc_trim
|
||||
except (OSError, AttributeError):
|
||||
return None
|
||||
trim.argtypes = [ctypes.c_size_t]
|
||||
trim.restype = ctypes.c_int
|
||||
return trim
|
||||
|
||||
|
||||
def release_free_memory() -> bool:
|
||||
"""Return free heap pages to the OS once (glibc); True when it ran."""
|
||||
trim = _malloc_trim()
|
||||
if trim is None:
|
||||
return False
|
||||
try:
|
||||
trim(0)
|
||||
except Exception: # a foreign libc symbol: never fail the caller's job
|
||||
return False
|
||||
return True
|
||||
+114
-22
@@ -15,18 +15,36 @@ create new thread at interpreter shutdown") only once the interpreter is
|
||||
marked finalizing, after the ``atexit`` callbacks. ``refused_at_shutdown``
|
||||
recognises both; ``note_refusal`` logs one DEBUG line per place, with no
|
||||
traceback.
|
||||
|
||||
One long-lived pool (#523): every ``map_parallel`` call shares one pool of
|
||||
``workers`` threads (``JOB_THREAD_PREFIX``), kept between jobs, instead of a
|
||||
new pool per call. Image work then reuses the same few threads, and their
|
||||
malloc arenas, rather than leaving each call's image-sized buffers in a new
|
||||
set of arenas. A different ``workers`` count (the Worker threads setting
|
||||
changed) replaces the pool; the old one finishes what it holds and exits.
|
||||
``shutdown_job_pool`` stops it (the package teardown); the next call starts
|
||||
a fresh one.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, wait
|
||||
|
||||
from .exit_signal import checked
|
||||
|
||||
SHUTDOWN_REFUSAL = "interpreter shutdown"
|
||||
# ``submit`` on a pool already shut down (this module's teardown or a resize).
|
||||
SHUT_POOL_REFUSAL = "after shutdown"
|
||||
JOB_THREAD_PREFIX = "pm-worker"
|
||||
# The package teardown's bound for the job threads to finish their item.
|
||||
JOB_POOL_JOIN_SECONDS = 5
|
||||
_logger = logging.getLogger("prompt_manager.parallel")
|
||||
_noted = set()
|
||||
_noted_lock = threading.Lock()
|
||||
_pool_lock = threading.Lock()
|
||||
_pool = None # (workers, ThreadPoolExecutor)
|
||||
_retired = [] # pools replaced by a new size, finishing what they hold
|
||||
|
||||
|
||||
def refused_at_shutdown(error):
|
||||
@@ -45,39 +63,113 @@ def note_refusal(logger, what):
|
||||
logger.debug(f"{what}: refused, the interpreter is exiting")
|
||||
|
||||
|
||||
def map_parallel(func, items, workers):
|
||||
"""Apply ``func`` to every item on up to ``workers`` threads, keeping order.
|
||||
def job_pool_started():
|
||||
"""Whether a job pool exists (the teardown stops it)."""
|
||||
return _pool is not None or bool(_retired)
|
||||
|
||||
``workers`` of 1 (or fewer) runs on the calling thread with no pool. An
|
||||
exception in ``func`` propagates to the caller, so callers that must
|
||||
survive bad input wrap ``func`` themselves. A pool refused at interpreter
|
||||
exit runs the items it did not start on the calling thread (#375). An item
|
||||
|
||||
def _job_pool(workers):
|
||||
"""The shared pool of ``workers`` threads, replacing one of another size."""
|
||||
global _pool
|
||||
with _pool_lock:
|
||||
if _pool is not None and _pool[0] == workers:
|
||||
return _pool[1]
|
||||
retired = _pool
|
||||
_pool = (
|
||||
workers,
|
||||
ThreadPoolExecutor(workers, thread_name_prefix=JOB_THREAD_PREFIX),
|
||||
)
|
||||
current = _pool[1]
|
||||
if retired is not None:
|
||||
retired[1].shutdown(wait=False) # it still runs what it holds
|
||||
with _pool_lock:
|
||||
_retired[:] = [p for p in _retired if _alive(p)] + [retired[1]]
|
||||
return current
|
||||
|
||||
|
||||
def shutdown_job_pool(timeout=None):
|
||||
"""Stop the job pool (and any pool a resize retired): queued items are
|
||||
cancelled (their callers run them on their own thread), running ones are
|
||||
joined up to ``timeout`` seconds. True when no job thread is left."""
|
||||
global _pool
|
||||
with _pool_lock:
|
||||
current, _pool = _pool, None
|
||||
executors = _retired[:] + ([] if current is None else [current[1]])
|
||||
_retired.clear()
|
||||
threads = []
|
||||
for executor in executors:
|
||||
# After shutdown: it takes the lock ``submit`` holds while it starts
|
||||
# and records a thread, so every thread is listed and none starts.
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
threads.extend(getattr(executor, "_threads", ()))
|
||||
deadline = None if timeout is None else time.monotonic() + timeout
|
||||
for thread in threads:
|
||||
thread.join(None if deadline is None else max(0.0, deadline - time.monotonic()))
|
||||
return not any(thread.is_alive() for thread in threads)
|
||||
|
||||
|
||||
def _alive(executor):
|
||||
return any(t.is_alive() for t in getattr(executor, "_threads", ()))
|
||||
|
||||
|
||||
def _forget(executor):
|
||||
"""Drop *executor* as the shared pool (a later call starts a new one)."""
|
||||
global _pool
|
||||
with _pool_lock:
|
||||
if _pool is not None and _pool[1] is executor:
|
||||
_pool = None
|
||||
|
||||
|
||||
def _on_job_thread():
|
||||
return threading.current_thread().name.startswith(JOB_THREAD_PREFIX + "_")
|
||||
|
||||
|
||||
def map_parallel(func, items, workers):
|
||||
"""Apply ``func`` to every item on the shared pool of ``workers``
|
||||
threads, keeping order.
|
||||
|
||||
``workers`` below 1 counts as 1. A call made from a job thread runs on
|
||||
that thread, so nested work never waits for its own pool. An exception
|
||||
in ``func`` propagates to the caller once the call's other items have
|
||||
stopped (those not started are dropped), so callers that must survive
|
||||
bad input wrap ``func`` themselves. A pool refused at interpreter exit
|
||||
runs the items it did not start on the calling thread (#375). An item
|
||||
that has not started when the interpreter starts exiting raises
|
||||
ExitRequested instead (#376).
|
||||
"""
|
||||
func = checked(func)
|
||||
items = list(items)
|
||||
workers = max(1, int(workers or 1))
|
||||
if workers == 1 or len(items) <= 1:
|
||||
if not items or _on_job_thread():
|
||||
return [func(item) for item in items]
|
||||
max_workers = min(workers, len(items))
|
||||
with ThreadPoolExecutor(max_workers, thread_name_prefix="pm-worker") as pool:
|
||||
futures = []
|
||||
try:
|
||||
for item in items:
|
||||
futures.append(pool.submit(func, item))
|
||||
except RuntimeError as error:
|
||||
if not refused_at_shutdown(error):
|
||||
raise
|
||||
pool = _job_pool(max(1, int(workers or 1)))
|
||||
futures = []
|
||||
try:
|
||||
for item in items:
|
||||
futures.append(pool.submit(func, item))
|
||||
except RuntimeError as error:
|
||||
if refused_at_shutdown(error):
|
||||
note_refusal(_logger, "A worker pool")
|
||||
# Drop what was queued but not started; it runs here instead.
|
||||
# (A refused Thread.start leaves its item queued without a future
|
||||
# here: it is cancelled too, unless a busy pool thread took it in
|
||||
# between, the one case where an item can run twice.)
|
||||
pool.shutdown(wait=True, cancel_futures=True)
|
||||
else:
|
||||
return [future.result() for future in futures]
|
||||
return _run_rest(func, items, futures)
|
||||
_forget(pool)
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
elif SHUT_POOL_REFUSAL not in str(error):
|
||||
raise
|
||||
return _collect(func, items, futures)
|
||||
|
||||
|
||||
def _collect(func, items, futures):
|
||||
"""``_run_rest``; on an error, first drop the call's items not started
|
||||
and wait for its running ones, as the old per-call pool's exit did."""
|
||||
try:
|
||||
return _run_rest(func, items, futures)
|
||||
except BaseException:
|
||||
for future in futures:
|
||||
future.cancel()
|
||||
wait(futures)
|
||||
raise
|
||||
|
||||
|
||||
def _run_rest(func, items, futures):
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
"""PNG text chunks without decoding pixels (#523).
|
||||
|
||||
Pillow's ``PngImageFile.text`` calls ``load()``, which decodes every pixel, to
|
||||
reach text chunks written after the image data. On a scan or a busy image
|
||||
monitor that is a full-size buffer per file for a few kilobytes of text.
|
||||
|
||||
``png_text`` returns the same dictionary as ``image.text`` for an opened,
|
||||
single-frame PNG, by walking the chunks after the first IDAT with Pillow's own
|
||||
chunk handlers (same decoding, limits and duplicate rule) and seeking over the
|
||||
pixel data. It follows Pillow's ``load_end`` loop: chunk checksums after the
|
||||
pixels are not checked, an unreadable chunk header ends the walk, and another
|
||||
chunk that runs past the end of the file fails like ``ImageFile._safe_read``
|
||||
(unless ``ImageFile.LOAD_TRUNCATED_IMAGES``).
|
||||
|
||||
The one difference is pixel data, which is never decoded: pixel data that is
|
||||
undecodable, or cut short, no longer hides the text read before it. Pillow
|
||||
itself accepts a file cut inside its last pixel chunk once the decoder has
|
||||
all rows, and only a decode can tell those apart, so the walk never reads
|
||||
less than ``.text`` did (the PR #170 "legacy parity floor").
|
||||
|
||||
Anything else (other formats, animated PNGs, an image already loaded) is
|
||||
answered by Pillow itself.
|
||||
"""
|
||||
|
||||
import os
|
||||
import struct
|
||||
|
||||
from PIL import ImageFile
|
||||
|
||||
_PIXEL_CHUNKS = (b"IDAT", b"fdAT")
|
||||
# Signature, IHDR chunk and IDAT chunk header: the earliest pixel data offset.
|
||||
_FIRST_DATA_OFFSET = 8 + 25 + 8
|
||||
|
||||
|
||||
def png_text(image) -> dict:
|
||||
"""``dict(image.text)`` for an opened image, without a pixel decode.
|
||||
|
||||
The image stays usable: its file position and Pillow state are restored.
|
||||
Non-PNG images answer ``{}`` (only PNG has text chunks in Pillow).
|
||||
"""
|
||||
if not _walkable(image):
|
||||
return dict(getattr(image, "text", None) or {})
|
||||
fp = image.fp
|
||||
resume = fp.tell()
|
||||
try:
|
||||
return _walk_after_pixels(image.png, fp, image.tile[0][2] - 8)
|
||||
finally:
|
||||
fp.seek(resume)
|
||||
|
||||
|
||||
def _walkable(image) -> bool:
|
||||
"""A plain single-frame PNG whose header pass stopped at its first IDAT."""
|
||||
tile = getattr(image, "tile", None)
|
||||
return (
|
||||
image.format == "PNG"
|
||||
and getattr(image, "png", None) is not None
|
||||
and getattr(image, "fp", None) is not None
|
||||
and not getattr(image, "is_animated", False)
|
||||
and isinstance(tile, list)
|
||||
and len(tile) == 1
|
||||
and tile[0][0] == "zip"
|
||||
# The first IDAT's data offset; anything else is not a layout we know.
|
||||
and isinstance(tile[0][2], int)
|
||||
and tile[0][2] >= _FIRST_DATA_OFFSET
|
||||
)
|
||||
|
||||
|
||||
def _walk_after_pixels(stream, fp, offset: int) -> dict:
|
||||
"""Pillow's ``load_end`` chunk loop from the first IDAT header, skipping pixels.
|
||||
|
||||
``stream`` is the image's own ``PngStream``: its text so far and its text
|
||||
memory budget carry on exactly as a ``load()`` would carry them. The text
|
||||
and info dictionaries are copied first, so ``image.text`` and
|
||||
``image.info`` are unchanged. Like ``load_end``, a handler for a chunk
|
||||
misplaced after the pixels (a second IHDR or PLTE) may still update the
|
||||
stream's other header fields; Pillow ignores them once the image is open.
|
||||
"""
|
||||
end = fp.seek(0, os.SEEK_END)
|
||||
saved = stream.im_text, stream.im_info, stream.text_memory
|
||||
stream.im_text, stream.im_info = dict(saved[0]), dict(saved[1])
|
||||
try:
|
||||
fp.seek(offset)
|
||||
while True:
|
||||
try:
|
||||
kind, position, length = stream.read()
|
||||
except (struct.error, SyntaxError):
|
||||
break
|
||||
if kind == b"IEND":
|
||||
break
|
||||
try:
|
||||
handled = kind not in _PIXEL_CHUNKS and _handle(
|
||||
stream, kind, position, length
|
||||
)
|
||||
except UnicodeDecodeError:
|
||||
break
|
||||
if not handled:
|
||||
if position + length > end:
|
||||
if kind in _PIXEL_CHUNKS or ImageFile.LOAD_TRUNCATED_IMAGES:
|
||||
break
|
||||
raise OSError("Truncated File Read")
|
||||
fp.seek(position + length + 4)
|
||||
continue
|
||||
fp.read(4) # checksum: not verified after the pixels, as in Pillow
|
||||
return stream.im_text
|
||||
finally:
|
||||
stream.im_text, stream.im_info, stream.text_memory = saved
|
||||
|
||||
|
||||
def _handle(stream, kind: bytes, position: int, length: int) -> bool:
|
||||
"""Run Pillow's handler; ``False`` when the chunk's bytes must be skipped."""
|
||||
try:
|
||||
stream.call(kind, position, length)
|
||||
except (EOFError, AttributeError):
|
||||
# A chunk Pillow reads past (EOFError) or has no handler for.
|
||||
return False
|
||||
return True
|
||||
@@ -62,6 +62,13 @@
|
||||
(typeof job.title === "string" && job.title.length > 0 && job.title.length <= 200))) {
|
||||
this.job = { id: job.id, title: job.title };
|
||||
}
|
||||
// A move refused for space (#320) carries its review, so the page
|
||||
// can show what would have moved and why it cannot start.
|
||||
this.plan = null;
|
||||
if (this.code === "insufficient-space" && current && current.plan &&
|
||||
typeof current.plan === "object" && !Array.isArray(current.plan)) {
|
||||
this.plan = current.plan;
|
||||
}
|
||||
this.source = null;
|
||||
if (this.code === "invalid-source" && current && typeof current.source === "string" &&
|
||||
current.source.length > 0 && current.source.length <= 4096) {
|
||||
|
||||
@@ -170,7 +170,10 @@ export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
bar.max = 100;
|
||||
const stats = element('dl', undefined, 'stat-row');
|
||||
const tileValues = new Map();
|
||||
progressPart.node.append(bar, stats);
|
||||
// #320 D198: a kind with phases (Move thumbnails) says which one runs and what it means, under the bar.
|
||||
const phase = element('p', '', 'small');
|
||||
phase.id = 'job-phase';
|
||||
progressPart.node.append(bar, phase, stats);
|
||||
|
||||
const resultsPart = section('Results', 'job-results-title');
|
||||
const search = element('label', undefined, 'search-field');
|
||||
@@ -228,6 +231,7 @@ export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
function layout(job) {
|
||||
view = viewOf(job);
|
||||
if (!view.tiles.length) stats.remove();
|
||||
if (typeof view.phase !== 'function') phase.remove();
|
||||
for (const [group, label, tone] of view.tiles) {
|
||||
const tile = element('div', undefined, tone ? `stat ${tone}` : 'stat');
|
||||
const value = element('dd', '0');
|
||||
@@ -341,6 +345,7 @@ export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
cancel.remove();
|
||||
}
|
||||
status.textContent = progressLine(job);
|
||||
if (typeof view?.phase === 'function') phase.textContent = view.phase(job);
|
||||
// D111: no value while the total is unknown; a job that ended with nothing counted shows how it ended.
|
||||
showProgress(bar, job.total > 0 || job.state !== 'finished' ? job : { ...job, progress: 100 });
|
||||
bar.setAttribute('aria-label', scan ? 'Scan progress' : `${job.title} progress`);
|
||||
|
||||
@@ -51,6 +51,26 @@ export const removedOf = job => counted(job, 'removed', 'removed');
|
||||
/** "12 imported" (System S10: the LoRA reference import). */
|
||||
export const importedOf = job => counted(job, 'imported', 'imported');
|
||||
|
||||
/** #320 D198: the step a Move thumbnails job is in, and what it means meanwhile (under the progress bar). */
|
||||
export function movePhase(job) {
|
||||
const from = job.scope?.from === 'folder' ? 'the old folder' : 'inside each source';
|
||||
if (job.state === 'running') {
|
||||
if (job.scope?.phase === 'remove') return 'Removing the old copies a move left. Thumbnails are read from the new location.';
|
||||
if (job.cancellable !== false) {
|
||||
return `Step 1 of 3: copying. Thumbnails are still read from ${from} until the switch. New thumbnails are not `
|
||||
+ 'written until the move ends; originals are shown instead.';
|
||||
}
|
||||
return 'Step 2 and 3 of 3: switching, then removing the old copies. This can no longer be cancelled.';
|
||||
}
|
||||
if (job.state === 'finished') {
|
||||
const left = job.counters?.left ?? 0;
|
||||
return left ? `Thumbnails are now read from the new location. ${number(left)} ${noun(left, 'file')} that `
|
||||
+ `${left === 1 ? 'is' : 'are'} not ${left === 1 ? 'a thumbnail was' : 'thumbnails were'} left in place.`
|
||||
: 'Thumbnails are now read from the new location.';
|
||||
}
|
||||
return 'Nothing changed: thumbnails are read from where they were.';
|
||||
}
|
||||
|
||||
/** "2 output folders": what a scan or a thumbnail job walks. */
|
||||
function foldersOf(job) {
|
||||
const folders = job.scope?.output_folders;
|
||||
@@ -159,6 +179,32 @@ export const KINDS = Object.freeze({
|
||||
outcomes: tiles([['generated', 'Generated'], ['errors', 'Errors'], ['skipped', 'Skipped']]),
|
||||
openable: groups('generated', 'skipped'),
|
||||
}),
|
||||
// #320 D198: Move thumbnails, a `thumbnails` job with `scope.mode` "move": Moved (green), Already there, Left in
|
||||
// place and Errors (red); Cancel only until the switch (the server says so through `cancellable`).
|
||||
'thumbnails:move': Object.freeze({
|
||||
...GENERIC,
|
||||
pillVerb: 'Moving thumbnails',
|
||||
finishToast: job => (job.status_line && job.state !== 'running' ? job.status_line : titleFinished(job)),
|
||||
line: job => (job.state === 'running' ? '' : job.status_line || ''),
|
||||
kept: job => counted(job, 'moved', 'moved'),
|
||||
tiles: tiles([['moved', 'Moved', 'added'], ['skipped', 'Already there', ''], ['left', 'Left in place', ''],
|
||||
['errors', 'Errors', 'bad']]),
|
||||
outcomes: tiles([['moved', 'Moved'], ['errors', 'Errors'], ['skipped', 'Already there'], ['left', 'Left in place']]),
|
||||
log: 'Move log',
|
||||
phase: movePhase,
|
||||
cancelLabel: 'Cancel move',
|
||||
cancel: 'Cancel move',
|
||||
}),
|
||||
// #320 D200: Remove thumbnails of removed sources (`scope.mode` "prune"): Removed, Left in place and Errors.
|
||||
'thumbnails:prune': Object.freeze({
|
||||
...GENERIC,
|
||||
pillVerb: 'Removing thumbnails',
|
||||
finishToast: job => (job.status_line && job.state !== 'running' ? job.status_line : titleFinished(job)),
|
||||
line: job => (job.state === 'running' ? '' : job.status_line || ''),
|
||||
kept: removedOf,
|
||||
tiles: tiles([['removed', 'Removed', 'added'], ['left', 'Left in place', ''], ['errors', 'Errors', 'bad']]),
|
||||
outcomes: tiles([['removed', 'Removed'], ['errors', 'Errors'], ['left', 'Left in place']]),
|
||||
}),
|
||||
// System S7: a reviewed maintenance plan's apply, titled after its operation; it runs without a known total.
|
||||
maintenance: Object.freeze({
|
||||
...GENERIC,
|
||||
@@ -243,7 +289,9 @@ export const held = job => job.cancellable === false && Object.keys(countsOf(job
|
||||
*/
|
||||
export function viewOf(job) {
|
||||
if (held(job)) return GENERIC;
|
||||
const view = kindOf(job.kind);
|
||||
// #320 D198: a thumbnails job chooses its tiles by its scope's mode (move, prune).
|
||||
const mode = job.kind === 'thumbnails' ? job.scope?.mode : null;
|
||||
const view = (mode === 'move' || mode === 'prune') ? KINDS[`thumbnails:${mode}`] : kindOf(job.kind);
|
||||
const counts = countsOf(job);
|
||||
const has = ([group]) => Object.hasOwn(counts, group);
|
||||
if (view.tiles.every(has) && view.outcomes.every(has)) return view;
|
||||
|
||||
@@ -176,6 +176,12 @@ function mountWorkspace() {
|
||||
function notifyCounts() { for (const listener of countListeners) listener(); }
|
||||
/** Re-read the given counts unless a read is already in flight. */
|
||||
function freshenCounts(ids) { refreshCounts(ids.filter(id => !countRequests.has(id))); }
|
||||
/** #555: the counts as System › Overview reads them, so its Images metric and the Images badge cannot differ. */
|
||||
const sharedCounts = Object.freeze({
|
||||
get: id => knownCounts.get(id),
|
||||
subscribe(listener) { countListeners.add(listener); return () => countListeners.delete(listener); },
|
||||
freshen: freshenCounts,
|
||||
});
|
||||
/** A count a page has just read itself: it replaces the badge, and an older read in flight no longer lands. */
|
||||
function setCount(id, count) {
|
||||
countRequests.get(id)?.abort();
|
||||
@@ -488,7 +494,7 @@ function mountWorkspace() {
|
||||
if (route.id === 'system') {
|
||||
// The concept's System (app.js renderSystem): its heading description, section nav and Save row.
|
||||
description.textContent = route.description;
|
||||
system = mountSystem(main, api, view, { section: route.section, activity });
|
||||
system = mountSystem(main, api, view, { section: route.section, activity, counts: sharedCounts });
|
||||
return;
|
||||
}
|
||||
if (route.id === 'inspect') {
|
||||
|
||||
@@ -97,6 +97,9 @@ export function reviewGrid(location) {
|
||||
? 'The database backups (config.json stays where PROMPT_MANAGER_CONFIG_PATH names it)'
|
||||
: 'config.json and the database backups'],
|
||||
['Old files', 'Renamed *.moved-to-…, never deleted'],
|
||||
// #320 D201: previews kept in one folder do not move with the library (D8A).
|
||||
...(typeof location.thumbnails_folder === 'string' && location.thumbnails_folder
|
||||
? [['Thumbnails', `Stay in ${location.thumbnails_folder}. Move them in System › Sources & scans.`]] : []),
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,13 @@ const linkedTo = (target,roles) => Object.keys(roles).filter(role=>Array.isArray
|
||||
const nodeLabel = target => `#${target.id} ${target.label}`;
|
||||
// A lost reply leaves the outcome unknown; anything else the host said is a refusal, and nothing was written.
|
||||
const LOST_REPLY = /timed out|aborted|disposed/i;
|
||||
const outcomeUnknown = (posted,error) => posted && (LOST_REPLY.test(error.message) || /unexpected acknowledgement/.test(error.message));
|
||||
const UNKNOWN_ADVICE = 'Check ComfyUI, then preview again.';
|
||||
/** The Destination node value that sends to every eligible node of the window (#542, D191). Node ids never take it. */
|
||||
export const ALL_NODES = '*';
|
||||
// "All" sends in node id order: 2 before 10, and a subgraph id "3:1" after "3".
|
||||
const byNodeId = (a,b) => a.id.localeCompare(b.id,'en',{numeric:true});
|
||||
const allMessage = (count,where) => `Sending to all ${count} eligible nodes in ${where}.`;
|
||||
|
||||
/** Why nothing in the discovered windows can receive these roles, most specific first (D173). */
|
||||
function unavailableMessage(found,roles) {
|
||||
@@ -40,23 +47,90 @@ function distinctLabels(items,suffix) {
|
||||
|
||||
export function createSendSession({client, roles:rawRoles, onChange}) {
|
||||
const roles = Object.freeze(validateRoles(rawRoles));
|
||||
let state = {phase:'idle',hosts:[],hostId:'',targetId:'',preview:null,outcomeUnknown:false,message:''};
|
||||
// `previews` holds the reviewed nodes of "All", in send order; `preview` the one node otherwise.
|
||||
let state = {phase:'idle',hosts:[],hostId:'',targetId:'',preview:null,previews:[],outcomeUnknown:false,message:''};
|
||||
let operation, generation = 0, disposed = false;
|
||||
const emit = patch => { if (!disposed) { state = Object.freeze({...state,...patch}); onChange(state); } };
|
||||
function invalidate() { generation++; operation?.abort(); operation = null; }
|
||||
function begin(phase) { invalidate(); operation = new AbortController(); emit({phase,preview:null,outcomeUnknown:false,message:''}); return {signal:operation.signal,version:generation}; }
|
||||
function begin(phase) { invalidate(); operation = new AbortController(); emit({phase,preview:null,previews:[],outcomeUnknown:false,message:''}); return {signal:operation.signal,version:generation}; }
|
||||
const current = request => !disposed && generation === request.version;
|
||||
const selected = () => state.hosts.find(host=>host.hostId===state.hostId)?.targets.find(target=>target.id===state.targetId);
|
||||
async function fetchPreview(request) {
|
||||
const preview=await client.preview(state.hostId,state.targetId,roles,{signal:request.signal});
|
||||
const chosenHost = () => state.hosts.find(host=>host.hostId===state.hostId);
|
||||
/** The nodes the current choice sends to: one, every eligible node of the window for "All", or none. */
|
||||
function chosen() {
|
||||
const host=chosenHost();
|
||||
if (!host) return [];
|
||||
if (state.targetId===ALL_NODES) return host.targets.length>1 ? host.targets.toSorted(byNodeId) : [];
|
||||
const target=host.targets.find(item=>item.id===state.targetId);
|
||||
return target ? [target] : [];
|
||||
}
|
||||
// checkRevision is off only for a later node of "All", whose revision our own earlier write may have moved.
|
||||
async function fetchPreview(request,hostId,target,checkRevision=true) {
|
||||
const preview=await client.preview(hostId,target.id,roles,{signal:request.signal});
|
||||
if (!current(request)) return null;
|
||||
const fields=preview?.fields;
|
||||
if(typeof preview?.ticket!=='string' || !preview.ticket || preview.target?.id!==state.targetId || !Array.isArray(fields) || fields.length!==Object.keys(roles).length || new Set(fields.map(field=>field?.role)).size!==fields.length || Array.from(fields).some(field=>!field || !Object.hasOwn(roles,field.role) || typeof field.before!=='string' || field.after!==roles[field.role])) throw Error('ComfyUI returned an unexpected preview. Find the destination and preview again.');
|
||||
if(typeof preview?.ticket!=='string' || !preview.ticket || preview.target?.id!==target.id || !Array.isArray(fields) || fields.length!==Object.keys(roles).length || new Set(fields.map(field=>field?.role)).size!==fields.length || Array.from(fields).some(field=>!field || !Object.hasOwn(roles,field.role) || typeof field.before!=='string' || field.after!==roles[field.role])) throw Error('ComfyUI returned an unexpected preview. Find the destination and preview again.');
|
||||
// Undefined on a tab still running older code, which has nothing to compare.
|
||||
const found=selected()?.revision;
|
||||
if(found!==undefined && preview.revision!==found) throw Error('The workflow changed since this window was found. Nothing was sent. Find windows again.');
|
||||
const found=target.revision;
|
||||
if(checkRevision && found!==undefined && preview.revision!==found) throw Error('The workflow changed since this window was found. Nothing was sent. Find windows again.');
|
||||
return preview;
|
||||
}
|
||||
function checkAcknowledgement(acknowledgement,targetId) {
|
||||
if(acknowledgement?.targetId!==targetId || !acknowledgement.values || Object.keys(acknowledgement.values).length!==Object.keys(roles).length || Object.keys(roles).some(role=>!Object.hasOwn(acknowledgement.values,role) || acknowledgement.values[role]!==roles[role])) throw Error('ComfyUI returned an unexpected acknowledgement.');
|
||||
}
|
||||
const reviewedText = change => JSON.stringify([change.fields.map(field=>[field.role,field.before]).sort(),change.composition ?? null]);
|
||||
const sameChanges = (reviewed,fresh) => reviewedText(reviewed)===reviewedText(fresh);
|
||||
/** Q542-2 C: every node of "All" previews before anything is written; the first refusal names its node. */
|
||||
async function previewEach(request,hostId,targets) {
|
||||
const reviewed=[];
|
||||
for (const target of targets) {
|
||||
try {
|
||||
const preview=await fetchPreview(request,hostId,target);
|
||||
if (!preview) return null;
|
||||
reviewed.push({target,label:target.label,preview});
|
||||
} catch(error) {
|
||||
if (!current(request)) return null;
|
||||
throw Error(`${target.label}: ${error.message}${/Nothing was sent/.test(error.message) ? '' : ' Nothing was sent.'}`);
|
||||
}
|
||||
}
|
||||
return reviewed;
|
||||
}
|
||||
/**
|
||||
* Sends the reviewed nodes one after another, best effort past refusals. Only the first node of a pass taken in this
|
||||
* click keeps its ticket: a held review may have expired at the host (30 s), and our own writes may make later
|
||||
* tickets stale, so every other node takes a fresh ticket that must show exactly the reviewed text.
|
||||
*/
|
||||
async function applyEach(request,hostId,reviewed,where,held) {
|
||||
const sent=[],refused=[];let written=false;
|
||||
for (const [index,{target,preview}] of reviewed.entries()) {
|
||||
let posted=false;
|
||||
try {
|
||||
let ticket=preview.ticket;
|
||||
if (index>0 || held) {
|
||||
// Until something was posted the revision is still the discovered one; after that our own write may move it.
|
||||
const fresh=await fetchPreview(request,hostId,target,!written);
|
||||
if (!fresh) return;
|
||||
if (!sameChanges(preview,fresh)) throw Error('Its prompt changed since the preview. Preview again.');
|
||||
ticket=fresh.ticket;
|
||||
}
|
||||
posted=written=true;
|
||||
const acknowledgement=await client.apply(hostId,ticket,{signal:request.signal});
|
||||
if (!current(request)) return;
|
||||
checkAcknowledgement(acknowledgement,target.id);
|
||||
sent.push(target);
|
||||
} catch(error) {
|
||||
if (!current(request)) return;
|
||||
const summary=[`Sent to ${sent.length} of ${reviewed.length} nodes in ${where}.`,...refused];
|
||||
if (outcomeUnknown(posted,error)) {
|
||||
const unsent=reviewed.slice(index+1).map(item=>item.label);
|
||||
emit({phase:'error',outcomeUnknown:true,message:[...summary,`No confirmation received from ${target.label}; the update may have applied.`,...(unsent.length ? [`Not sent: ${unsent.join(', ')}.`] : []),UNKNOWN_ADVICE,error.message].join(' ')});
|
||||
return;
|
||||
}
|
||||
refused.push(`${target.label}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
const message=[`Sent to ${sent.length} of ${reviewed.length} nodes in ${where}.`,...refused].join(' ');
|
||||
emit(refused.length ? {phase:'error',message} : {phase:'success',message});
|
||||
}
|
||||
function hostChoice(host) {
|
||||
if (host.targets.length>1) return {hostId:host.hostId,targetId:'',message:`Choose a destination node in ${host.label}.`};
|
||||
const [target] = host.targets;
|
||||
@@ -79,38 +153,61 @@ export function createSendSession({client, roles:rawRoles, onChange}) {
|
||||
else { const choice=hostChoice(hosts[0]); emit({phase:'select',hosts,...choice,message:choice.message+note}); }
|
||||
} catch (error) { if (current(request)) emit({phase:'error',message:error.message}); }
|
||||
},
|
||||
selectHost(hostId) { if (disposed) return; invalidate(); const host=state.hosts.find(host=>host.hostId===hostId); emit({phase:'select',preview:null,outcomeUnknown:false,...(host ? hostChoice(host) : {hostId:'',targetId:'',message:'Choose a ComfyUI window.'})}); },
|
||||
selectTarget(targetId) { if (disposed) return; invalidate(); const host=state.hosts.find(host=>host.hostId===state.hostId); const chosen=host?.targets.some(target=>target.id===targetId); emit({phase:'select',outcomeUnknown:false,targetId:chosen?targetId:'',preview:null,message:chosen?'Apply to send it, or preview the changes first.':'Choose a destination node, then preview changes.'}); },
|
||||
selectHost(hostId) { if (disposed) return; invalidate(); const host=state.hosts.find(host=>host.hostId===hostId); emit({phase:'select',preview:null,previews:[],outcomeUnknown:false,...(host ? hostChoice(host) : {hostId:'',targetId:'',message:'Choose a ComfyUI window.'})}); },
|
||||
selectTarget(targetId) {
|
||||
if (disposed) return; invalidate(); const host=chosenHost();
|
||||
// D173: "All" is a choice only where there are two or more nodes, and it is never made for you.
|
||||
const all=targetId===ALL_NODES && host?.targets.length>1;
|
||||
const chosenOne=all || Boolean(host?.targets.some(target=>target.id===targetId));
|
||||
const message=all ? `${allMessage(host.targets.length,host.label)} Apply to send it, or preview the changes first.` : chosenOne ? 'Apply to send it, or preview the changes first.' : 'Choose a destination node, then preview changes.';
|
||||
emit({phase:'select',outcomeUnknown:false,targetId:chosenOne?targetId:'',preview:null,previews:[],message});
|
||||
},
|
||||
async preview() {
|
||||
if (disposed || !selected() || ['previewing','applying'].includes(state.phase)) return;
|
||||
const targets=chosen();
|
||||
if (disposed || !targets.length || ['previewing','applying'].includes(state.phase)) return;
|
||||
const all=state.targetId===ALL_NODES; const hostId=state.hostId; const windowLabel=chosenHost()?.label;
|
||||
const request=begin('previewing');
|
||||
try {
|
||||
const preview=await fetchPreview(request);
|
||||
if (all) {
|
||||
const previews=await previewEach(request,hostId,targets);
|
||||
if (previews) emit({phase:'ready',previews,message:`${allMessage(targets.length,windowLabel)} Review the complete changes, then apply.`});
|
||||
return;
|
||||
}
|
||||
const preview=await fetchPreview(request,hostId,targets[0]);
|
||||
if (!preview) return;
|
||||
const windowLabel=state.hosts.find(host=>host.hostId===state.hostId)?.label;
|
||||
emit({phase:'ready',preview,message:`Sending to ${nodeLabel({id:preview.target.id,label:preview.target.label ?? ''})} in ${windowLabel}. Review the complete changes, then apply.`});
|
||||
} catch(error) { if(current(request)) emit({phase:'error',preview:null,message:error.message}); }
|
||||
} catch(error) { if(current(request)) emit({phase:'error',preview:null,previews:[],message:error.message}); }
|
||||
},
|
||||
async apply() {
|
||||
// Success stays closed: the second click of a double click must not send again.
|
||||
if(disposed || !selected() || !['select','ready','error'].includes(state.phase) || state.outcomeUnknown) return;
|
||||
const held=state.phase==='ready' ? state.preview : null;
|
||||
const targetId=state.targetId; const hostId=state.hostId;
|
||||
const targets=chosen();
|
||||
if(disposed || !targets.length || !['select','ready','error'].includes(state.phase) || state.outcomeUnknown) return;
|
||||
const all=state.targetId===ALL_NODES;
|
||||
const held=state.phase==='ready' ? (all ? state.previews : state.preview) : null;
|
||||
const hostId=state.hostId; const windowLabel=chosenHost()?.label;
|
||||
const request=begin('applying');
|
||||
emit({message:'Waiting for ComfyUI confirmation. Closing this dialog does not undo a sent update.'});
|
||||
if (all) {
|
||||
let reviewed;
|
||||
try { reviewed=held?.length ? held : await previewEach(request,hostId,targets); }
|
||||
catch(error) { if(current(request)) emit({phase:'error',message:error.message}); return; }
|
||||
if (reviewed) await applyEach(request,hostId,reviewed,windowLabel,Boolean(held?.length));
|
||||
return;
|
||||
}
|
||||
const [target]=targets;
|
||||
let posted=false;
|
||||
try {
|
||||
const preview=held ?? await fetchPreview(request);
|
||||
const preview=held ?? await fetchPreview(request,hostId,target);
|
||||
if(!preview || !current(request)) return;
|
||||
posted=true;
|
||||
const acknowledgement=await client.apply(hostId,preview.ticket,{signal:request.signal});
|
||||
if(!current(request)) return;
|
||||
if(acknowledgement?.targetId!==targetId || !acknowledgement.values || Object.keys(acknowledgement.values).length!==Object.keys(roles).length || Object.keys(roles).some(role=>!Object.hasOwn(acknowledgement.values,role) || acknowledgement.values[role]!==roles[role])) throw Error('ComfyUI returned an unexpected acknowledgement.');
|
||||
checkAcknowledgement(acknowledgement,target.id);
|
||||
emit({phase:'success',message:'Sent to ComfyUI.'});
|
||||
} catch(error) {
|
||||
if(!current(request)) return;
|
||||
const unknown=posted && (LOST_REPLY.test(error.message) || /unexpected acknowledgement/.test(error.message));
|
||||
emit({phase:'error',preview:null,outcomeUnknown:unknown,message:unknown ? `No confirmation received. The update may have applied. Check ComfyUI, then preview again. ${error.message}` : error.message});
|
||||
const unknown=outcomeUnknown(posted,error);
|
||||
emit({phase:'error',preview:null,outcomeUnknown:unknown,message:unknown ? `No confirmation received. The update may have applied. ${UNKNOWN_ADVICE} ${error.message}` : error.message});
|
||||
}
|
||||
},
|
||||
dispose() { if(disposed) return; disposed=true; invalidate(); client.dispose(); },
|
||||
@@ -159,7 +256,7 @@ export function createPromptSender({scope,baseUrl=window.ApiClient.resolveUrl('/
|
||||
inspect.disabled=true;apply.disabled=true;
|
||||
const actions=element('footer',undefined,'prompt-sender-actions');actions.append(find,inspect,apply);
|
||||
dialog.append(header,description,controls,status,preview,actions);document.body.append(dialog);
|
||||
let session,closed=false;
|
||||
let session,closed=false,headingIds=0;
|
||||
function dispose() {
|
||||
if(closed) return;closed=true;local.dispose();session?.dispose();
|
||||
if(dialog.open) dialog.close();dialog.remove();
|
||||
@@ -176,8 +273,8 @@ export function createPromptSender({scope,baseUrl=window.ApiClient.resolveUrl('/
|
||||
function options(select,placeholder,entries,value) {
|
||||
select.replaceChildren(...[["",placeholder],...entries].map(([value,label])=>{const option=element('option',label);option.value=value;return option;}));select.value=value;
|
||||
}
|
||||
function textBlock(parent,label,value) {
|
||||
parent.append(element('h4',label));const text=element('pre');text.tabIndex=0;text.append(element('code',value === '' ? '(empty)' : value));parent.append(text);
|
||||
function textBlock(parent,label,value,level=4) {
|
||||
parent.append(element(`h${Math.min(level,6)}`,label));const text=element('pre');text.tabIndex=0;text.append(element('code',value === '' ? '(empty)' : value));parent.append(text);
|
||||
}
|
||||
function render(state) {
|
||||
if(closed) return;
|
||||
@@ -187,28 +284,43 @@ export function createPromptSender({scope,baseUrl=window.ApiClient.resolveUrl('/
|
||||
dialog.setAttribute('aria-busy',String(busy));
|
||||
options(host,'Choose a ComfyUI window',state.hosts.map(item=>[item.hostId,item.label]),state.hostId);
|
||||
const selected=state.hosts.find(item=>item.hostId===state.hostId);
|
||||
options(target,'Choose a destination node',(selected?.targets || []).map(item=>[item.id,item.label]),state.targetId);
|
||||
const nodes=(selected?.targets || []).map(item=>[item.id,item.label]);
|
||||
// Q542-4 A: the last option sends to every eligible node of this window.
|
||||
if(nodes.length>1) nodes.push([ALL_NODES,`All ${nodes.length} eligible nodes`]);
|
||||
options(target,'Choose a destination node',nodes,state.targetId);
|
||||
host.disabled=busy;target.disabled=busy || !selected;
|
||||
find.disabled=busy;inspect.disabled=busy || !state.targetId;apply.disabled=busy || !state.targetId || state.phase==='success' || state.outcomeUnknown;
|
||||
preview.replaceChildren();
|
||||
if(state.preview) {
|
||||
preview.append(element('h3','Review changes'));
|
||||
for(const field of state.preview.fields || []) {
|
||||
const section=element('section',undefined,`prompt-sender-field prompt-role-${field.role}`);
|
||||
section.append(element('h3',field.role==='positive'?'Positive prompt':'Negative prompt'));
|
||||
textBlock(section,'Before',field.before);textBlock(section,'After',field.after);preview.append(section);
|
||||
}
|
||||
const composition=state.preview.composition;
|
||||
if(composition) {
|
||||
const section=element('section',undefined,'prompt-sender-composition');section.append(element('h3','Positive composition'));
|
||||
textBlock(section,'Prepend (preserved)',composition.prepend || '');textBlock(section,'Append (preserved)',composition.append || '');
|
||||
textBlock(section,'Combined before',composition.before ?? 'Resolved at execution from connected inputs.');
|
||||
textBlock(section,'Combined after',composition.after ?? 'Resolved at execution from connected inputs.');
|
||||
if(composition.prependConnected || composition.appendConnected) section.append(element('p','Connected prepend and append inputs are preserved and resolved when the workflow runs.'));
|
||||
preview.append(section);
|
||||
changes(preview,state.preview,3);
|
||||
}
|
||||
if(state.previews?.length) {
|
||||
preview.append(element('h3','Review changes'));
|
||||
// One section per node of "All", in the order they will be sent, named by its heading.
|
||||
for(const item of state.previews) {
|
||||
const node=element('section',undefined,'prompt-sender-node');const heading=element('h4',item.label);
|
||||
heading.id=`prompt-sender-node-${++headingIds}`;node.setAttribute('aria-labelledby',heading.id);
|
||||
node.append(heading);changes(node,item.preview,5);preview.append(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
function changes(parent,change,level) {
|
||||
for(const field of change.fields || []) {
|
||||
const section=element('section',undefined,`prompt-sender-field prompt-role-${field.role}`);
|
||||
section.append(element(`h${level}`,field.role==='positive'?'Positive prompt':'Negative prompt'));
|
||||
textBlock(section,'Before',field.before,level+1);textBlock(section,'After',field.after,level+1);parent.append(section);
|
||||
}
|
||||
const composition=change.composition;
|
||||
if(composition) {
|
||||
const section=element('section',undefined,'prompt-sender-composition');section.append(element(`h${level}`,'Positive composition'));
|
||||
textBlock(section,'Prepend (preserved)',composition.prepend || '',level+1);textBlock(section,'Append (preserved)',composition.append || '',level+1);
|
||||
textBlock(section,'Combined before',composition.before ?? 'Resolved at execution from connected inputs.',level+1);
|
||||
textBlock(section,'Combined after',composition.after ?? 'Resolved at execution from connected inputs.',level+1);
|
||||
if(composition.prependConnected || composition.appendConnected) section.append(element('p','Connected prepend and append inputs are preserved and resolved when the workflow runs.'));
|
||||
parent.append(section);
|
||||
}
|
||||
}
|
||||
dialog.showModal();
|
||||
try {
|
||||
const url=new URL(baseUrl,window.location.href);
|
||||
|
||||
@@ -179,13 +179,27 @@ export function reviewRefusal(error) {
|
||||
return REVIEW_FAILED;
|
||||
}
|
||||
|
||||
/**
|
||||
* #583: the review uploads not answered yet. WebKit can read a FormData's file part after the fetch call returned (it
|
||||
* did whenever a request interceptor, here Playwright's, was armed); a garbage collection in between dropped a form
|
||||
* nothing referenced any more, and the server got an empty file. Each form is held here until its upload is answered.
|
||||
* Exported for the unit tests only.
|
||||
*/
|
||||
export const uploadsInFlight = new Set();
|
||||
|
||||
/** Stage `file` for review; the server's answer, or an ApiError naming its refusal. */
|
||||
export async function uploadRestore(file, { fetchUrl = window.ApiClient.fetchUrl, ApiError = window.ApiClient.ApiError,
|
||||
signal } = {}) {
|
||||
const form = new FormData();
|
||||
form.append('database_file', file, file.name);
|
||||
const response = await fetchUrl(PREVIEW_URL, { method: 'POST', body: form, cache: 'no-store', credentials: 'same-origin',
|
||||
signal });
|
||||
uploadsInFlight.add(form);
|
||||
let response;
|
||||
try {
|
||||
response = await fetchUrl(PREVIEW_URL, { method: 'POST', body: form, cache: 'no-store', credentials: 'same-origin',
|
||||
signal });
|
||||
} finally {
|
||||
uploadsInFlight.delete(form);
|
||||
}
|
||||
let data = null;
|
||||
try {
|
||||
data = await response.json();
|
||||
|
||||
@@ -11,8 +11,10 @@
|
||||
* D99: its Review thumbnails opens the same "Thumbnail maintenance" dialog as Sources (thumbnail-review.mjs).
|
||||
* #317: it reads "N missing · M outdated · K failed"; while previews are outdated (another format than the
|
||||
* configured one) a second button, "Convert outdated", opens that dialog on its Convert outdated scope (D131).
|
||||
* - D96: the Statistics dialog lists the real runs per category, then "Library totals" (D153: no rating column or
|
||||
* rated-images total).
|
||||
* - #555: metric 3, "Images", is the shell's Images count (the Images page's total, its sidebar badge), not the
|
||||
* overview's indexed rows, which leave out files no row links: a dash until it is read or when its read failed.
|
||||
* - D96: the Statistics dialog lists the real runs per category, then "Database records" (D153: no rating column or
|
||||
* rated-images total; #555: the legacy row counts, labelled as records).
|
||||
* - D95: the "Health report" lists every check the server ran, Missing files per image source, with a chip per real
|
||||
* result and when it was checked. The check runs from Maintenance's notice ("Run health checks", D123) as a
|
||||
* read job of the registry, tracked in the Activity store (its event stream); the report opens when it ends.
|
||||
@@ -26,6 +28,7 @@ import { showWorkspaceToast } from './toast.mjs';
|
||||
import { noun } from './plural.mjs';
|
||||
import { operationRow } from './system-controls.mjs';
|
||||
import { openThumbnailReview } from './thumbnail-review.mjs';
|
||||
import { noticeSlot } from './system-thumbnail-location.mjs';
|
||||
import { isLocation, locationRow, onMoveEnded } from './library-location.mjs';
|
||||
import { STATUS_LOADING, STATUS_URL, backupStatusText } from './system-backups.mjs';
|
||||
|
||||
@@ -61,12 +64,20 @@ export function integrityText(health) {
|
||||
return health.status === 'passed' ? 'Passed' : 'Incomplete';
|
||||
}
|
||||
|
||||
/** The concept's four metrics `[value, label]`, in its order. */
|
||||
export function metricValues(overview) {
|
||||
/** #555: the Images metric's value from the shell's Images count; a dash while unread (undefined) or failed (null). */
|
||||
export function imagesText(images) {
|
||||
return Number.isSafeInteger(images) && images >= 0 ? number(images) : '–';
|
||||
}
|
||||
|
||||
/**
|
||||
* The concept's four metrics `[value, label]`, in its order. `images` is the shell's Images count (#555): the concept's
|
||||
* `images.length`, the same number as the Images sidebar badge.
|
||||
*/
|
||||
export function metricValues(overview, images) {
|
||||
return [
|
||||
[number(overview.library.positive_prompts), 'Positive prompts'],
|
||||
[number(overview.library.negative_variants), 'Negative variants'],
|
||||
[number(overview.images.indexed), 'Images'],
|
||||
[imagesText(images), 'Images'],
|
||||
[integrityText(overview.health), 'Database integrity'],
|
||||
];
|
||||
}
|
||||
@@ -174,10 +185,15 @@ export function categoryRows(statistics) {
|
||||
return statistics.by_category.map(row => [row.category ?? 'Uncategorized', number(row.runs)]);
|
||||
}
|
||||
|
||||
/** D96: the added "Library totals" rows `[label, count]` (the legacy statistics; D153 drops the rated images). */
|
||||
/**
|
||||
* D96: the added "Database records" rows `[label, count]` (the legacy statistics; D153 drops the rated images). #555:
|
||||
* they count raw rows (every prompt row, tags on a prompt, every image row), so they say so and never read as the
|
||||
* Library or Images page counts.
|
||||
*/
|
||||
export function totalsRows(statistics) {
|
||||
const { totals } = statistics;
|
||||
return [['Prompts', totals.total_prompts], ['Tags', totals.total_tags], ['Images', totals.total_images]]
|
||||
return [['Prompt records', totals.total_prompts], ['Tags on prompts', totals.total_tags],
|
||||
['Image records', totals.total_images]]
|
||||
.map(([label, value]) => [label, number(value)]);
|
||||
}
|
||||
|
||||
@@ -208,11 +224,11 @@ function table(caption, head, rows, empty) {
|
||||
return node;
|
||||
}
|
||||
|
||||
/** The Statistics dialog: "Runs by category" (the concept's table), then "Library totals" (D96). */
|
||||
/** The Statistics dialog: "Runs by category" (the concept's table), then "Database records" (D96, #555). */
|
||||
export function statisticsNodes(statistics) {
|
||||
return [
|
||||
table('Runs by category', ['Category', 'Runs'], categoryRows(statistics), 'No prompts in the library yet.'),
|
||||
table('Library totals', ['Total', 'Count'], totalsRows(statistics), ''),
|
||||
table('Database records', ['Total', 'Count'], totalsRows(statistics), ''),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -332,9 +348,9 @@ export function healthNotice({ api, activity, scope, navigate = navigateTo, noti
|
||||
return notice;
|
||||
}
|
||||
|
||||
function metricsNode(overview) {
|
||||
function metricsNode(overview, images) {
|
||||
const metrics = el('div', undefined, 'summary-metrics');
|
||||
for (const [value, label] of metricValues(overview)) {
|
||||
for (const [value, label] of metricValues(overview, images)) {
|
||||
const metric = el('div', undefined, 'metric');
|
||||
metric.append(el('strong', value), el('span', label));
|
||||
metrics.append(metric);
|
||||
@@ -349,7 +365,7 @@ function surface(heading, rows) {
|
||||
}
|
||||
|
||||
/** The Overview's nodes, and its thumbnail row (painted by its own read). */
|
||||
function overviewNodes(overview, { api, activity, notify, scope, navigate, openStatistics, openThumbnails }) {
|
||||
function overviewNodes(overview, { api, activity, notify, scope, navigate, openStatistics, openThumbnails, images }) {
|
||||
const go = section => () => navigate(`#/system/${section}`);
|
||||
const review = operationRow({ scope, title: ambiguousTitle(overview.needs_review),
|
||||
description: Number.isSafeInteger(overview.needs_review?.pending)
|
||||
@@ -374,26 +390,36 @@ function overviewNodes(overview, { api, activity, notify, scope, navigate, openS
|
||||
operationRow({ scope, title: 'One display configuration', description: 'Grid and density settings are shared across views.',
|
||||
label: 'Appearance', onClick: go('appearance') }),
|
||||
operationRow({ scope, title: sourcesTitle, description: sourcesDescription, label: 'Manage sources', onClick: go('sources') }),
|
||||
operationRow({ scope, title: 'Statistics', description: 'Runs by category, with library totals.', label: 'Explore',
|
||||
operationRow({ scope, title: 'Statistics', description: 'Runs by category, with database record counts.', label: 'Explore',
|
||||
onClick: openStatistics }),
|
||||
]);
|
||||
const columns = el('div', undefined, 'two-col');
|
||||
columns.append(health, preferences);
|
||||
return { nodes: [metricsNode(overview), columns], thumbnails, backup: backupRow };
|
||||
const metrics = metricsNode(overview, images);
|
||||
return { nodes: [metrics, columns], thumbnails, backup: backupRow, metrics };
|
||||
}
|
||||
|
||||
/**
|
||||
* The Overview section's mount (`mountSystem`'s `sections.overview`): returns its loading line, which the server's
|
||||
* answer replaces in place (the concept's metrics and columns sit directly in `.settings-layout`). A read that a newer
|
||||
* one superseded, or one answering after the section left, paints nothing.
|
||||
*
|
||||
* `counts` is the shell's sidebar counts (`{get(id), subscribe(listener), freshen(ids)}`, index.mjs): the Images metric
|
||||
* shows its `images` count and follows it, and a visit re-reads it as the workspace Overview does (#555).
|
||||
*/
|
||||
export function mountOverview(container, { api, activity, scope, navigate = navigateTo, notify = showWorkspaceToast,
|
||||
latestOnly = window.ApiClient.latestOnly, STALE = window.ApiClient.STALE }) {
|
||||
export function mountOverview(container, { api, activity, scope, counts = null, drafts = null, navigate = navigateTo,
|
||||
notify = showWorkspaceToast, latestOnly = window.ApiClient.latestOnly, STALE = window.ApiClient.STALE }) {
|
||||
const overviewRead = latestOnly({ lifetime: scope.signal });
|
||||
const statisticsRead = latestOnly({ lifetime: scope.signal });
|
||||
const thumbnailsRead = latestOnly({ lifetime: scope.signal });
|
||||
const backupRead = latestOnly({ lifetime: scope.signal });
|
||||
let shown = [];
|
||||
// #555: the Images metric's value node, repainted whenever the shell's Images count changes.
|
||||
let imagesValue = null;
|
||||
const imagesCount = () => counts?.get('images');
|
||||
const paintImages = () => {
|
||||
if (imagesValue) imagesValue.textContent = imagesText(imagesCount());
|
||||
};
|
||||
// The backup status (undefined while read, null when it failed), painted into its row once both are there (S9).
|
||||
let backupStatus;
|
||||
let backupRow = null;
|
||||
@@ -439,8 +465,11 @@ export function mountOverview(container, { api, activity, scope, navigate = navi
|
||||
try {
|
||||
const overview = await overviewRead(signal => api.get(OVERVIEW_URL, { signal, cache: 'no-store' }));
|
||||
if (overview === STALE || scope.signal.aborted) return;
|
||||
const { nodes, thumbnails, backup } = overviewNodes(overview, { api, activity, notify, scope, navigate, openStatistics,
|
||||
openThumbnails: trigger => openThumbnailReview({ api, store: activity, scope, trigger, navigate }) });
|
||||
const { nodes, thumbnails, backup, metrics } = overviewNodes(overview, { api, activity, notify, scope, navigate,
|
||||
openStatistics, images: imagesCount(),
|
||||
openThumbnails: trigger => openThumbnailReview({ api, store: activity, scope, trigger, navigate,
|
||||
location: drafts?.get('host', 'thumbnail_location') }) });
|
||||
imagesValue = metrics.children[2].querySelector('strong');
|
||||
thumbnailRow = thumbnails;
|
||||
backupRow = backup;
|
||||
sources = overview.sources;
|
||||
@@ -488,8 +517,19 @@ export function mountOverview(container, { api, activity, scope, navigate = navi
|
||||
const first = swap(loading());
|
||||
// D129: a finished Move library changes where the library lives.
|
||||
scope.cleanup(onMoveEnded(activity, load));
|
||||
if (counts) {
|
||||
scope.cleanup(counts.subscribe(paintImages));
|
||||
counts.freshen(['images']);
|
||||
}
|
||||
load();
|
||||
loadThumbnails();
|
||||
loadBackup();
|
||||
return first;
|
||||
// #320 D199: the unavailable thumbnail folder's notice, first on the page while it lasts.
|
||||
if (!drafts) return first;
|
||||
const unavailable = noticeSlot({ api, scope, notify, location: () => drafts.get('host', 'thumbnail_location'),
|
||||
onChange: () => navigate('#/system/sources'), onDone: () => drafts.reloadHost?.() });
|
||||
scope.cleanup(drafts.subscribe(() => unavailable.paint()));
|
||||
// The anchor goes in first, so the notice (inserted after it) leads the section once the anchor is mounted.
|
||||
unavailable.paint();
|
||||
return [unavailable.anchor, ...first];
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ import { createButton } from '../shared/toolkit/core.js';
|
||||
import { closeWorkspaceDialog, field, openContentDialog } from './workspace-dialogs.mjs';
|
||||
import { openMetadataScan } from './activity-scan.mjs';
|
||||
import { openThumbnailReview } from './thumbnail-review.mjs';
|
||||
import { locationBlock, noticeSlot } from './system-thumbnail-location.mjs';
|
||||
import { settingRow, operationRow } from './system-controls.mjs';
|
||||
import { HOST_UNAVAILABLE } from './system-appearance.mjs';
|
||||
import { showWorkspaceToast } from './toast.mjs';
|
||||
@@ -234,6 +235,19 @@ export function mountSources(container, { api, activity, scope, drafts, notify =
|
||||
nodes.push(notice);
|
||||
}
|
||||
|
||||
// #320 D199: the unavailable thumbnail folder's notice, first on the page while it lasts.
|
||||
const thumbnailLocation = () => drafts.get('host', 'thumbnail_location');
|
||||
const location = locationBlock({ api, store: activity, scope, location: thumbnailLocation, notify });
|
||||
const unavailable = noticeSlot({ api, scope, location: thumbnailLocation, notify, onChange: () => location.focus(),
|
||||
onDone: () => drafts.reloadHost?.() });
|
||||
const paintLocation = () => {
|
||||
location.paint();
|
||||
unavailable.paint();
|
||||
};
|
||||
scope.cleanup(drafts.subscribe(paintLocation));
|
||||
nodes.push(unavailable.anchor);
|
||||
paintLocation();
|
||||
|
||||
const surface = el('section', undefined, 'surface');
|
||||
const heading = el('div', undefined, 'panel-heading');
|
||||
const addButton = createButton('+ Add source');
|
||||
@@ -266,7 +280,10 @@ export function mountSources(container, { api, activity, scope, drafts, notify =
|
||||
operationRow({ scope, title: 'Enrich existing metadata', description: 'Resolve both prompts, deduplicate recipes, preserve uncertain links.',
|
||||
label: 'Review scan', onClick: scanAll }),
|
||||
operationRow({ scope, title: 'Thumbnail generation', description: 'Generate missing previews; rebuilding all is a separate explicit option.',
|
||||
label: 'Review options', onClick: trigger => openThumbnailReview({ api, store: activity, scope, trigger }) }),
|
||||
label: 'Review options', onClick: trigger => openThumbnailReview({ api, store: activity, scope, trigger,
|
||||
location: thumbnailLocation() }) }),
|
||||
// #320 D196: where previews are kept; never a draft (Move thumbnails is its only writer).
|
||||
location.node,
|
||||
// D98: the legacy gallery's Refresh, which rereads every source's file list.
|
||||
operationRow({ scope, title: 'Refresh files', description: 'Read every source again to find files added or removed outside ComfyUI.',
|
||||
label: 'Refresh', onClick: refresh }));
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
/**
|
||||
* Thumbnail location (#320, beyond the concept: D196–D199, approved static mock `320-mock`): where previews are kept.
|
||||
*
|
||||
* - D196 `locationBlock`: a `.setting-block` in System › Sources & scans after "Thumbnail generation": the radios
|
||||
* "Inside each source" / "One folder", a "Folder path" field (enabled with One folder, prefilled with the folder in
|
||||
* use or the server's `default_path`), the status line and "Review move…" (enabled when the choice differs from the
|
||||
* location in use). Not a host draft: the Save row never saves it; the Move thumbnails job is the only writer (D4B).
|
||||
* A refusal of the review shows in place under the field (`aria-invalid`), in the server's words.
|
||||
* - D197 `openMoveReview`: the shared review (Move library's shape) over the server's plan
|
||||
* (`POST /images/thumbnails/plan`): From, To, Thumbnails, Free space, Too long for Windows, Removed after the move,
|
||||
* Left in place, Links skipped; a folder that is not the default asks for the confirmation checkbox first. "Move
|
||||
* thumbnails" starts the job (`POST /jobs/thumbnails {mode: "move", token, confirmed}`) and opens it in Activity.
|
||||
* - D199 `locationNotice`: while the thumbnail folder is unavailable, a `.notice` on Overview and Sources & scans
|
||||
* says why, with "Recreate folder" (a missing folder, after a confirmation) or "Use this folder" (another folder's
|
||||
* marker), and "Change location".
|
||||
*
|
||||
* The location comes from GET /settings' `thumbnail_location` (`{mode, path, default_path, available, problem,
|
||||
* cleanup_pending}`), read with the host settings.
|
||||
*/
|
||||
import { createButton } from '../shared/toolkit/core.js';
|
||||
import { closeWorkspaceDialog, openContentDialog, openReviewDialog } from './workspace-dialogs.mjs';
|
||||
import { busyToast } from './activity-scan.mjs';
|
||||
import { showWorkspaceToast } from './toast.mjs';
|
||||
import { navigateTo } from './state.mjs';
|
||||
import { noun } from './plural.mjs';
|
||||
|
||||
export const PLAN_URL = '/prompt_manager/images/thumbnails/plan';
|
||||
export const RECREATE_URL = '/prompt_manager/images/thumbnails/location/recreate';
|
||||
export const ADOPT_URL = '/prompt_manager/images/thumbnails/location/adopt';
|
||||
export const LOCATION_HINT = 'Where previews are kept. Originals are never moved. Avoid cloud-synced folders.';
|
||||
export const SENTENCE = 'Every thumbnail is copied and checked before Prompt Manager switches. Scans, deletes and '
|
||||
+ 'tagging wait until it finishes; closing this page does not stop it. Originals are never touched.';
|
||||
export const REVIEW_FAILED = 'The move could not be reviewed. Try again.';
|
||||
export const START_FAILED = 'Move thumbnails could not start. Try again.';
|
||||
export const NOTICE_TITLE = 'The thumbnail folder is unavailable';
|
||||
const BACK = 'Originals are shown and no thumbnails are written until it is back.';
|
||||
const CHECK = 'If its drive is unplugged, connect it; Prompt Manager checks again every few seconds.';
|
||||
|
||||
const number = value => Number(value || 0).toLocaleString('en-US');
|
||||
const isFolder = location => location?.mode === 'folder';
|
||||
|
||||
/** "1.2 GB" (binary units, one decimal, as the server writes them). */
|
||||
export function sizeText(bytes) {
|
||||
let size = Number(bytes) || 0;
|
||||
for (const unit of ['B', 'KB', 'MB', 'GB']) {
|
||||
if (size < 1024 || unit === 'GB') return unit === 'B' ? `${size} B` : `${size.toFixed(1)} ${unit}`;
|
||||
size /= 1024;
|
||||
}
|
||||
return `${size.toFixed(1)} GB`;
|
||||
}
|
||||
|
||||
export const SUGGESTED = 'The suggested folder is next to your library; Review move… shows what happens before anything '
|
||||
+ 'is copied.';
|
||||
|
||||
/** The status line under the row; `choice` (`{mode, path}`) adds the suggestion while the default folder is chosen. */
|
||||
export function statusText(location, choice = null) {
|
||||
if (!location) return '';
|
||||
if (!isFolder(location) && choice?.mode === 'folder' && location.default_path
|
||||
&& choice.path?.trim() === location.default_path) {
|
||||
return `Now: inside each source. ${SUGGESTED}`;
|
||||
}
|
||||
let text = isFolder(location) ? `Now: one folder, ${location.path}` : 'Now: inside each source';
|
||||
if (isFolder(location) && location.available === false) return `${text} · unavailable. Originals are shown.`;
|
||||
text += '.';
|
||||
if (isFolder(location) && location.path && location.path === location.default_path) {
|
||||
text += ' Using the default folder next to your library.';
|
||||
}
|
||||
if (location.cleanup_pending > 0) {
|
||||
text += ` Old thumbnails in ${number(location.cleanup_pending)} ${noun(location.cleanup_pending, 'place')} wait to `
|
||||
+ 'be removed until it is connected.';
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
/** Whether the row's choice (`{mode, path}`) differs from the location in use (Review move… enabled). */
|
||||
export function choiceChanged(location, choice) {
|
||||
if (!location) return false;
|
||||
if (choice.mode !== 'folder') return isFolder(location) || Boolean(location.problem);
|
||||
const path = (choice.path || '').trim();
|
||||
return Boolean(path) && (!isFolder(location) || path !== location.path);
|
||||
}
|
||||
|
||||
/** The D197 review rows (`[term, value]`, a value may be `[lead, items]`) for the server's plan. */
|
||||
export function reviewRows(plan) {
|
||||
const totals = plan.totals ?? {};
|
||||
const sources = totals.sources ?? plan.sources?.length ?? 0;
|
||||
const inside = `Inside each source (${number(sources)} ${noun(sources, 'source')})`;
|
||||
const from = plan.from?.mode === 'folder' ? plan.from.path : inside;
|
||||
let to = inside;
|
||||
if (plan.to?.mode === 'folder') {
|
||||
const state = plan.to.default ? (plan.to.will_create ? 'new, next to your library' : 'next to your library')
|
||||
: (plan.to.will_create ? 'new folder' : 'empty folder');
|
||||
to = `${plan.to.path} (${state})`;
|
||||
}
|
||||
const rows = [['From', from], ['To', to],
|
||||
['Thumbnails', `${number(totals.files)} in ${number(sources)} ${noun(sources, 'source')} · ${sizeText(totals.bytes)}`]];
|
||||
if (Number.isFinite(plan.free_bytes)) rows.push(['Free space', `${sizeText(plan.free_bytes)} free`]);
|
||||
if (totals.too_long > 0) {
|
||||
const more = totals.too_long - (plan.too_long_listed?.length ?? 0);
|
||||
rows.push(['Too long for Windows', [`${number(totals.too_long)} ${noun(totals.too_long, 'thumbnail')} cannot be `
|
||||
+ 'stored there. They show the original instead; choose a shorter folder to keep them.',
|
||||
[...(plan.too_long_listed ?? []), ...(more > 0 ? [`and ${number(more)} more`] : [])]]]);
|
||||
}
|
||||
const removed = [];
|
||||
if (totals.orphans > 0) removed.push(`${number(totals.orphans)} ${noun(totals.orphans, 'thumbnail')} whose original is gone`);
|
||||
if (totals.temps > 0) removed.push(`${number(totals.temps)} unfinished temporary ${noun(totals.temps, 'file')}`);
|
||||
for (const source of plan.unlisted ?? []) {
|
||||
removed.push(`1 source no longer listed: ${number(source.files)} ${noun(source.files, 'thumbnail')} are removed; `
|
||||
+ 'they are made again if it returns');
|
||||
}
|
||||
const oldCopies = plan.from?.mode === 'folder'
|
||||
? 'The thumbnail folders of these sources inside it. The folder itself is removed once empty if Prompt Manager created it.'
|
||||
: 'The old copies in each source’s thumbnails folder';
|
||||
rows.push(['Removed after the move', [removed.length ? `${oldCopies}, and:` : `${oldCopies}.`, removed]]);
|
||||
if (totals.left_in_place > 0) {
|
||||
rows.push(['Left in place', [`${number(totals.left_in_place)} ${noun(totals.left_in_place, 'file')} that `
|
||||
+ `${totals.left_in_place === 1 ? 'is' : 'are'} not ${totals.left_in_place === 1 ? 'a thumbnail' : 'thumbnails'}:`,
|
||||
plan.left_listed ?? []]]);
|
||||
}
|
||||
if (totals.links > 0) {
|
||||
rows.push(['Links skipped', `${number(totals.links)} ${noun(totals.links, 'link')} inside a thumbnails folder `
|
||||
+ `${totals.links === 1 ? 'is' : 'are'} not followed`]);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** The notice text for an unavailable `location` (D199). */
|
||||
export function noticeText(location) {
|
||||
const where = location?.path || 'The thumbnail folder';
|
||||
switch (location?.problem) {
|
||||
case 'missing': return `${where} is missing. ${BACK} ${CHECK}`;
|
||||
case 'unreachable': return `${where} can’t be reached. ${BACK} ${CHECK}`;
|
||||
case 'marker-mismatch': return `${where} holds another thumbnail folder. ${BACK} Use it if it is yours, or change `
|
||||
+ 'the location.';
|
||||
case 'invalid': return `The thumbnail location in config.json can’t be used. ${BACK} Change the location.`;
|
||||
default: return `${where} is being checked. ${BACK}`;
|
||||
}
|
||||
}
|
||||
|
||||
function el(tag, text, className) {
|
||||
const node = document.createElement(tag);
|
||||
if (text !== undefined) node.textContent = text;
|
||||
if (className) node.className = className;
|
||||
return node;
|
||||
}
|
||||
|
||||
function radio(value, label, hint) {
|
||||
const node = el('label');
|
||||
const input = el('input');
|
||||
input.type = 'radio';
|
||||
input.name = 'thumb-location';
|
||||
input.value = value;
|
||||
input.id = `loc-${value}`;
|
||||
node.append(input, ` ${label} `, el('span', hint, 'small'));
|
||||
return { node, input };
|
||||
}
|
||||
|
||||
/** Start the move with the plan's `token`; opens it in Activity, or says why not. */
|
||||
export async function startMove({ store, scope, token, confirmed, notify = showWorkspaceToast, navigate = navigateTo }) {
|
||||
try {
|
||||
const result = await store.start('thumbnails', { mode: 'move', token, confirmed });
|
||||
if (scope.signal.aborted) return null;
|
||||
if (result.busy !== undefined) {
|
||||
notify(busyToast(result.busy));
|
||||
return null;
|
||||
}
|
||||
navigate(`#/activity/${result.job.id}`);
|
||||
return result.job;
|
||||
} catch (error) {
|
||||
if (!scope.signal.aborted) notify(typeof error?.code === 'string' && error.message ? error.message : START_FAILED);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Open the D197 review over `plan`; a plan the server refused for space keeps Move thumbnails unavailable. */
|
||||
export function openMoveReview({ scope, store, plan, refusal = null, trigger = null, notify, navigate }) {
|
||||
const options = [];
|
||||
let box = null;
|
||||
if (plan.confirm_required && !refusal) {
|
||||
const label = el('label', undefined, 'review-confirm');
|
||||
box = el('input');
|
||||
box.type = 'checkbox';
|
||||
box.id = 'review-confirmed';
|
||||
label.append(box, el('span', `Prompt Manager may create its own folders and a .promptmanager-thumbnails.json file in ${plan.to.path}.`));
|
||||
options.push(label);
|
||||
}
|
||||
if (refusal) options.push(el('p', refusal, 'small review-refusal'));
|
||||
const review = openReviewDialog({ scope, trigger, title: 'Move thumbnails', grid: reviewRows(plan), options,
|
||||
sentence: refusal ? null : SENTENCE, cancelLabel: 'Cancel', primary: { label: 'Move thumbnails' },
|
||||
onStart: () => { startMove({ store, scope, token: plan.token, confirmed: Boolean(box?.checked), notify, navigate }); } });
|
||||
const locked = () => Boolean(refusal) || (box !== null && !box.checked);
|
||||
review.setPrimary({ label: 'Move thumbnails', disabled: locked() });
|
||||
if (box) scope.listen(box, 'change', () => review.setPrimary({ label: 'Move thumbnails', disabled: locked() }));
|
||||
return review;
|
||||
}
|
||||
|
||||
/**
|
||||
* The D196 row; `location()` gives the server's `thumbnail_location` (undefined while read). Returns `{node, paint,
|
||||
* focus}`; `paint()` follows a new location.
|
||||
*/
|
||||
export function locationBlock({ api, store, scope, location, notify = showWorkspaceToast, navigate = navigateTo }) {
|
||||
const block = el('div', undefined, 'setting-block');
|
||||
block.id = 'location-block';
|
||||
const text = el('div', undefined, 'setting-block-text');
|
||||
const label = el('span', 'Thumbnail location', 'setting-block-label');
|
||||
label.id = 'location-label';
|
||||
text.append(label, el('p', LOCATION_HINT));
|
||||
const choice = el('div', undefined, 'location-choice');
|
||||
choice.setAttribute('role', 'radiogroup');
|
||||
choice.setAttribute('aria-labelledby', 'location-label');
|
||||
const source = radio('source', 'Inside each source', '<source>/thumbnails');
|
||||
const folder = radio('folder', 'One folder', 'one subfolder per source');
|
||||
choice.append(source.node, folder.node);
|
||||
const pathField = el('div', undefined, 'location-path field');
|
||||
const pathLabel = el('label', 'Folder path');
|
||||
pathLabel.htmlFor = 'location-path';
|
||||
const input = el('input');
|
||||
input.id = 'location-path';
|
||||
input.autocomplete = 'off';
|
||||
input.spellcheck = false;
|
||||
const problem = el('p', '', 'small location-problem');
|
||||
problem.id = 'location-problem';
|
||||
problem.setAttribute('role', 'alert');
|
||||
pathField.append(pathLabel, input, problem);
|
||||
const status = el('p', '', 'small location-status');
|
||||
status.id = 'location-status';
|
||||
status.setAttribute('role', 'status');
|
||||
const actions = el('div', undefined, 'row-actions');
|
||||
const review = createButton('Review move…');
|
||||
review.id = 'review-move';
|
||||
actions.append(review);
|
||||
block.append(text, choice, pathField, status, actions);
|
||||
|
||||
let shown;
|
||||
const setProblem = message => {
|
||||
problem.textContent = message || '';
|
||||
pathField.setAttribute('aria-invalid', String(Boolean(message)));
|
||||
};
|
||||
const current = () => ({ mode: folder.input.checked ? 'folder' : 'source', path: input.value });
|
||||
const repaintButton = () => {
|
||||
const now = location();
|
||||
input.disabled = !folder.input.checked || !now;
|
||||
source.input.disabled = folder.input.disabled = !now;
|
||||
const changed = choiceChanged(now, current());
|
||||
status.textContent = now ? statusText(now, current()) : '';
|
||||
review.setAttribute('aria-disabled', String(!changed));
|
||||
review.classList.toggle('ca-primary', changed);
|
||||
};
|
||||
const paint = () => {
|
||||
const now = location();
|
||||
if (now && now !== shown) {
|
||||
shown = now;
|
||||
(isFolder(now) ? folder : source).input.checked = true;
|
||||
input.value = (isFolder(now) ? now.path : now.default_path) || '';
|
||||
setProblem(null);
|
||||
}
|
||||
repaintButton();
|
||||
};
|
||||
for (const control of [source.input, folder.input]) {
|
||||
scope.listen(control, 'change', () => {
|
||||
if (folder.input.checked && !input.value) input.value = location()?.default_path || '';
|
||||
setProblem(null);
|
||||
repaintButton();
|
||||
});
|
||||
}
|
||||
scope.listen(input, 'input', () => { setProblem(null); repaintButton(); });
|
||||
scope.listen(review, 'click', async () => {
|
||||
if (review.getAttribute('aria-disabled') === 'true') return;
|
||||
const chosen = current();
|
||||
const body = { mode: 'move', location: chosen.mode === 'folder' ? { mode: 'folder', path: chosen.path.trim() }
|
||||
: { mode: 'source' } };
|
||||
review.setAttribute('aria-disabled', 'true');
|
||||
try {
|
||||
const plan = await api.post(PLAN_URL, body, { signal: scope.signal });
|
||||
if (!scope.signal.aborted) openMoveReview({ scope, store, plan, trigger: review, notify, navigate });
|
||||
} catch (error) {
|
||||
if (scope.signal.aborted) return;
|
||||
if (error?.status === 507 && error?.plan) {
|
||||
openMoveReview({ scope, store, plan: error.plan, refusal: error.plan.insufficient?.text || error.message,
|
||||
trigger: review });
|
||||
} else if (error?.status === 409 && error?.code === 'busy') {
|
||||
notify(busyToast(error.running ?? null));
|
||||
} else if (typeof error?.code === 'string' && error.code.startsWith('location-') && error.message) {
|
||||
setProblem(error.message);
|
||||
input.focus();
|
||||
} else {
|
||||
notify(typeof error?.message === 'string' && error?.code ? error.message : REVIEW_FAILED);
|
||||
}
|
||||
} finally {
|
||||
repaintButton();
|
||||
}
|
||||
});
|
||||
paint();
|
||||
return {
|
||||
node: block,
|
||||
paint,
|
||||
focus() {
|
||||
folder.input.checked = true;
|
||||
repaintButton();
|
||||
input.focus();
|
||||
block.scrollIntoView?.({ block: 'center' });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Open the Recreate confirmation for `location`; resolves once it closed. */
|
||||
export function openRecreate({ api, scope, location, trigger = null, onDone, notify = showWorkspaceToast }) {
|
||||
const grid = el('dl', undefined, 'meta-grid');
|
||||
for (const [term, value] of [['Folder', location.path], ['Made', 'An empty folder with a new .promptmanager-thumbnails.json'],
|
||||
['Thumbnails', 'Made again as you browse, or with Thumbnail maintenance']]) {
|
||||
const entry = el('div');
|
||||
entry.append(el('dt', term), el('dd', value));
|
||||
grid.append(entry);
|
||||
}
|
||||
const why = el('p', '', 'small location-problem');
|
||||
const cancel = createButton('Cancel');
|
||||
const go = createButton('Recreate folder', 'ca-primary');
|
||||
const actions = el('div', undefined, 'dialog-actions');
|
||||
actions.append(cancel, go);
|
||||
scope.listen(cancel, 'click', () => closeWorkspaceDialog());
|
||||
scope.listen(go, 'click', async () => {
|
||||
try {
|
||||
const result = await api.post(RECREATE_URL, {}, { signal: scope.signal });
|
||||
if (scope.signal.aborted) return;
|
||||
closeWorkspaceDialog();
|
||||
onDone?.(result.location);
|
||||
} catch (error) {
|
||||
if (scope.signal.aborted) return;
|
||||
why.textContent = error?.message || 'The folder could not be recreated.';
|
||||
if (!error?.message) notify(why.textContent);
|
||||
}
|
||||
});
|
||||
openContentDialog({ scope, title: 'Recreate the thumbnail folder?', trigger, nodes: [grid,
|
||||
el('p', 'If the folder is on a drive that is not connected, connect it instead: recreating would put thumbnails on '
|
||||
+ 'another disk.', 'small'), why, actions] });
|
||||
}
|
||||
|
||||
/**
|
||||
* The D199 notice for `location` (null while it is available): Recreate folder for a missing folder, Use this folder
|
||||
* for another folder's marker, and Change location (`onChange`).
|
||||
*/
|
||||
export function locationNotice({ api, scope, location, onChange, onDone, notify = showWorkspaceToast }) {
|
||||
if (!isFolder(location) || location.available !== false) return null;
|
||||
const notice = el('div', undefined, 'notice');
|
||||
notice.id = 'unavailable-notice';
|
||||
const text = el('div');
|
||||
text.append(el('h2', NOTICE_TITLE), el('p', noticeText(location)));
|
||||
const actions = el('div', undefined, 'row-actions');
|
||||
if (location.problem === 'missing') {
|
||||
const recreate = createButton('Recreate folder');
|
||||
scope.listen(recreate, 'click', () => openRecreate({ api, scope, location, trigger: recreate, onDone, notify }));
|
||||
actions.append(recreate);
|
||||
} else if (location.problem === 'marker-mismatch') {
|
||||
const adopt = createButton('Use this folder');
|
||||
scope.listen(adopt, 'click', async () => {
|
||||
try {
|
||||
const result = await api.post(ADOPT_URL, {}, { signal: scope.signal });
|
||||
if (!scope.signal.aborted) onDone?.(result.location);
|
||||
} catch (error) {
|
||||
if (!scope.signal.aborted) notify(error?.message || 'The folder could not be used.');
|
||||
}
|
||||
});
|
||||
actions.append(adopt);
|
||||
}
|
||||
const change = createButton('Change location', 'ca-primary');
|
||||
scope.listen(change, 'click', () => onChange?.());
|
||||
actions.append(change);
|
||||
notice.append(text, actions);
|
||||
return notice;
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a section shows the D199 notice: a comment `anchor` (no element, so the concept comparisons see nothing while the
|
||||
* folder is available) after which `paint()` keeps the notice for `location()`.
|
||||
*/
|
||||
export function noticeSlot({ api, scope, location, onChange, onDone, notify = showWorkspaceToast }) {
|
||||
const anchor = document.createComment('thumbnail-location-notice');
|
||||
let shownFor;
|
||||
let notice = null;
|
||||
return {
|
||||
anchor,
|
||||
paint() {
|
||||
const now = location();
|
||||
// Not mounted yet: the next paint (a microtask, or the host read) places it.
|
||||
if (!anchor.parentNode) {
|
||||
queueMicrotask(() => { if (anchor.parentNode && !scope.signal.aborted) this.paint(); });
|
||||
return;
|
||||
}
|
||||
if (now === shownFor) return;
|
||||
shownFor = now;
|
||||
notice?.remove();
|
||||
notice = locationNotice({ api, scope, location: now, onChange, onDone, notify });
|
||||
if (notice) anchor.after(notice);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -86,7 +86,9 @@ export function systemDrafts(api) {
|
||||
api.get(LORA_STATUS_URL, { cache: 'no-store' }).catch(() => null)]);
|
||||
// The store reads a missing value as none and a missing revision as unreadable (no host save). #317: which
|
||||
// thumbnail formats the server can write rides along as a read-only value (never drafted or sent).
|
||||
return { values: { ...result.settings, thumbnail_encoders: result.thumbnail_encoders, ...(lora ? loraValues(lora) : {}) },
|
||||
// #320: where previews live rides along the same way (read-only; only Move thumbnails changes it).
|
||||
return { values: { ...result.settings, thumbnail_encoders: result.thumbnail_encoders,
|
||||
thumbnail_location: result.thumbnail_location, ...(lora ? loraValues(lora) : {}) },
|
||||
revision: result.revision };
|
||||
},
|
||||
async writeHost(changes, { expected_revision }) {
|
||||
@@ -135,7 +137,7 @@ export const DELIVERED_SECTIONS = Object.freeze({
|
||||
/**
|
||||
* Mount System under the heading `renderHeading` wrote in `main`. Returns `{show(section)}`, which shows another
|
||||
* section in place. `scope` is the page's view scope; `sections` maps a section id to a delivered section's mount
|
||||
* (`(container, {api, activity, scope, drafts}) => nodes`) for every section. `activity` is the page's Activity store, which follows the jobs a section starts.
|
||||
* (`(container, {api, activity, scope, drafts, counts}) => nodes`; `counts` is the shell's sidebar counts, #555) for every section. `activity` is the page's Activity store, which follows the jobs a section starts.
|
||||
*
|
||||
* The host values are read on mount (and by the store again after a conflict), so a save carries the current revision;
|
||||
* until a read returned one, a host save fails without a request (system-state.mjs). The browser values are read on
|
||||
@@ -159,7 +161,8 @@ function legacyPagesLine() {
|
||||
return line;
|
||||
}
|
||||
|
||||
export function mountSystem(main, api, scope, { section, activity, sections = DELIVERED_SECTIONS, navigate = navigateTo }) {
|
||||
export function mountSystem(main, api, scope, { section, activity, counts = null, sections = DELIVERED_SECTIONS,
|
||||
navigate = navigateTo }) {
|
||||
const store = systemDrafts(api);
|
||||
jobs = activity;
|
||||
store.reloadHost();
|
||||
@@ -208,7 +211,7 @@ export function mountSystem(main, api, scope, { section, activity, sections = DE
|
||||
for (const button of nav.children) button.setAttribute('aria-pressed', String(button.dataset.value === next));
|
||||
while (layout.firstChild !== saveRow) layout.firstChild.remove();
|
||||
// Every section is delivered (S12), and the router only names known sections (state.mjs).
|
||||
const nodes = sections[next](layout, { api, activity, scope: sectionScope, drafts: store });
|
||||
const nodes = sections[next](layout, { api, activity, scope: sectionScope, drafts: store, counts });
|
||||
layout.prepend(...nodes);
|
||||
if (next === 'overview') main.append(legacyPages);
|
||||
else legacyPages.remove();
|
||||
|
||||
@@ -23,6 +23,7 @@ import { busyToast } from './activity-scan.mjs';
|
||||
import { showWorkspaceToast } from './toast.mjs';
|
||||
import { navigateTo } from './state.mjs';
|
||||
import { noun } from './plural.mjs';
|
||||
import { sizeText } from './system-thumbnail-location.mjs';
|
||||
|
||||
export const THUMBNAIL_PLAN_URL = '/prompt_manager/images/thumbnails/plan';
|
||||
export const THUMBNAILS_KIND = 'thumbnails';
|
||||
@@ -33,7 +34,9 @@ export const PLAN_FAILED = 'The previews could not be counted.';
|
||||
export const START_FAILED = 'Thumbnail generation could not start. Try again.';
|
||||
export const SETTINGS_UNKNOWN = 'Previews follow the thumbnail settings in System › Appearance.';
|
||||
const FORMAT_LABELS = Object.freeze({ webp: 'WebP', jxl: 'JPEG XL', jpeg: 'JPEG' });
|
||||
const MODES = new Set(SCOPE_OPTIONS.map(([value]) => value));
|
||||
// #320 D200: offered only while previews are kept in one folder (31A).
|
||||
export const PRUNE_OPTION = Object.freeze(['prune', 'Remove thumbnails of removed sources']);
|
||||
const MODES = new Set([...SCOPE_OPTIONS.map(([value]) => value), PRUNE_OPTION[0]]);
|
||||
|
||||
const count = value => value.toLocaleString('en-US');
|
||||
const isCount = value => Number.isSafeInteger(value) && value >= 0;
|
||||
@@ -47,6 +50,7 @@ export function settingsText(plan) {
|
||||
|
||||
/** D99: the plan line for the server's `{mode, targets, fresh, remaining}`; an answer without counts failed. */
|
||||
export function planText(plan) {
|
||||
if (plan?.mode === 'prune') return typeof plan.line === 'string' && plan.line ? plan.line : PLAN_FAILED;
|
||||
if (!plan || !MODES.has(plan.mode) || ![plan.targets, plan.fresh, plan.remaining].every(isCount)) {
|
||||
return PLAN_FAILED;
|
||||
}
|
||||
@@ -101,11 +105,18 @@ function select(options, value) {
|
||||
* Activity store; `trigger` gets focus back when the dialog closes.
|
||||
*/
|
||||
export function openThumbnailReview({ api, store, scope, trigger = null, mode: initial = 'missing', notify = showWorkspaceToast,
|
||||
navigate = navigateTo }) {
|
||||
const mode = select(SCOPE_OPTIONS, MODES.has(initial) ? initial : 'missing');
|
||||
navigate = navigateTo, location = null }) {
|
||||
const folderMode = location?.mode === 'folder';
|
||||
const options = folderMode ? [...SCOPE_OPTIONS, PRUNE_OPTION] : SCOPE_OPTIONS;
|
||||
const allowed = new Set(options.map(([value]) => value));
|
||||
const mode = select(options, allowed.has(initial) ? initial : 'missing');
|
||||
const line = el('p', PLAN_LOADING, 'small');
|
||||
line.setAttribute('role', 'status');
|
||||
const settings = el('p', SETTINGS_UNKNOWN, 'small');
|
||||
// D200: what the prune removes, under the plan line.
|
||||
const pruneDetail = el('p', '', 'small');
|
||||
const pruneGrid = el('dl', undefined, 'meta-grid');
|
||||
pruneDetail.hidden = pruneGrid.hidden = true;
|
||||
const cancel = createButton('Cancel');
|
||||
const start = createButton('Start', 'ca-primary');
|
||||
const actions = el('div', undefined, 'dialog-actions');
|
||||
@@ -125,7 +136,21 @@ export function openThumbnailReview({ api, store, scope, trigger = null, mode: i
|
||||
// A later choice painted its own line meanwhile, or the dialog closed.
|
||||
if (!open || scope.signal.aborted || mode.value !== chosen) return;
|
||||
line.textContent = planText(plan);
|
||||
settings.textContent = settingsText(plan);
|
||||
const prune = chosen === 'prune';
|
||||
settings.hidden = prune;
|
||||
pruneDetail.hidden = pruneGrid.hidden = !prune || !plan?.sources?.length;
|
||||
if (prune) {
|
||||
pruneDetail.textContent = `Only Prompt Manager’s thumbnails in their folders inside ${plan?.folder ?? 'the thumbnail '
|
||||
+ 'folder'} are removed. If a source comes back, its thumbnails are made again as you browse.`;
|
||||
pruneGrid.replaceChildren(...(plan?.sources ?? []).map(source => {
|
||||
const entry = el('div');
|
||||
entry.append(el('dt', source.key), el('dd', `${count(source.files)} ${noun(source.files, 'thumbnail')} · `
|
||||
+ `${sizeText(source.bytes)}`));
|
||||
return entry;
|
||||
}));
|
||||
} else {
|
||||
settings.textContent = settingsText(plan);
|
||||
}
|
||||
}
|
||||
|
||||
scope.listen(mode, 'change', paint);
|
||||
@@ -136,7 +161,7 @@ export function openThumbnailReview({ api, store, scope, trigger = null, mode: i
|
||||
startThumbnails({ store, scope, options, notify, navigate });
|
||||
});
|
||||
openContentDialog({ scope, title: 'Thumbnail maintenance', trigger,
|
||||
nodes: [field('Scope', mode, 'thumb-mode'), line, settings, actions],
|
||||
nodes: [field('Scope', mode, 'thumb-mode'), line, settings, ...(folderMode ? [pruneDetail, pruneGrid] : []), actions],
|
||||
onClose: () => { open = false; } });
|
||||
paint();
|
||||
}
|
||||
|
||||
@@ -460,9 +460,13 @@ export function backupNotice({ copy, onBackup }) {
|
||||
return notice;
|
||||
}
|
||||
|
||||
/** A grid value: text, or (#320) `[lead, items]`, a lead sentence and the list under it. */
|
||||
const gridValue = value => typeof value === 'string' || (Array.isArray(value) && value.length === 2
|
||||
&& typeof value[0] === 'string' && Array.isArray(value[1]) && value[1].every(item => typeof item === 'string'));
|
||||
|
||||
function requireGrid(grid) {
|
||||
if (!Array.isArray(grid) || grid.length === 0 || grid.some(row => !Array.isArray(row) || row.length !== 2 ||
|
||||
typeof row[0] !== 'string' || !row[0] || typeof row[1] !== 'string')) {
|
||||
typeof row[0] !== 'string' || !row[0] || !gridValue(row[1]))) {
|
||||
throw new TypeError('The review needs its [term, value] grid');
|
||||
}
|
||||
}
|
||||
@@ -495,7 +499,12 @@ export function openReviewDialog({ scope, title, heading = null, grid, options =
|
||||
values.clear();
|
||||
list.replaceChildren(...rows.map(([term, value]) => {
|
||||
const entry = el('div');
|
||||
const dd = el('dd', value);
|
||||
const dd = el('dd', typeof value === 'string' ? value : value[0]);
|
||||
if (typeof value !== 'string' && value[1].length) {
|
||||
const items = el('ul', undefined, 'review-list');
|
||||
items.append(...value[1].map(item => el('li', item)));
|
||||
dd.append(items);
|
||||
}
|
||||
values.set(term, dd);
|
||||
entry.append(el('dt', term), dd);
|
||||
return entry;
|
||||
|
||||
@@ -416,6 +416,9 @@ pre.prompt-text { font: inherit; font-size: 12px; line-height: 1.75; margin-bloc
|
||||
.prompt-sender-preview { margin-block: 20px; min-inline-size: 0; /* D48 */ }
|
||||
.prompt-sender-field { border: 1px solid var(--ca-line); border-radius: 7px; padding: 14px; margin-block: 12px; min-inline-size: 0; /* D48 */ }
|
||||
.prompt-sender-field :where(pre, textarea), .prompt-sender-composition { white-space: pre-wrap; overflow-wrap: anywhere; font: inherit; font-size: 12px; line-height: 1.65; max-inline-size: 100%; /* D48 */ }
|
||||
/* D191 (#542): each node of "All eligible nodes" reviews under its own heading. */
|
||||
.prompt-sender-node { border-block-start: 1px solid var(--ca-line); padding-block-start: 14px; margin-block: 18px; min-inline-size: 0; /* D191 */ }
|
||||
.prompt-sender-node > h4 { font-size: 15px; /* D191 */ }
|
||||
.prompt-sender-field textarea { inline-size: 100%; min-block-size: 100px; resize: vertical; /* D48 */ }
|
||||
.prompt-sender-composition { padding: 12px; background: var(--ca-field); border-radius: 6px; /* D48 */ }
|
||||
.prompt-sender-actions { justify-content: flex-end; margin-block-start: 18px; /* D48 */ }
|
||||
@@ -573,3 +576,32 @@ pre.prompt-text { font: inherit; font-size: 12px; line-height: 1.75; margin-bloc
|
||||
.review-panel { position: sticky; inset-block-start: 20px; }
|
||||
@media (max-width: 700px) { .review-panel { position: static; } }
|
||||
@media (min-width: 701px) and (max-height: 720px) { .review-panel { max-block-size: calc(100dvh - 40px); overflow: auto; } }
|
||||
|
||||
/* #320 D196: Thumbnail location, a setting wider than a .setting-row column. */
|
||||
.setting-block { padding: 17px 0; border-top: 1px solid var(--line); display: grid; gap: 12px; }
|
||||
.setting-block-text .setting-block-label { font-size: 12px; font-weight: 500; display: block; }
|
||||
.setting-block-text p { font-size: 11px; color: var(--ca-muted); margin-top: 4px; max-width: 640px; }
|
||||
.location-choice { display: flex; flex-wrap: wrap; gap: 8px 24px; font-size: 12px; }
|
||||
.location-choice label { display: inline-flex; gap: 6px; align-items: baseline; }
|
||||
.location-choice .small { color: var(--ca-muted); }
|
||||
.location-path { max-width: 640px; margin: 0; }
|
||||
.location-path input { width: 100%; min-width: 0; font-family: var(--ca-mono, ui-monospace, monospace); font-size: 12px; }
|
||||
.location-path input:disabled { color: var(--ca-muted); }
|
||||
.location-path[aria-invalid="true"] input { border-color: var(--red); }
|
||||
.location-problem { color: var(--red); margin-top: 6px; }
|
||||
.location-problem:empty { display: none; }
|
||||
.location-status { margin: 0; overflow-wrap: anywhere; }
|
||||
.setting-block .row-actions { justify-content: flex-start; }
|
||||
|
||||
/* #320 D197: the move review's lists and confirmation. */
|
||||
.review-list { margin: 4px 0 0; padding-inline-start: 18px; font-size: 11px; color: var(--ca-muted); }
|
||||
.review-list li { overflow-wrap: anywhere; }
|
||||
.review-confirm { display: flex; gap: 8px; align-items: flex-start; font-size: 12px; margin: 12px 0 0;
|
||||
padding: 10px 12px; border: 1px solid var(--line); border-radius: 6px; background: var(--ca-field); }
|
||||
.review-confirm input { margin-top: 2px; flex-shrink: 0; }
|
||||
.review-refusal { color: var(--red); }
|
||||
.meta-grid dd { overflow-wrap: anywhere; }
|
||||
|
||||
@media (max-width: 700px) {
|
||||
.location-choice { flex-direction: column; }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user