Compare commits
19
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4afb042b07 | ||
|
|
62815c9e3e | ||
|
|
a1ec65442d | ||
|
|
e52cc39cd5 | ||
|
|
eff6ece6a4 | ||
|
|
d02a911748 | ||
|
|
bc3f09d844 | ||
|
|
e0b86712a8 | ||
|
|
66ca988d2a | ||
|
|
20dbc2df21 | ||
|
|
dd247e3b6f | ||
|
|
c416c84347 | ||
|
|
599dd828e6 | ||
|
|
a4a7b7129a | ||
|
|
f334004f8f | ||
|
|
0126705045 | ||
|
|
652458e995 | ||
|
|
89d94cdb2b | ||
|
|
b6e0439421 |
@@ -21,15 +21,16 @@ env:
|
||||
PYTHONFAULTHANDLER: "1"
|
||||
|
||||
# Time limit rule as in test.yml: ceil(1.5 x the slowest of three recent green runs on main) + 3
|
||||
# minutes, never under 5. Measured 2026-10-26 in test.yml on runs 40257217681 (3be0ada),
|
||||
# 40240706568 (fb9134f), 40228167328 (f9835aa), before these legs moved here in #420:
|
||||
# slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
# minutes, never under 5, timed as whole-job wall time. One matrix job, so the slowest leg of each
|
||||
# run counts (tests/vendor/ci-workflow.test.mjs recomputes it, #493).
|
||||
# Measured 2026-10-29 on main runs 40935337099 (dd247e3), 40938407252 (dd247e3), 40941240624 (dd247e3):
|
||||
# slowest leg: 7m24s (py3.12 windows) / 7m31s (py3.13 windows) / 7m32s (py3.14 windows) -> 15 min
|
||||
# (windows 4m56s-7m32s, macos 4m39s-6m14s)
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 27
|
||||
timeout-minutes: 15
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -51,5 +52,27 @@ jobs:
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
# The runner's TEMP is on C:, the OS disk, where a flush took about 6 ms
|
||||
# and a test database's create-to-delete about 111 ms; on the D: work
|
||||
# disk the same took 0.5 ms and 23 ms (macOS: 19 ms). Tests write their
|
||||
# databases under TEMP, so it moves to a folder on D: (measured in #487,
|
||||
# run 40915866444). The runner's TEMP was an 8.3 short name
|
||||
# (C:\Users\RUNNER~1\...), so the new one is spelled by its short name too
|
||||
# and paths still have to be resolved. D: makes no short names by default
|
||||
# (run 40922502209), so they are switched on for it first.
|
||||
- name: Keep test files on the work disk (Windows)
|
||||
if: runner.os == 'Windows'
|
||||
shell: pwsh
|
||||
run: |
|
||||
fsutil 8dot3name set D: 0
|
||||
$folder = New-Item -ItemType Directory -Force -Path (Join-Path $env:RUNNER_TEMP "PromptManagerTests")
|
||||
$short = (New-Object -ComObject Scripting.FileSystemObject).GetFolder($folder.FullName).ShortPath
|
||||
Write-Output "test TEMP: $short (long name $($folder.FullName))"
|
||||
if ($short -eq $folder.FullName) { Write-Output "::warning::D: gave the test TEMP no 8.3 short name" }
|
||||
"TEMP=$short" >> $env:GITHUB_ENV
|
||||
"TMP=$short" >> $env:GITHUB_ENV
|
||||
|
||||
- name: Run tests
|
||||
env:
|
||||
PM_TEST_SLOWEST: "40"
|
||||
run: python -m tests.run_suite -v
|
||||
|
||||
@@ -28,8 +28,8 @@ env:
|
||||
# py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
# 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs.
|
||||
# Windows and macOS moved to portability.yml (#420), which keeps this 27-minute limit; this
|
||||
# Linux-only matrix keeps it too until it is re-measured.
|
||||
# Windows and macOS moved to portability.yml (#420), which has its own measured limit (#493);
|
||||
# this Linux-only matrix keeps 27 minutes until it is re-measured.
|
||||
# real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min
|
||||
# coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min
|
||||
# perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s,
|
||||
|
||||
@@ -36,6 +36,11 @@ concurrency:
|
||||
# 12m53s, node units and the viewer suite before 11.8m of workflows), so coverage is the tight one
|
||||
# in its 18 minutes. This holds only when the npm cache hits: a cold cache adds up to 5m to npm ci,
|
||||
# and then the job limit can still end the job before the step limit does.
|
||||
# #473: firefox/webkit shard 1/2 reached its 18-minute step limit (queue run 40866252975: webkit 622 tests
|
||||
# passed in 17m56s, 48.5 worker-minutes vs 28.0 in shard 2/2; recent shard 1/2 jobs 13.7m-18.9m, shard 2/2
|
||||
# 7.3m-12.2m), because --shard splits by test count and the slow concept specs sort into shard 1. The page
|
||||
# workflows now run in three shards per browser with the limits unchanged; the per-shard times are
|
||||
# re-measured after three green main runs.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
@@ -175,7 +180,7 @@ jobs:
|
||||
retention-days: 14
|
||||
|
||||
browser-workflows:
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/3)
|
||||
# Merge queue, main and manual runs only (#420); pull requests run the Chromium/touch coverage shards.
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
@@ -194,7 +199,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: [firefox, webkit]
|
||||
shard: [1, 2]
|
||||
shard: [1, 2, 3]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -221,9 +226,9 @@ jobs:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/3
|
||||
timeout-minutes: 18
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/3
|
||||
# Traces and the blob report (test-results/blob/) of a failed, timed-out or cancelled step.
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
|
||||
+244
-20
@@ -251,12 +251,29 @@ the process exits non-zero.
|
||||
stacks.
|
||||
- When the watchdog fires under coverage, that run's coverage data is lost; the
|
||||
job fails anyway and the dump is what matters.
|
||||
- `PM_TEST_SLOWEST=N` (off by default) lists the N slowest tests after the
|
||||
summary, each timed from its `start:` line to the next one like the hang
|
||||
limit. The Windows and macOS jobs (`portability.yml`) set it to 40, so every
|
||||
run shows where its time went (#487). The list prints only when the run
|
||||
ends: a job GitHub cancels or a watchdog exit prints none, so read the
|
||||
timestamps between `start:` lines instead.
|
||||
- On the Windows runners `TEMP` sits on the OS disk C:, where a flush took about
|
||||
6 ms and a test database's create-to-delete about 111 ms (D: 0.5 ms and 23 ms,
|
||||
macOS 19 ms). `portability.yml` points `TEMP`/`TMP` at a folder under
|
||||
`RUNNER_TEMP` on D: before the suite. It switches 8.3 short names on for D:
|
||||
and spells the folder by its short name, as the runner's own TEMP was
|
||||
(`C:\Users\RUNNER~1\...`), logs it, and warns when there is none.
|
||||
A test that waits on a fixed busy timeout, a network name (`\\server\share`:
|
||||
use `\\localhost\<missing share>`) or a default TLS context (it reads the
|
||||
Windows certificate store) costs seconds there; the slowest list shows it.
|
||||
|
||||
Every job in `test.yml`, `code-quality.yml` and `viewerjs-check.yml` has
|
||||
`timeout-minutes` (`tests/vendor/ci-workflow.test.mjs` checks it). The rule is
|
||||
ceil(1.5 x the slowest of three recent green runs on `main`) + 3 minutes, never
|
||||
under 5; the three run ids, their times and the resulting limits are written
|
||||
in a comment above `jobs:`. The 3 minutes leave room for one watchdog firing, so
|
||||
Every job in `test.yml`, `code-quality.yml`, `viewerjs-check.yml` and
|
||||
`portability.yml` has `timeout-minutes` (`tests/vendor/ci-workflow.test.mjs`
|
||||
checks it). The rule is ceil(1.5 x the slowest of three recent green runs on
|
||||
`main`) + 3 minutes, never under 5, timed as whole-job wall time; the three run
|
||||
ids, their times and the resulting limits are written in a comment above
|
||||
`jobs:`. For `portability.yml` the tooling test recomputes the limit from that
|
||||
comment (#493); its one matrix job takes the slowest Windows or macOS leg. The 3 minutes leave room for one watchdog firing, so
|
||||
the dump prints before GitHub cancels the job. A job that hits its limit, or a
|
||||
watchdog dump, is a failure like any other: read the dump, find the blocking
|
||||
call, fix it. Never raise a limit or rerun to get green; a limit changes only
|
||||
@@ -1830,8 +1847,9 @@ Inspect plan keeps D60–D66 (plan D60 → D83, D61 → D84, D65 → D88, D68
|
||||
blocks' context ("Current file image tags" in History). An empty source draws no
|
||||
section. Repeats are removed within a source only, so a linked tag that is also an
|
||||
image tag shows in both sections and each count is true. D21's filter rules are
|
||||
unchanged. History's current-file read carries no `image_tags` facet (D88), so there
|
||||
only the linked section shows. `tests/browser/inspector-concept.spec.mjs` wraps the
|
||||
unchanged. History's current-file read carries the same `image_tags` facet as the Images
|
||||
read (#457), so History's inline current file and its viewer show the image tags section
|
||||
too (`test_current_inspection_carries_the_files_image_tags`). `tests/browser/inspector-concept.spec.mjs` wraps the
|
||||
concept's row in the same sections at runtime (`withTagSections`, the D154 rule
|
||||
injected) and compares the measured sections (frame, heading, label, row); a
|
||||
mutation of the label color, the row margin or the top rule fails it. Browser tests:
|
||||
@@ -2398,6 +2416,86 @@ npm run --prefix tools/vendor test:workflows -- workspace-collections.spec.mjs c
|
||||
images-concept.spec.mjs inspector-concept.spec.mjs shell-concept.spec.mjs --project chromium --project touch
|
||||
```
|
||||
|
||||
### Deleting images (#480)
|
||||
|
||||
Images › Delete… removes the checked images (or "Select all N matching", at most 500, or
|
||||
the one file of the side panel or the viewer) from the library, and on request moves the
|
||||
files to the system trash. **No code path unlinks or overwrites a media file.**
|
||||
|
||||
- **D159** (#480, maintainer request): the concept's "Remove links…" (`danger`, app.js
|
||||
432) returns as **Delete…** in the same place (after Export metadata, before Clear), and
|
||||
as the last action of the side panel and after Download in the Images viewer. Its
|
||||
review is the concept's `maintenancePlan` (`dl.meta-grid`, Cancel and a primary) with a
|
||||
radio group first, "What should happen to the files?", nothing checked (the ruling: ask
|
||||
every time): "Remove from the library only" or "Also move the files to the trash". The
|
||||
rows follow the chosen mode (Scope, Not changed, Prompt links, Image details, Backup,
|
||||
Media on disk) and the primary is `danger`: "Remove N images from the library" / "Move
|
||||
N images to the trash". `tests/browser/images-delete-concept.spec.mjs` compares the bar
|
||||
button and the review's grid and actions with the concept.
|
||||
- **Routes.** `POST /prompt_manager/images/delete/preview {items | filter}` reviews both
|
||||
modes at once under the maintenance preview lease and answers a reviewed plan
|
||||
(`_plan_response`, operation `delete_images`): `library.count`, `trash.count`, `refused`
|
||||
and `trash_refused` (`{count, counts, rows ≤ 50}` with coded reasons), `records`,
|
||||
`orphaned_prompts` and the backup. Each target is matched to its entry in the current
|
||||
listing (`not-listed` otherwise). `POST /prompt_manager/maintenance/apply {token, mode}`
|
||||
(`mode` required for this plan, refused for any other) starts job kind `image_delete`
|
||||
(admission `records`, not cancellable; groups `trashed`, `removed`, `not_changed`,
|
||||
`errors`): sources unchanged, the promised safety backup, then per file a recheck
|
||||
(containment and fingerprint) and
|
||||
- library mode: one transaction deletes the file's records and writes its removal mark;
|
||||
- trash mode: the file moves first (`utils/trash.py`); a refused move changes nothing;
|
||||
then its records go. A records failure stops the job (`records-failed`); later files
|
||||
are "Not attempted".
|
||||
- **What goes** (`database/image_removal.py`): image links under every spelling (pair
|
||||
links by cascade, workflow documents released), image tags, auto-tag marks, pending
|
||||
undecided Needs review items, the own category, collection memberships and the
|
||||
favorite. A decided or revised review item keeps its decision and the annotation rows it
|
||||
cites. Prompts, recorded runs and the vocabulary stay. Previews are removed.
|
||||
- **Removal marks** (`image_removals`, `image_removals_meta`): a library-only removal
|
||||
hides exactly that file (locator, size, `mtime_ns`) from the workspace listing, the
|
||||
output scan, the duplicate find, Tag output images and linking. A modified or replaced
|
||||
file shows again. An explicit tag, category, favorite or collection write clears the
|
||||
mark. Older builds ignore the tables (removed files show there); restoring an older
|
||||
backup shows every file again; restoring a pre-delete backup after trash mode brings
|
||||
back records of files that are in the trash (Missing-file links cleans them up). The
|
||||
legacy gallery lists every file on disk, removed or not. Hard links have one mark per
|
||||
path, and on FAT/exFAT/SMB a same-size replacement can stay hidden.
|
||||
Since #501 the thumbnail jobs (System › Thumbnails' counts and its missing, rebuild and
|
||||
convert runs, and the previews after a scan) and the legacy `GET /scan_output_dir`
|
||||
listing skip hidden files too; an on-demand preview the legacy gallery asks for is still
|
||||
made, and a one-image auto-tag (`/autotag/single` + `/autotag/apply`) is an explicit
|
||||
write that shows the file again. System › Maintenance **Missing-file links** also
|
||||
forgets the marks that hide nothing any more (the file is gone from a folder that still
|
||||
exists, or its bytes changed), counted in its scope ("K removed images whose files are
|
||||
gone or changed") and job line, judged only under a source that is readable and lists
|
||||
something (an unplugged drive's empty mount point keeps its marks), never for a source
|
||||
no longer configured, and re-checked at apply (`plan-changed`). The sidebar's Images,
|
||||
Favorites and Needs review counts and the collection counts are read again when an
|
||||
`image_delete` job ends in the tab that followed it (`JOB_END_COUNTS`, `state.mjs`).
|
||||
Tests: `tests/test_image_removal_gaps.py`, the sidebar case in
|
||||
`tests/browser/workspace-images-delete.spec.mjs`.
|
||||
- **Trash folders inside an output root** (#501, `utils/media_folders.py`): no walker
|
||||
descends a folder below a root named `.Trash`, `.Trash-<digits>`, `.Trashes` or
|
||||
`$RECYCLE.BIN` (any case), as none descends `thumbnails`: the workspace and legacy
|
||||
listings (`_walk_media`, `_settled_entry`), the metadata scan and duplicate find
|
||||
(`_collect_media_files`), the legacy auto-tag listing (`_collect_scan_files`) and the
|
||||
delete review (`trash-folder`). The image monitor tests every folder of the new file's
|
||||
absolute path, so it also ignores outputs written into a source that itself lies
|
||||
inside such a folder. Tests: `tests/test_media_folders.py`,
|
||||
`tests/test_image_removal_gaps.py`.
|
||||
- **Trash backends** (`utils/trash.py`, no dependency): Linux and BSD implement the
|
||||
freedesktop Trash specification (home trash on the same mount, else the mount's
|
||||
`.Trash/$uid` or `.Trash-$uid`), reserve the name with `O_EXCL` and move with
|
||||
`renameat2(RENAME_NOREPLACE)`; without it, across devices, on a read-only mount or with
|
||||
a trash inside an output root the move is refused (`no-trash`, `trash-in-root`). Windows
|
||||
never uses the Recycle Bin (#500, maintainer ruling (a)): files larger than the bin's
|
||||
capacity are deleted permanently and IFileOperation's `PreDeleteItem` guard cannot detect
|
||||
it (native probe runs 40951037575, 40953347904), so every file answers `windows-recycle`
|
||||
("Moving files to the Recycle Bin isn't available on Windows: files larger than the bin's
|
||||
capacity would be deleted permanently."). macOS reports `no-trash` until #499 lands;
|
||||
library-only removal works everywhere. Tests use `tests/fake_trash.py`; the browser test server installs it, so no
|
||||
test touches a real trash.
|
||||
|
||||
### Prompt library cards and recorded deviations (#249 PR E)
|
||||
|
||||
The shelf uses the shared concept toolbar and caption (E1) and the concept's
|
||||
@@ -2442,10 +2540,38 @@ Recorded deviations on this page:
|
||||
view" filters are active, and plain "Filters" otherwise; the text is its accessible
|
||||
name, so the count is spoken. The concept has no indicator, and without one a Tags
|
||||
filter set by Tags & categories Browse was invisible on the shelf. The Library counts
|
||||
Tags; Images counts Source, Folder, Media and Tags (each once). The toolbar category is
|
||||
Tags, Folder and Tag match (D150); Images counts Source, Folder, Media and Tags (each once). The toolbar category is
|
||||
not a Refine filter and is not counted. The caption's count is a `role="status"`
|
||||
region, written only when it changes and kept shown while the shelf reloads, so
|
||||
Load more and scrolling never re-announce it.
|
||||
- **D150** (#444, cutover #334): "Refine this view" on the Library holds the concept
|
||||
Filters dialog's **Folder** ("All output folders", then each folder below an output
|
||||
root as a tree built by path segment (each folder directly under its own parent,
|
||||
case variants as distinct siblings, siblings case-insensitive then exact), indented
|
||||
by depth, read from `GET /prompt_manager/library/options?include=folders`
|
||||
when the dialog opens; the chosen folder stays listed even when the list lacks it) and
|
||||
**Tags**, plus **Tag match** ("All of these tags" / "Any of these tags" / "Prompts without tags", D149) drawn as the
|
||||
concept's own select; the concept's "Prompt evidence" and prototype note are absent.
|
||||
`GET /prompt_manager/library` takes `folder` (`/`-joined; `\` is a separator only on
|
||||
Windows, on POSIX it is part of a folder's name as the listing shows it; `.`, `..`,
|
||||
empty segments and, on Windows, drives are 400) and `tag_mode` (`all` default, `any`, `untagged` from D149), and echoes
|
||||
both; a client whose non-default value is not echoed (an un-restarted older server)
|
||||
shows the read error, never the unfiltered shelf. A folder keeps a parent with one
|
||||
linked image (`generated_images.prompt_id`) in it or below it under any spelling of
|
||||
any output root (the root as configured or detected and after `realpath`, so a
|
||||
symlinked output folder works): `file_path` (`normcase(normpath(abspath))`) is
|
||||
compared with prefixes built by the same normalization, so case folding is the
|
||||
platform's own, including non-ASCII on Windows. The folder is an outer AND around the
|
||||
parent and pair arms; Any puts one `t.name IN (...)` in each arm, so a card matches
|
||||
when the parent or one active pair carries any tag (text and category still AND
|
||||
inside each arm); with one tag Any equals All. Filters counts Tags, Folder and a Tag
|
||||
match other than All (D44). Saved Library views store `tag_mode` (`all|any|untagged`) and
|
||||
`folder` (text 1024); older rows read the defaults. Tests:
|
||||
`tests/test_library_folders.py` (pure prefixes and names with `posixpath`/`ntpath`,
|
||||
drive, UNC, non-ASCII, symlinked root, the any-tag union), `test_api_library.py`,
|
||||
`test_saved_views.py`, the library-state and saved-views JS units, and
|
||||
`tests/browser/workspace-library-filters.spec.mjs` (behavior, the Refine dialog against
|
||||
the concept Filters dialog, artifact captures). Windows: run `portability.yml` on the PR.
|
||||
|
||||
`tests/browser/library-concept.spec.mjs` compares the shelf (#249 E3) on the
|
||||
concept-shaped fixture, which creates the concept's prompts last to first so Recently
|
||||
@@ -2495,7 +2621,8 @@ Category". It is built only from the concept's Images parts:
|
||||
input[type=checkbox]`); Home, End and the arrow keys still move between titles.
|
||||
- `div#library-bulkbar` sits between the toolbar and the caption, empty until something
|
||||
is checked, then the concept's `.bulkbar`: "N selected", Set category, Clear (D151
|
||||
adds Add tags before Set category, in the Images bar's order). The
|
||||
adds Add tags before Set category, in the Images bar's order; D152 adds Delete… before
|
||||
Clear, and Restore alone on the archived shelf). The
|
||||
selection is a set of prompt ids over the loaded shelf: windowed-out cards keep their
|
||||
state, Load more keeps it, and any shelf reload (search, category, sort, tags, a
|
||||
write) and leaving the page clear it. Clicking a checkbox toggles it and sets the
|
||||
@@ -2576,6 +2703,97 @@ during a write); `tests/browser/library-concept.spec.mjs` (the bar comparison no
|
||||
Add tags); units `tests/js/workspace-library-selection.test.js`; Python
|
||||
`tests/test_bulk_library_tags.py`. A Python change: ComfyUI needs a restart.
|
||||
|
||||
### Deleting and archiving selected prompts (#443)
|
||||
|
||||
**D152** (beyond the concept, on D148's bar; the concept archives a prompt from its recipe
|
||||
editor's "Archive…", `app.js` 976, reviewed in `maintenancePlan`, 953–958, and maps the legacy
|
||||
delete to a "reference-safe archive", 783): the Prompt library deletes or archives the
|
||||
selected prompts in one reviewed step, replacing the legacy admin page's bulk and single
|
||||
Delete, which refuses a whole batch when any prompt belongs to a saved pair (maintainer
|
||||
rulings of 2026-10-28):
|
||||
|
||||
- The bar reads "N selected", [Select all N loaded], Add tags, Set category, **Delete…**
|
||||
(the concept's `danger` class, as its Images bar's "Remove links…"), Clear.
|
||||
- Delete… posts `{prompt_ids, expected_epoch}` to `POST
|
||||
/prompt_manager/library/delete/preview`, which classifies each id in this order: not
|
||||
found (missing or not a Library parent), protected (tagged `__protected__`), already
|
||||
archived, archive (the positive or negative of any saved pair, archived pair or not),
|
||||
delete (everything else). From the press until the review opens, every bar action, Clear
|
||||
too, is `aria-disabled` and Delete… reads "Reviewing…" (a double press sends one preview).
|
||||
- The review is the shared record-changing review (`openReviewDialog`), titled "Delete
|
||||
selected prompts", "Archive selected prompts" or "Delete or archive selected prompts", its
|
||||
`dl.meta-grid` rows Delete permanently ("N prompts not in a saved pair") and Archive ("N prompts in a saved
|
||||
pair: hidden, restorable", never negative variants: a pair's prompt with none is archived too) (always), Protected (kept) with up to
|
||||
three titles, Already archived, No longer exist, and, when something is deleted, Image
|
||||
links removed and Backup ("Create and verify before deleting", or "Off: no safety backup
|
||||
(System › Backup & export › Recovery policy)"); then `p.small` sentences: why paired
|
||||
prompts are archived and how to restore them, what a delete removes and that it cannot be
|
||||
undone except by restoring the safety backup (which returns the whole library to that
|
||||
moment; kept among the last 10), that a later scan, new image or node run with the same
|
||||
text adds a deleted prompt again, and how to unprotect. The primary is `danger`: "Delete D
|
||||
prompts", "Archive A prompts" or "Delete D and archive A prompts". When nothing can change,
|
||||
or the disk has no room for the promised backup, the review is close-only with the reason.
|
||||
The concept's "Review maintenance plan" title, its demo note, Scope/Protected/Media on
|
||||
disk rows and `ca-primary` start are replaced by these.
|
||||
- Start posts the reviewed lists `{archive_ids, delete_ids, expected_epoch, backup}` to `POST
|
||||
/prompt_manager/library/delete` while Delete… reads "Deleting…" (or "Archiving…"). The
|
||||
server takes a `records` hold ("Delete prompts (Library)", refused 409 `busy` beside a scan,
|
||||
restore, maintenance, move or Optimize), checks the disk (507 `insufficient-space`) and,
|
||||
when a delete remains and the reviewed `backup` was "on" or the policy is on now, takes a
|
||||
verified safety backup (500 `backup-failed`, nothing changed). One `BEGIN IMMEDIATE`
|
||||
transaction then deletes only reviewed deletes still unpaired whose prompt row, tags,
|
||||
title and image links are unchanged since the backup (exactly the legacy rows: image
|
||||
link rows, cascaded tags, Library state and pair links of images, released workflow
|
||||
documents; files never), archives the paired ones and any reviewed delete now paired, and
|
||||
counts the rest (`already_archived`, `not_found`, `changed`). Archiving writes a
|
||||
`prompt_archive` row, the prompt's `updated_at` and, when its stored hash is stale or
|
||||
missing and no other row holds the canonical one, the canonical hash, so scans, the
|
||||
monitor and node saves find the archived row instead of adding a copy. A changed epoch
|
||||
answers 409 `storage-changed`; nothing is written.
|
||||
- Outcomes: "Deleted D prompts." "Archived A prompts." with already archived, no longer
|
||||
exist, changed and kept-protected sentences and "Safety backup saved: NAME (System ›
|
||||
Backup & export)."; the shelf, categories, cached Library views and sidebar counts reload.
|
||||
Storage restored: "The library was restored or moved since this page loaded. Nothing was
|
||||
deleted or archived."; busy: "NAME is running. Nothing was changed; try again when it
|
||||
finishes."; a failed backup or a short disk: the server's sentence; no answer, 502, 503 or
|
||||
504: "The server did not confirm the change. The Library was reloaded: check the shelf and
|
||||
Show archived prompts."; other refusals: "The selected prompts could not be changed. Try
|
||||
again.", the selection kept.
|
||||
- **Show archived prompts (N)** (heading actions, before Export, `aria-pressed`; N the
|
||||
archived prompts matching the filters, `archived_total`; the shelf counterpart of #235's
|
||||
Show archived) lists only archived prompts (`GET /prompt_manager/library?shelf=archived`,
|
||||
a cursor never crosses shelves), each card with an "Archived" `.chip`, the caption "N
|
||||
archived prompts · M negative variants hidden with them", the bar "N selected", Restore,
|
||||
Clear, and "No archived prompts." / "No archived prompts match." when empty. The pressed
|
||||
state stays with the shelf visit, never in a saved view; Browse and a saved view open the
|
||||
shelf. Restore posts `{prompt_ids, expected_epoch}` to `POST
|
||||
/prompt_manager/library/restore` (every requested archived id, protected or not): "Restored
|
||||
N prompts to the Prompt library."
|
||||
- An archived prompt's page shows the concept's `.notice` ("Archived" / "This prompt is archived
|
||||
because it is in a saved pair. It and its N negative variants are hidden from the Prompt library; nothing was
|
||||
removed.", or "It is hidden from…" when it has none) with Restore;
|
||||
everything else on the page works. New prompt or Add negative on an archived prompt's
|
||||
text keeps it archived and toasts "This text is an archived prompt. Restore it to show it
|
||||
in the Prompt library again."
|
||||
- Archived prompts: hidden from the shelf, its search, `total`, the sidebar count, Tags &
|
||||
categories' prompt counts (a value only archived prompts hold stays listed with 0) and
|
||||
System › Overview and the restore review's positive counts; still counted by rename and
|
||||
delete impacts and usage statistics; still shown as another prompt's negative variant, on
|
||||
recorded runs and Images, by the legacy pages and the ComfyUI node's search; still reached
|
||||
by run counting, scans and the monitor; never pruned by maintenance (a new protection,
|
||||
"archived in the Library"); exported with `archived: true`. Older builds ignore the table
|
||||
and show them again; the rows stay.
|
||||
- Deviation from legacy: a selected `__protected__` prompt is never deleted or archived (the
|
||||
legacy delete removed it); it is listed in the review and the toast.
|
||||
|
||||
Tests: `tests/browser/workspace-library-delete.spec.mjs` (mixed review and result on the
|
||||
server, Cancel, Show archived prompts and Restore from the bar and the notice, double press,
|
||||
failed backup, busy, unconfirmed, restore fence); `tests/browser/library-delete-concept.spec.mjs`
|
||||
(the review against the concept's `maintenancePlan`, the bar's Delete… against the concept's
|
||||
"Remove links…"); units `tests/js/workspace-library-delete.test.js`; Python
|
||||
`tests/test_prompt_archive_schema.py`, `tests/test_bulk_library_delete.py`,
|
||||
`tests/test_library_delete_routes.py`. A Python change: ComfyUI needs a restart.
|
||||
|
||||
### No 1–5 prompt rating in the workspace (#447)
|
||||
|
||||
**D153** (maintainer ruling 2026-10-28: the star is only for favorites; the concept keeps
|
||||
@@ -2840,6 +3058,12 @@ mapped differences"):
|
||||
change within the workspace waits the same way: the shell renders the next route once a
|
||||
video still starting on Inspect settles (`leaveGuard`; the latest route wins). Route
|
||||
changes at `loadstart` + 0–7 ms hung 4–5 of 120 pages before and 0 of 120 after.
|
||||
The shared viewer's own video gets the same guard (#483): a new selection while the
|
||||
viewer D155 opened is still starting its video leaves the viewer open, untouched, until
|
||||
that video reports metadata or an error (or 15 s), then closes it; the URL is revoked
|
||||
after that, and a viewer asked for meanwhile (the new file's D155 open) opens then. The
|
||||
merge queue hung once without it (run 40903238387: the PNG selection 12 ms after the
|
||||
inline video showed, with the viewer's new video still starting).
|
||||
A real page leave cannot wait. With no page code releasing anything, navigating
|
||||
away hung 8 of 120 pages and reload 22 of 120 (3 and 8 of 120 with the page's own
|
||||
release), so that WebKit teardown stays a known limitation; dispose still releases at
|
||||
@@ -3794,8 +4018,8 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
- **D104** (Backup & export, S9, Q4): Recovery policy keeps the concept's "Proposed" chip.
|
||||
"Create a safety backup before maintenance" is the server's `safety_backup`
|
||||
(`maintenance.safety_backup`) policy. Its line says what it does ("Keep a verified copy
|
||||
of the database on disk before maintenance, cleanups and LoRA imports change records. A
|
||||
restore always keeps one."); the concept's "Preview a verified backup before applying
|
||||
of the database on disk before maintenance, cleanups, LoRA imports and prompt deletes
|
||||
change records. A restore always keeps one."); the concept's "Preview a verified backup before applying
|
||||
operations that change records." describes nothing the server does. It is drafted behind the Save row and saved with
|
||||
`expected_revision` (a configuration changed elsewhere is the conflict status); it is
|
||||
disabled with "Server settings could not be read." when the read failed. "Scheduled
|
||||
@@ -6332,7 +6556,7 @@ The versioned public reads are:
|
||||
| `GET /prompt_manager/library/{positive_id}/history/pairs/{pair_id}/newest` | The pair's newest recorded run for the history run dot (#249 D26): `{run: {run_id, anchor_id, context_id}}` or `{run: null}` |
|
||||
| `GET /prompt_manager/history/runs/{run_id}` | Frozen fields, per-field provenance, capture and outcome for a known UUID |
|
||||
| `GET /prompt_manager/history/runs/{run_id}/outputs` | Ordered host-reported occurrences, including repeated locations |
|
||||
| `GET /prompt_manager/history/runs/{run_id}/output/current/{media,thumbnail,inspection}` | Authorized current file at that reported location, using the returned epoch/scope URL |
|
||||
| `GET /prompt_manager/history/runs/{run_id}/output/current/{media,thumbnail,inspection}` | Authorized current file at that reported location, using the returned epoch/scope URL; `inspection` carries the shared inspector envelope including the file's `image_tags` facet (#457) |
|
||||
|
||||
The newest-pair read takes no query. It answers 404 when the positive is not an
|
||||
eligible saved prompt or the pair is not one of its pairs. It walks the positive's exact
|
||||
@@ -6636,12 +6860,12 @@ Deploying this backend needs a ComfyUI restart (Python and route changes).
|
||||
|
||||
`#/tags` ("Tags & categories", page title "Tags") is a native workspace route; it no
|
||||
longer hands off to the legacy tag manager. That page stays reachable from Overview's
|
||||
"Open existing tag manager" card for multi-tag AND/OR browsing (tag search and sort
|
||||
and the untagged entry moved here with D149, #444); bulk prompt delete stays on the
|
||||
legacy admin page (bulk tags: the Prompt library's selection bar, D151). A
|
||||
prompt's category is changed in the Prompt library editor, on many prompts at once with
|
||||
the Prompt library's selection bar (D148, #437), or with the legacy admin page's "Set
|
||||
Category" (both `POST /prompt_manager/bulk/category`);
|
||||
"Open existing tag manager" card for its own page; tag search and sort, the untagged
|
||||
entry (D149) and multi-tag OR (the Library's Tag match, D150) moved to the workspace
|
||||
(#444); bulk prompt delete stays on the legacy admin page (bulk tags: the Prompt
|
||||
library's selection bar, D151). A prompt's category is changed in the Prompt library
|
||||
editor, on many prompts at once with the Prompt library's selection bar (D148, #437), or
|
||||
with the legacy admin page's "Set Category" (both `POST /prompt_manager/bulk/category`);
|
||||
the Images selection bar's Set category sets images' own categories only (D140, #332),
|
||||
and Manage category renames a value on prompts, recipes and images together (D144). The screen consumes only the `/prompt_manager/vocabulary/*` routes
|
||||
above and adds no route, so PR B needed no ComfyUI restart; D149 (#444) adds read
|
||||
@@ -6714,10 +6938,10 @@ fixture-only tags after the concept's sample tags.
|
||||
counted every prompt without `prompt_tags` and hid the entry at 0; this row always
|
||||
shows) and echoes `search` and `sort`. Browse opens the Library with `tag_mode=untagged`
|
||||
(`GET /prompt_manager/library`, echoed as `tag_mode`; with a `tag` it is 400): "Filters ·
|
||||
1", and Refine this view adds **Tag match** after Tags ("All of these tags" / "Prompts
|
||||
1", and Refine this view adds **Tag match** after Tags ("All of these tags" / "Any of these tags" (D150) / "Prompts
|
||||
without tags", the latter disables Tags and drops them on Apply). A client whose request
|
||||
the server did not echo (an un-restarted older server) shows the read error with Retry,
|
||||
never an unfiltered list. Saved Library views store `tag_mode` (`all|untagged`, default
|
||||
never an unfiltered list. Saved Library views store `tag_mode` (`all|any|untagged`, default
|
||||
`all`); untagged with tags is refused on write and reads back as `all`; an older build
|
||||
opens an untagged view as the whole Library. The concept comparison holds the row to the
|
||||
concept's first category row, its top to the concept table's offset, the table to that
|
||||
|
||||
@@ -40,6 +40,7 @@ from .annotation_schema import (
|
||||
METADATA_TABLE,
|
||||
)
|
||||
from .autotag_schema import AUTOTAG_TABLE
|
||||
from .image_removal_schema import clear_marks
|
||||
from .library_commands import _UUID, LibraryCommandError, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .vocabulary import validate_vocabulary_tag_name
|
||||
@@ -236,6 +237,9 @@ class AnnotationOperations:
|
||||
|
||||
@staticmethod
|
||||
def _annotation_owner(conn: sqlite3.Connection, locator: str, now: str) -> str:
|
||||
# A file with image tags or Needs review state is in the library: a
|
||||
# removal mark never hides it (#480).
|
||||
clear_marks(conn, [locator])
|
||||
row = conn.execute(_OWNER, (locator,)).fetchone()
|
||||
if row is not None:
|
||||
return row[0]
|
||||
|
||||
@@ -7,7 +7,11 @@ from typing import Callable, ContextManager, Iterator
|
||||
|
||||
try:
|
||||
from ..utils.execution_history_events import RunFinish, encode_command
|
||||
from ..utils.hashing import stored_prompt_text, stored_prompt_text_sql
|
||||
from ..utils.hashing import (
|
||||
generate_prompt_hash,
|
||||
stored_prompt_text,
|
||||
stored_prompt_text_sql,
|
||||
)
|
||||
from ..utils.execution_contexts import (
|
||||
CAPTURE_REASONS,
|
||||
ContextBundle,
|
||||
@@ -16,7 +20,11 @@ try:
|
||||
)
|
||||
except ImportError:
|
||||
from utils.execution_history_events import RunFinish, encode_command
|
||||
from utils.hashing import stored_prompt_text, stored_prompt_text_sql
|
||||
from utils.hashing import (
|
||||
generate_prompt_hash,
|
||||
stored_prompt_text,
|
||||
stored_prompt_text_sql,
|
||||
)
|
||||
from utils.execution_contexts import (
|
||||
CAPTURE_REASONS,
|
||||
ContextBundle,
|
||||
@@ -149,9 +157,12 @@ def _role_hints(role: str) -> str:
|
||||
|
||||
# Set form of the read-time prompt-hint rule in get_execution_contexts():
|
||||
# a stored role hint is live only in the current storage epoch, and only when
|
||||
# exactly one prompt has its raw snapshot text or that text's stored form
|
||||
# (outer whitespace stripped, #438), and that is the hinted one (_prompt_hint).
|
||||
# Evaluated once per statement, never per prompt row.
|
||||
# the hinted prompt is the one match of _prompt_hint: by text (the raw
|
||||
# snapshot text or its stored form, outer whitespace stripped, #438), or, only
|
||||
# when no prompt has either text, by dedup hash (case folded too, #460).
|
||||
# The hash is a fallback, so a hint #438 kept is never made ambiguous.
|
||||
# Needs register_prompt_functions (pm_prompt_hash). The hint rows are the
|
||||
# outer loop (CROSS JOIN), so each prompts lookup is by index.
|
||||
# Defines the CTE ``live_hint_prompts(prompt_id)``; prefix it with WITH.
|
||||
LIVE_HINT_CTES = (
|
||||
"stored_hints(prompt_id, raw, stored) AS ("
|
||||
@@ -160,7 +171,12 @@ LIVE_HINT_CTES = (
|
||||
+ _role_hints("negative")
|
||||
+ "), hint_matches(prompt_id, raw, stored, match_id) AS ("
|
||||
"SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h "
|
||||
"JOIN prompts p ON p.text COLLATE BINARY IN (h.raw, h.stored)"
|
||||
"CROSS JOIN prompts p ON p.text COLLATE BINARY IN (h.raw, h.stored) "
|
||||
"UNION ALL "
|
||||
"SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h "
|
||||
"CROSS JOIN prompts p ON p.hash = pm_prompt_hash(h.raw) "
|
||||
"WHERE NOT EXISTS (SELECT 1 FROM prompts q "
|
||||
"WHERE q.text COLLATE BINARY IN (h.raw, h.stored))"
|
||||
"), live_hint_prompts(prompt_id) AS ("
|
||||
"SELECT prompt_id FROM hint_matches GROUP BY prompt_id, raw, stored "
|
||||
"HAVING COUNT(*) = 1 AND MIN(match_id) = prompt_id)"
|
||||
@@ -169,20 +185,32 @@ LIVE_HINT_PROMPT_IDS_SQL = (
|
||||
"WITH " + LIVE_HINT_CTES + " SELECT DISTINCT prompt_id FROM live_hint_prompts"
|
||||
)
|
||||
|
||||
_BY_TEXT = "SELECT id FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 2"
|
||||
|
||||
|
||||
def _text_matches(conn: sqlite3.Connection, raw: str) -> list:
|
||||
return [row[0] for row in conn.execute(_BY_TEXT, (raw, stored_prompt_text(raw)))]
|
||||
|
||||
|
||||
def _prompt_hint(conn: sqlite3.Connection, raw: str | None) -> int | None:
|
||||
"""The one prompt whose text is ``raw`` or the stored form of it (#438).
|
||||
"""The one prompt that stores ``raw`` (#438), else the one it counts toward.
|
||||
|
||||
The Library stores ``text.strip()``; the exact form keeps hints of older
|
||||
rows whose stored text kept outer whitespace.
|
||||
By text first: ``raw`` or ``raw.strip()`` (the exact form keeps hints of
|
||||
older rows that kept outer whitespace). Only when no prompt has either
|
||||
text, by the dedup hash, which also folds case (#460).
|
||||
"""
|
||||
if raw is None:
|
||||
return None
|
||||
rows = conn.execute(
|
||||
"SELECT id FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 2",
|
||||
(raw, stored_prompt_text(raw)),
|
||||
).fetchall()
|
||||
return rows[0][0] if len(rows) == 1 else None
|
||||
matches = _text_matches(conn, raw)
|
||||
if not matches:
|
||||
matches = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT id FROM prompts WHERE hash=? LIMIT 2",
|
||||
(generate_prompt_hash(raw),),
|
||||
)
|
||||
]
|
||||
return matches[0] if len(matches) == 1 else None
|
||||
|
||||
|
||||
def _hints(
|
||||
@@ -191,24 +219,16 @@ def _hints(
|
||||
pos_id, neg_id = _prompt_hint(conn, positive), _prompt_hint(conn, negative)
|
||||
stored = None if negative is None else stored_prompt_text(negative)
|
||||
pair_id = None
|
||||
# A known negative that names no prompt because several share its text
|
||||
negative_ambiguous = (
|
||||
stored is not None
|
||||
and neg_id is None
|
||||
and conn.execute(
|
||||
"SELECT 1 FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 1",
|
||||
(negative, stored),
|
||||
).fetchone()
|
||||
is not None
|
||||
stored is not None and neg_id is None and bool(_text_matches(conn, negative))
|
||||
)
|
||||
if pos_id is not None and stored is not None and not negative_ambiguous:
|
||||
rows = conn.execute(
|
||||
"SELECT pp.id FROM prompt_pairs pp LEFT JOIN prompts n ON "
|
||||
"n.id=pp.negative_prompt_id "
|
||||
"SELECT pp.id FROM prompt_pairs pp "
|
||||
"WHERE pp.positive_prompt_id=? AND ((pp.negative_prompt_id IS NULL AND "
|
||||
"?='') "
|
||||
"OR (pp.negative_prompt_id=? AND n.text COLLATE BINARY IN (?,?))) "
|
||||
"LIMIT 2",
|
||||
(pos_id, stored, neg_id, negative, stored),
|
||||
"?='') OR pp.negative_prompt_id=?) LIMIT 2",
|
||||
(pos_id, stored, neg_id),
|
||||
).fetchall()
|
||||
if len(rows) == 1:
|
||||
pair_id = rows[0][0]
|
||||
|
||||
+3
-7
@@ -23,10 +23,10 @@ from pathlib import PurePosixPath, PureWindowsPath
|
||||
from urllib.parse import quote
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
from .models import normalize_image_path
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.hashing import register_prompt_functions
|
||||
from database.models import normalize_image_path
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.health")
|
||||
@@ -125,12 +125,8 @@ def _foreign_keys(conn, _roots):
|
||||
)
|
||||
|
||||
|
||||
def _prompt_hash(text):
|
||||
return generate_prompt_hash(text) if isinstance(text, str) else None
|
||||
|
||||
|
||||
def _canonical_hashes(conn, _roots):
|
||||
conn.create_function("pm_prompt_hash", 1, _prompt_hash, deterministic=True)
|
||||
register_prompt_functions(conn) # pm_prompt_hash, shared with history (#460)
|
||||
return _counted(
|
||||
f"Prompt {row[0]}: the stored hash does not match its text"
|
||||
for row in conn.execute(
|
||||
|
||||
+23
-16
@@ -16,9 +16,9 @@ from .execution_outputs import _capture, _REFERENCE_COLUMNS
|
||||
from .library import _ELIGIBLE
|
||||
|
||||
try:
|
||||
from ..utils.hashing import stored_prompt_text, stored_prompt_text_sql
|
||||
from ..utils.hashing import prompt_key, stored_prompt_text
|
||||
except ImportError:
|
||||
from utils.hashing import stored_prompt_text, stored_prompt_text_sql
|
||||
from utils.hashing import prompt_key, stored_prompt_text
|
||||
|
||||
RunKey = tuple[int, str, int, str]
|
||||
|
||||
@@ -115,12 +115,12 @@ _MATCHING = (
|
||||
)
|
||||
|
||||
|
||||
def _scope_text_ids(conn: sqlite3.Connection, scope: str) -> str:
|
||||
"""Every recorded raw text whose stored form is ``scope``, as a JSON array.
|
||||
def _scope_text_ids(conn: sqlite3.Connection, key: str) -> str:
|
||||
"""Every recorded raw text whose prompt key is ``key``, as a JSON array.
|
||||
|
||||
The Library stores ``text.strip()`` while a run keeps its exact raw text,
|
||||
so a raw text joins the scope of the prompt it saves (#438). The values
|
||||
are compared, never only their digests.
|
||||
A run keeps its exact raw text, while the Library saves and counts it as
|
||||
the prompt with its key (outer whitespace #438, case #460). The values are
|
||||
compared, never only their digests; ``pm_prompt_key`` is Python's key.
|
||||
"""
|
||||
return json.dumps(
|
||||
[
|
||||
@@ -130,16 +130,16 @@ def _scope_text_ids(conn: sqlite3.Connection, scope: str) -> str:
|
||||
"SELECT t.text_id FROM (SELECT DISTINCT raw_text_id AS text_id "
|
||||
"FROM execution_prompt_snapshots) s CROSS JOIN "
|
||||
"execution_context_texts t WHERE t.text_id=s.text_id AND "
|
||||
f"{stored_prompt_text_sql('t.text_value')}=? COLLATE BINARY",
|
||||
(scope,),
|
||||
"pm_prompt_key(t.text_value)=? COLLATE BINARY",
|
||||
(key,),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _scope_text_id(raw: str | None) -> str | None:
|
||||
"""The scope digest of a raw text: that of the text the Library stores."""
|
||||
return None if raw is None else content_digest("text", (stored_prompt_text(raw),))
|
||||
"""The scope digest of a raw text: that of its prompt key."""
|
||||
return None if raw is None else content_digest("text", (prompt_key(raw),))
|
||||
|
||||
|
||||
def _limit(limit: int) -> None:
|
||||
@@ -290,7 +290,14 @@ class HistoryReadOperations:
|
||||
return None
|
||||
text = saved["text"]
|
||||
raw_id = _scope_text_id(text)
|
||||
if after is not None and after.raw_text_id != raw_id:
|
||||
# A position from before #460 names the scope by the stripped text
|
||||
# alone; it is the same scope, so a page open across the upgrade resumes.
|
||||
legacy_id = (
|
||||
None
|
||||
if text is None
|
||||
else content_digest("text", (stored_prompt_text(text),))
|
||||
)
|
||||
if after is not None and after.raw_text_id not in (raw_id, legacy_id):
|
||||
raise ValueError("History position text scope changed")
|
||||
group = {
|
||||
"positive_id": positive_id,
|
||||
@@ -309,7 +316,7 @@ class HistoryReadOperations:
|
||||
)
|
||||
if raw_id is None:
|
||||
return result
|
||||
scope = _scope_text_ids(conn, stored_prompt_text(text))
|
||||
scope = _scope_text_ids(conn, prompt_key(text))
|
||||
where = f"r.run_id IN ({_MATCHING})"
|
||||
params = [scope]
|
||||
if after is not None:
|
||||
@@ -465,7 +472,7 @@ class HistoryReadOperations:
|
||||
if saved["text"] is None:
|
||||
return {"run": None}
|
||||
epoch = self.get_execution_epoch()
|
||||
stored = stored_prompt_text(saved["text"])
|
||||
key = prompt_key(saved["text"])
|
||||
candidates = conn.execute(
|
||||
"SELECT r.run_id,rc.anchor_id,rc.context_id,"
|
||||
"c.positive_snapshot_id,c.negative_snapshot_id "
|
||||
@@ -477,7 +484,7 @@ class HistoryReadOperations:
|
||||
"AND rc.reference_epoch=? "
|
||||
f"ORDER BY {_ORDER},rc.anchor_id COLLATE BINARY LIMIT ?",
|
||||
(
|
||||
_scope_text_ids(conn, stored),
|
||||
_scope_text_ids(conn, key),
|
||||
pair_id,
|
||||
epoch,
|
||||
_PAIR_RUN_CANDIDATES,
|
||||
@@ -489,7 +496,7 @@ class HistoryReadOperations:
|
||||
if (
|
||||
positive is None
|
||||
or positive["raw"] is None
|
||||
or stored_prompt_text(positive["raw"]) != stored
|
||||
or prompt_key(positive["raw"]) != key
|
||||
):
|
||||
continue
|
||||
hints = _hints(conn, positive["raw"], negative["raw"] if negative else None)
|
||||
|
||||
@@ -221,6 +221,7 @@ class ImageCategoryOperations:
|
||||
sources. Returns ``{updated, unchanged}``; a file that already had
|
||||
exactly this value is ``unchanged`` and keeps its ``updated_at``.
|
||||
"""
|
||||
from .image_removal_schema import clear_marks_of_keys
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
keys = list(dict.fromkeys(tuple(key) for key in file_keys))
|
||||
@@ -234,6 +235,7 @@ class ImageCategoryOperations:
|
||||
continue
|
||||
conn.execute(UPSERT_SQL, (*key, category, now))
|
||||
counts["updated"] += 1
|
||||
clear_marks_of_keys(conn, keys) # #480: a categorized file is listed
|
||||
return counts
|
||||
|
||||
def clear_image_categories(self, file_keys) -> int:
|
||||
|
||||
@@ -402,6 +402,7 @@ class ImageCollectionOperations:
|
||||
is created only when at least one item is inserted. Returns
|
||||
``{collection, created, added, already}``.
|
||||
"""
|
||||
from .image_removal_schema import clear_marks_of_keys
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
if (collection_id is None) == (name is None):
|
||||
@@ -438,6 +439,7 @@ class ImageCollectionOperations:
|
||||
).rowcount
|
||||
for root, path in pairs
|
||||
)
|
||||
clear_marks_of_keys(conn, pairs) # #480: a collected file is listed
|
||||
return {
|
||||
"collection": collection,
|
||||
"created": created,
|
||||
|
||||
@@ -104,11 +104,14 @@ class ImageFavoriteOperations:
|
||||
"""Mark one file as a favorite; repeating it is a no-op. Returns True."""
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
from .image_removal_schema import clear_marks_of_keys
|
||||
|
||||
with self.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO image_favorites VALUES (?, ?, ?)",
|
||||
(root_key, relative_path, utc_now_iso()),
|
||||
)
|
||||
clear_marks_of_keys(conn, [(root_key, relative_path)]) # #480
|
||||
return True
|
||||
|
||||
def clear_image_favorite(self, root_key: str, relative_path: str) -> bool:
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
"""Remove a file's records from the library, and its removal mark (#480).
|
||||
|
||||
A *removal* describes one physical file the way the API resolved it from the
|
||||
walked listing (``py/api/image_delete.py``):
|
||||
|
||||
- ``spellings``: every path string records may name it by
|
||||
(``duplicate_spellings.Spellings.of``);
|
||||
- ``locator``: its private canonical locator;
|
||||
- ``favorites``: ``(root key, relative key)`` under every listed root holding
|
||||
it, the keys of its favorite and its collection memberships;
|
||||
- ``category``: its own category's file key, or None.
|
||||
|
||||
``remove_image_records`` deletes, in one ``BEGIN IMMEDIATE`` transaction:
|
||||
|
||||
- its image records (``generated_images``) under every spelling, plus legacy
|
||||
rows matched by ``image_path``; their saved-pair links go by cascade and the
|
||||
workflow documents they held are released;
|
||||
- the image tags and auto-tag marks of its byte records. The annotation file
|
||||
and byte rows go too, unless a Needs review item still cites them
|
||||
(``media_review_items.byte_id`` is ``ON DELETE RESTRICT``): a decided or
|
||||
revised item keeps its decision;
|
||||
- its pending, undecided Needs review items (settled as duplicate removal
|
||||
settles them);
|
||||
- its own category, favorite and collection memberships.
|
||||
|
||||
With ``mark`` it also writes the file's removal mark (library-only removal):
|
||||
the file stays on disk, hidden until its bytes change. Prompts, recorded runs
|
||||
and the media file itself are never touched here.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from .annotation_schema import BYTES_TABLE, FILES_TABLE
|
||||
from .annotations import _BUMP_INDEX
|
||||
from .autotag_schema import AUTOTAG_TABLE
|
||||
from .duplicate_copies import (
|
||||
DELETE_FAVORITE_SQL,
|
||||
DELETE_LINK_SQL,
|
||||
DELETE_MEMBERSHIPS_SQL,
|
||||
_file_link_rows,
|
||||
)
|
||||
from .image_categories import DELETE_SQL as DELETE_CATEGORY_SQL
|
||||
from .image_removal_schema import (
|
||||
META_TABLE,
|
||||
TABLE,
|
||||
clear_marks,
|
||||
has_marks_table,
|
||||
mark_of,
|
||||
write_mark as _write_mark,
|
||||
)
|
||||
from .library import display_title
|
||||
from .models import utc_now_iso
|
||||
from .review import _BUMP_REVIEW_INDEX, _PENDING_OF_LOCATOR, _SETTLE_ITEM
|
||||
from .review_changes import notify_review_change
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
BYTES_OF_LOCATOR_SQL = (
|
||||
f"SELECT b.id FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b " # nosec B608
|
||||
"ON b.file_id=f.id WHERE f.locator=?"
|
||||
)
|
||||
TAG_COUNT_SQL = "SELECT COUNT(*) FROM media_image_tags WHERE byte_id=?"
|
||||
AUTOTAG_COUNT_SQL = (
|
||||
f"SELECT COUNT(*) FROM {AUTOTAG_TABLE} WHERE byte_id=?" # nosec B608
|
||||
)
|
||||
CLEAR_TAGS_SQL = "DELETE FROM media_image_tags WHERE byte_id=?"
|
||||
CLEAR_AUTOTAG_SQL = f"DELETE FROM {AUTOTAG_TABLE} WHERE byte_id=?" # nosec B608
|
||||
BUMP_TAG_REVISION_SQL = (
|
||||
f"UPDATE {BYTES_TABLE} SET tag_revision=tag_revision+1 WHERE id=?" # nosec B608
|
||||
)
|
||||
REVIEW_REFS_SQL = (
|
||||
f"SELECT COUNT(*) FROM {FILES_TABLE} f " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
"JOIN media_review_items i ON i.byte_id=b.id WHERE f.locator=?"
|
||||
)
|
||||
DELETE_FILE_SQL = f"DELETE FROM {FILES_TABLE} WHERE locator=?" # nosec B608
|
||||
FILE_EXISTS_SQL = f"SELECT 1 FROM {FILES_TABLE} WHERE locator=?" # nosec B608
|
||||
FAVORITE_EXISTS_SQL = (
|
||||
"SELECT COUNT(*) FROM image_favorites WHERE root_key=? AND relative_path=?"
|
||||
)
|
||||
MEMBERSHIP_COUNT_SQL = (
|
||||
"SELECT COUNT(*) FROM image_collection_items WHERE root_key=? AND relative_path=?"
|
||||
)
|
||||
CATEGORY_EXISTS_SQL = (
|
||||
"SELECT COUNT(*) FROM image_categories WHERE root_key=? AND relative_path=?"
|
||||
)
|
||||
PAIR_LINK_COUNT_SQL = "SELECT COUNT(*) FROM image_prompt_pairs WHERE image_id=?"
|
||||
PROMPT_LINKS_SQL = "SELECT id FROM generated_images WHERE prompt_id=?"
|
||||
PROMPT_TITLE_SQL = (
|
||||
"SELECT p.text, s.title FROM prompts p "
|
||||
"LEFT JOIN prompt_library_state s ON s.prompt_id=p.id WHERE p.id=?"
|
||||
)
|
||||
MARKS_SQL = f"SELECT locator, size, mtime_ns FROM {TABLE}" # nosec B608
|
||||
REVISION_SQL = f"SELECT revision FROM {META_TABLE} WHERE singleton_id=1" # nosec B608
|
||||
MAX_ORPHAN_TITLES = 3
|
||||
RECORD_KINDS = (
|
||||
"prompt_links",
|
||||
"pair_links",
|
||||
"image_tags",
|
||||
"auto_tagged",
|
||||
"categories",
|
||||
"collections",
|
||||
"favorites",
|
||||
"review_items",
|
||||
)
|
||||
|
||||
|
||||
def _link_ids(conn, removal):
|
||||
"""``{image id: prompt id}`` of the records naming the file, any spelling."""
|
||||
found = {}
|
||||
for spelled in dict.fromkeys(removal["spellings"]):
|
||||
for image_id, prompt_id in _file_link_rows(conn, spelled):
|
||||
found[image_id] = prompt_id
|
||||
return found
|
||||
|
||||
|
||||
def _byte_ids(conn, locator):
|
||||
return [row[0] for row in conn.execute(BYTES_OF_LOCATOR_SQL, (locator,))]
|
||||
|
||||
|
||||
def file_locator(path) -> str:
|
||||
"""The locator of an absolute path (``normcase(realpath)``)."""
|
||||
return os.path.normcase(os.path.realpath(str(path)))
|
||||
|
||||
|
||||
def removed_from_library(conn, path) -> bool:
|
||||
"""Whether *path* is hidden by a removal mark whose bytes still match.
|
||||
|
||||
A mark whose file changed (other size or ``mtime_ns``) or is gone is
|
||||
stale: it is deleted on *conn* (the caller's write transaction), and the
|
||||
file is not hidden.
|
||||
"""
|
||||
locator = file_locator(path)
|
||||
mark = mark_of(conn, locator)
|
||||
if mark is None:
|
||||
return False
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except OSError:
|
||||
clear_marks(conn, [locator])
|
||||
return False
|
||||
if (info.st_size, info.st_mtime_ns) == tuple(mark):
|
||||
return True
|
||||
clear_marks(conn, [locator])
|
||||
return False
|
||||
|
||||
|
||||
def mark_hides(marks: dict, locator: str, path) -> bool:
|
||||
"""Whether a removal mark in *marks* hides the file at *path* (blocking).
|
||||
|
||||
*marks* is ``{locator: (size, mtime_ns)}``. The mark of *locator* hides
|
||||
the file while ``os.lstat(path)`` still has the size and ``mtime_ns`` it
|
||||
was removed with; a file that cannot be looked at is not hidden.
|
||||
"""
|
||||
mark = marks.get(locator)
|
||||
if mark is None:
|
||||
return False
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except OSError:
|
||||
return False
|
||||
return (info.st_size, info.st_mtime_ns) == tuple(mark)
|
||||
|
||||
|
||||
def write_mark(conn, locator, size, mtime_ns):
|
||||
_write_mark(conn, locator, size, mtime_ns, utc_now_iso())
|
||||
|
||||
|
||||
class ImageRemovalOperations:
|
||||
"""PromptDatabase mixin: remove a file's records; removal marks."""
|
||||
|
||||
def count_removal_records(self, removals) -> dict:
|
||||
"""Counts of what removing *removals* deletes, in one read snapshot.
|
||||
|
||||
``{records: {kind: n}, orphaned_prompts: {count, titles}}``: prompts
|
||||
whose every image record is among these files lose their last image
|
||||
(they stay in the Library).
|
||||
"""
|
||||
from .maintenance import _read_snapshot
|
||||
|
||||
def read(conn):
|
||||
totals = dict.fromkeys(RECORD_KINDS, 0)
|
||||
links = {}
|
||||
for removal in removals:
|
||||
found = _link_ids(conn, removal)
|
||||
links.update(found)
|
||||
for image_id in found:
|
||||
totals["pair_links"] += conn.execute(
|
||||
PAIR_LINK_COUNT_SQL, (image_id,)
|
||||
).fetchone()[0]
|
||||
tagged = autotagged = False
|
||||
for byte_id in _byte_ids(conn, removal["locator"]):
|
||||
tagged |= bool(
|
||||
conn.execute(TAG_COUNT_SQL, (byte_id,)).fetchone()[0]
|
||||
)
|
||||
autotagged |= bool(
|
||||
conn.execute(AUTOTAG_COUNT_SQL, (byte_id,)).fetchone()[0]
|
||||
)
|
||||
totals["image_tags"] += int(tagged)
|
||||
totals["auto_tagged"] += int(autotagged)
|
||||
totals["review_items"] += len(
|
||||
conn.execute(_PENDING_OF_LOCATOR, (removal["locator"],)).fetchall()
|
||||
)
|
||||
totals["favorites"] += int(
|
||||
any(
|
||||
conn.execute(FAVORITE_EXISTS_SQL, key).fetchone()[0]
|
||||
for key in removal["favorites"]
|
||||
)
|
||||
)
|
||||
totals["collections"] += sum(
|
||||
conn.execute(MEMBERSHIP_COUNT_SQL, key).fetchone()[0]
|
||||
for key in removal["favorites"]
|
||||
)
|
||||
if removal.get("category"):
|
||||
totals["categories"] += conn.execute(
|
||||
CATEGORY_EXISTS_SQL, removal["category"]
|
||||
).fetchone()[0]
|
||||
totals["prompt_links"] = len(links)
|
||||
return {
|
||||
"records": totals,
|
||||
"orphaned_prompts": self._orphaned_prompts(conn, links),
|
||||
}
|
||||
|
||||
return _read_snapshot(self.model.db_path, read)
|
||||
|
||||
@staticmethod
|
||||
def _orphaned_prompts(conn, links):
|
||||
removed = set(links)
|
||||
orphans = []
|
||||
for prompt_id in dict.fromkeys(links.values()):
|
||||
remaining = [
|
||||
row[0]
|
||||
for row in conn.execute(PROMPT_LINKS_SQL, (prompt_id,))
|
||||
if row[0] not in removed
|
||||
]
|
||||
if not remaining:
|
||||
orphans.append(prompt_id)
|
||||
titles = []
|
||||
for prompt_id in orphans[:MAX_ORPHAN_TITLES]:
|
||||
row = conn.execute(PROMPT_TITLE_SQL, (prompt_id,)).fetchone()
|
||||
if row is not None:
|
||||
titles.append(display_title((row[1],), row[0]) or "")
|
||||
return {"count": len(orphans), "titles": titles}
|
||||
|
||||
def remove_image_records(self, removal, *, mark=None) -> dict:
|
||||
"""Delete every record of one file (one transaction); see the module.
|
||||
|
||||
``mark``: ``(size, mtime_ns)`` to write the file's removal mark, or
|
||||
None (the file was moved to the trash). Returns the counts deleted.
|
||||
"""
|
||||
counts = dict.fromkeys(RECORD_KINDS, 0)
|
||||
locator = removal["locator"]
|
||||
with self._pair_transaction() as conn:
|
||||
links = _link_ids(conn, removal)
|
||||
if links:
|
||||
documents = documents_of_rows(conn, "id", list(links))
|
||||
for image_id in links:
|
||||
counts["pair_links"] += conn.execute(
|
||||
PAIR_LINK_COUNT_SQL, (image_id,)
|
||||
).fetchone()[0]
|
||||
conn.execute(DELETE_LINK_SQL, (image_id,))
|
||||
release_documents(conn, documents)
|
||||
counts["prompt_links"] = len(links)
|
||||
counts.update(self._remove_annotations(conn, locator))
|
||||
for key in removal["favorites"]:
|
||||
counts["favorites"] += conn.execute(DELETE_FAVORITE_SQL, key).rowcount
|
||||
counts["collections"] += conn.execute(
|
||||
DELETE_MEMBERSHIPS_SQL, key
|
||||
).rowcount
|
||||
if removal.get("category"):
|
||||
counts["categories"] += conn.execute(
|
||||
DELETE_CATEGORY_SQL, tuple(removal["category"])
|
||||
).rowcount
|
||||
if mark is not None:
|
||||
write_mark(conn, locator, *mark)
|
||||
if counts["review_items"]:
|
||||
notify_review_change(self.model.db_path)
|
||||
return counts
|
||||
|
||||
def _remove_annotations(self, conn, locator) -> dict:
|
||||
"""Tags, auto-tag marks, pending review items and, when free, the rows."""
|
||||
counts = {"image_tags": 0, "auto_tagged": 0, "review_items": 0}
|
||||
bytes_ = _byte_ids(conn, locator)
|
||||
for byte_id in bytes_:
|
||||
tags = conn.execute(CLEAR_TAGS_SQL, (byte_id,)).rowcount
|
||||
autotags = conn.execute(CLEAR_AUTOTAG_SQL, (byte_id,)).rowcount
|
||||
if tags:
|
||||
conn.execute(BUMP_TAG_REVISION_SQL, (byte_id,))
|
||||
counts["image_tags"] |= bool(tags)
|
||||
counts["auto_tagged"] |= bool(autotags)
|
||||
for (item_id,) in conn.execute(_PENDING_OF_LOCATOR, (locator,)).fetchall():
|
||||
counts["review_items"] += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
||||
if counts["review_items"]:
|
||||
conn.execute(_BUMP_REVIEW_INDEX)
|
||||
if conn.execute(FILE_EXISTS_SQL, (locator,)).fetchone():
|
||||
if not conn.execute(REVIEW_REFS_SQL, (locator,)).fetchone()[0]:
|
||||
conn.execute(DELETE_FILE_SQL, (locator,))
|
||||
conn.execute(_BUMP_INDEX)
|
||||
counts["image_tags"] = int(counts["image_tags"])
|
||||
counts["auto_tagged"] = int(counts["auto_tagged"])
|
||||
return counts
|
||||
|
||||
def removal_marks(self) -> dict:
|
||||
"""``{locator: (size, mtime_ns)}`` of every removal mark (one read)."""
|
||||
conn = self.model.get_connection()
|
||||
if not has_marks_table(conn):
|
||||
return {}
|
||||
return {row[0]: (row[1], row[2]) for row in conn.execute(MARKS_SQL)}
|
||||
|
||||
def removal_revision(self) -> int:
|
||||
conn = self.model.get_connection()
|
||||
if not has_marks_table(conn):
|
||||
return 0
|
||||
row = conn.execute(REVISION_SQL).fetchone()
|
||||
return row[0] if row else 0
|
||||
|
||||
def clear_removal_marks(self, locators) -> int:
|
||||
"""Show these files again (their marks go); the count removed."""
|
||||
locators = list(locators)
|
||||
if not locators:
|
||||
return 0
|
||||
with self._pair_transaction() as conn:
|
||||
return clear_marks(conn, locators)
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Removal marks (#480): files removed from the library that stay on disk.
|
||||
|
||||
``image_removals`` holds one row per hidden file: its private canonical
|
||||
locator (the ``media_annotation_files.locator`` form) and the ``size`` and
|
||||
``mtime_ns`` it had when it was removed. A file at that locator with other
|
||||
bytes is a new file and shows again. ``image_removals_meta`` is one row, the
|
||||
marks revision, bumped in the same transaction as every mark change so the
|
||||
workspace listing caches can tell marks changed.
|
||||
|
||||
The group installs like ``image_favorites``: optional in old backups,
|
||||
required once a database is current. Older builds ignore it.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
from .annotation_schema import MAX_LOCATOR_LENGTH
|
||||
|
||||
TABLE = "image_removals"
|
||||
META_TABLE = "image_removals_meta"
|
||||
TABLES = (TABLE, META_TABLE)
|
||||
_TABLE_DDL = {
|
||||
TABLE: (
|
||||
f"CREATE TABLE {TABLE} (locator TEXT PRIMARY KEY, "
|
||||
"size INTEGER NOT NULL CHECK(size>=0), mtime_ns INTEGER NOT NULL, "
|
||||
"removed_at TEXT NOT NULL) WITHOUT ROWID"
|
||||
),
|
||||
META_TABLE: (
|
||||
f"CREATE TABLE {META_TABLE} (singleton_id INTEGER PRIMARY KEY "
|
||||
"CHECK(singleton_id=1), revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(revision>=0))"
|
||||
),
|
||||
}
|
||||
_SEED_META = (
|
||||
f"INSERT OR IGNORE INTO {META_TABLE} (singleton_id, revision) VALUES (1, 0)"
|
||||
)
|
||||
_COLUMNS = {
|
||||
TABLE: (
|
||||
("locator", "TEXT", 1, 1),
|
||||
("size", "INTEGER", 1, 0),
|
||||
("mtime_ns", "INTEGER", 1, 0),
|
||||
("removed_at", "TEXT", 1, 0),
|
||||
),
|
||||
META_TABLE: (
|
||||
("singleton_id", "INTEGER", 0, 1),
|
||||
("revision", "INTEGER", 1, 0),
|
||||
),
|
||||
}
|
||||
_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection, allow_missing: bool) -> str | None:
|
||||
rows = conn.execute(
|
||||
"SELECT type,name,tbl_name,sql FROM sqlite_master WHERE name IN (?,?) "
|
||||
"OR tbl_name IN (?,?)",
|
||||
TABLES + TABLES,
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return None if allow_missing else "Missing image removal marks schema"
|
||||
tables = {name: sql for kind, name, _tbl, sql in rows if kind == "table"}
|
||||
others = {(kind, name) for kind, name, _tbl, _sql in rows if kind != "table"}
|
||||
if set(tables) != set(TABLES) or others - {("index", _AUTOINDEX)}:
|
||||
return "Image removal marks name is reserved by an unsupported schema object"
|
||||
for table in TABLES:
|
||||
if _compact(tables[table]) != _compact(_TABLE_DDL[table]):
|
||||
return "Image removal marks have an invalid table definition"
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({table})")
|
||||
)
|
||||
if columns != _COLUMNS[table]:
|
||||
return "Image removal marks have invalid columns"
|
||||
return None
|
||||
|
||||
|
||||
def _values_error(conn: sqlite3.Connection) -> str | None:
|
||||
limit = MAX_LOCATOR_LENGTH * 4
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {TABLE} WHERE typeof(locator) != 'text' " # nosec B608
|
||||
f"OR length(CAST(locator AS BLOB)) > {limit} OR locator = '' "
|
||||
"OR typeof(size) != 'integer' OR typeof(mtime_ns) != 'integer' "
|
||||
"OR typeof(removed_at) != 'text' OR length(CAST(removed_at AS BLOB)) > 64 "
|
||||
"LIMIT 1"
|
||||
).fetchone():
|
||||
return "Image removal marks contain invalid values"
|
||||
meta = conn.execute(
|
||||
f"SELECT singleton_id, typeof(revision) FROM {META_TABLE}" # nosec B608
|
||||
).fetchall()
|
||||
if meta != [(1, "integer")]:
|
||||
return "Image removal marks have an invalid revision"
|
||||
return None
|
||||
|
||||
|
||||
def image_removals_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> str | None:
|
||||
"""Reject unsupported schemas and invalid persisted values (reads only)."""
|
||||
error = _structure_error(conn, allow_missing)
|
||||
if error:
|
||||
return error
|
||||
exists = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone()
|
||||
return _values_error(conn) if exists else None
|
||||
|
||||
|
||||
def migrate_image_removals(conn: sqlite3.Connection) -> None:
|
||||
"""Install both tables atomically; validate before and after."""
|
||||
conn.execute("SAVEPOINT image_removals_schema")
|
||||
try:
|
||||
error = image_removals_schema_error(conn)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
for sql in _TABLE_DDL.values():
|
||||
conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
conn.execute(_SEED_META)
|
||||
error = image_removals_schema_error(conn, allow_missing=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE image_removals_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO image_removals_schema")
|
||||
conn.execute("RELEASE image_removals_schema")
|
||||
raise
|
||||
|
||||
|
||||
_UNMARK_SQL = f"DELETE FROM {TABLE} WHERE locator=?" # nosec B608
|
||||
_MARK_SQL = (
|
||||
f"INSERT OR REPLACE INTO {TABLE} " # nosec B608
|
||||
"(locator, size, mtime_ns, removed_at) VALUES (?, ?, ?, ?)"
|
||||
)
|
||||
_MARK_OF_SQL = f"SELECT size, mtime_ns FROM {TABLE} WHERE locator=?" # nosec B608
|
||||
BUMP_MARKS_SQL = (
|
||||
f"UPDATE {META_TABLE} SET revision=revision+1 WHERE singleton_id=1" # nosec B608
|
||||
)
|
||||
|
||||
|
||||
# Bumped whenever this process writes or clears a mark (even in a transaction
|
||||
# that later rolls back): readers that cache the marks reload when it moves.
|
||||
_generation = 0
|
||||
|
||||
|
||||
def generation() -> int:
|
||||
"""The marks generation of this process (see ``_generation``)."""
|
||||
return _generation
|
||||
|
||||
|
||||
def _bump() -> None:
|
||||
global _generation
|
||||
_generation += 1
|
||||
|
||||
|
||||
def has_marks_table(conn: sqlite3.Connection) -> bool:
|
||||
return (
|
||||
conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def locator_of_key(root_key: str, relative_key: str) -> str:
|
||||
"""The locator of a favorite-style file key (root key + relative key)."""
|
||||
return os.path.normcase(os.path.join(root_key, *relative_key.split("/")))
|
||||
|
||||
|
||||
def mark_of(conn: sqlite3.Connection, locator: str):
|
||||
"""``(size, mtime_ns)`` of the file's removal mark, or None."""
|
||||
if not has_marks_table(conn):
|
||||
return None
|
||||
row = conn.execute(_MARK_OF_SQL, (locator,)).fetchone()
|
||||
return (row[0], row[1]) if row else None
|
||||
|
||||
|
||||
def write_mark(
|
||||
conn: sqlite3.Connection, locator: str, size: int, mtime_ns: int, now: str
|
||||
):
|
||||
conn.execute(_MARK_SQL, (locator, int(size), int(mtime_ns), now))
|
||||
conn.execute(BUMP_MARKS_SQL)
|
||||
_bump()
|
||||
|
||||
|
||||
def clear_marks(conn: sqlite3.Connection, locators) -> int:
|
||||
"""Delete the marks of *locators* on *conn* (no commit); the count removed.
|
||||
|
||||
Every record write for a file calls this (#480): a file with records is
|
||||
in the library, so it is never hidden.
|
||||
"""
|
||||
locators = list(dict.fromkeys(locators))
|
||||
if not locators or not has_marks_table(conn):
|
||||
return 0
|
||||
removed = 0
|
||||
for locator in locators:
|
||||
removed += conn.execute(_UNMARK_SQL, (locator,)).rowcount
|
||||
if removed:
|
||||
conn.execute(BUMP_MARKS_SQL)
|
||||
_bump()
|
||||
return removed
|
||||
|
||||
|
||||
def clear_marks_of_keys(conn: sqlite3.Connection, file_keys) -> int:
|
||||
"""``clear_marks`` for favorite-style ``(root key, relative key)`` pairs."""
|
||||
return clear_marks(conn, [locator_of_key(*key) for key in file_keys])
|
||||
+97
-31
@@ -5,7 +5,9 @@ import sqlite3
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
from .library_folders import folder_condition, list_library_folders
|
||||
from .library_tokens import library_edit_token
|
||||
from .prompt_archive import ARCHIVED, SHELVED
|
||||
|
||||
_ELIGIBLE = """(EXISTS(SELECT 1 FROM prompt_pairs ep WHERE ep.positive_prompt_id=p.id)
|
||||
OR EXISTS(SELECT 1 FROM generated_images eg WHERE eg.prompt_id=p.id)
|
||||
@@ -105,6 +107,12 @@ ARCHIVE_SCOPES = {
|
||||
}
|
||||
|
||||
|
||||
# Shelves (#443): the parents on the shelf, or the archived ones.
|
||||
PARENT_SCOPES = {"active": SHELVED, "archived": ARCHIVED}
|
||||
# A positive pair owner that is on the shelf, as a condition on alias pp.
|
||||
SHELVED_PAIR = SHELVED.replace("p.id", "pp.positive_prompt_id")
|
||||
|
||||
|
||||
def _escape(value: str) -> str:
|
||||
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
@@ -121,6 +129,9 @@ def _archive(value: str) -> str:
|
||||
return ARCHIVE_SCOPES[value]
|
||||
|
||||
|
||||
# How the Tags filter matches (#444): every tag (AND), any of them (OR), or
|
||||
# parents without any tag.
|
||||
TAG_MODES = ("all", "any", "untagged")
|
||||
# A parent that no tag's Browse returns (#444 "Untagged prompts"): no tag of its
|
||||
# own and no active pair carrying one. The vocabulary's untagged count uses it too.
|
||||
UNTAGGED = (
|
||||
@@ -128,8 +139,6 @@ UNTAGGED = (
|
||||
"AND NOT EXISTS(SELECT 1 FROM prompt_pairs up JOIN prompt_pair_tags upt "
|
||||
"ON upt.pair_id=up.id WHERE up.positive_prompt_id=p.id AND up.archived=0)"
|
||||
)
|
||||
# How the Tags filter matches: every tag (AND), or parents without any tag.
|
||||
TAG_MODES = ("all", "untagged")
|
||||
|
||||
|
||||
def _tag_mode(tag_mode: object, tags: list[str]) -> str:
|
||||
@@ -140,8 +149,33 @@ def _tag_mode(tag_mode: object, tags: list[str]) -> str:
|
||||
return tag_mode
|
||||
|
||||
|
||||
def _tag_conditions(tags: list[str], any_tag: bool) -> tuple[list, list, list]:
|
||||
"""Parent-arm and pair-arm tag conditions and the tags' parameters."""
|
||||
parent_exists = (
|
||||
"EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.prompt_id=p.id AND t.name{test})"
|
||||
)
|
||||
pair_exists = (
|
||||
"EXISTS(SELECT 1 FROM prompt_pair_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.pair_id=pp.id AND t.name{test})"
|
||||
)
|
||||
if any_tag and tags:
|
||||
# One condition per arm: the card, or one active pair, carries any tag.
|
||||
test = " IN (" + ",".join("?" for _ in tags) + ")"
|
||||
return [parent_exists.format(test=test)], [pair_exists.format(test=test)], tags
|
||||
return (
|
||||
[parent_exists.format(test="=?") for _ in tags],
|
||||
[pair_exists.format(test="=?") for _ in tags],
|
||||
tags,
|
||||
)
|
||||
|
||||
|
||||
def _filter(
|
||||
text: str, category: str | None, tags: list[str], tag_mode: str = "all"
|
||||
text: str,
|
||||
category: str | None,
|
||||
tags: list[str],
|
||||
tag_mode: str = "all",
|
||||
folder_prefixes: list[str] | None = None,
|
||||
) -> tuple[str, list]:
|
||||
if not isinstance(text, str) or len(text) > 500:
|
||||
raise ValueError("Search text must be a string of at most 500 characters")
|
||||
@@ -153,7 +187,8 @@ def _filter(
|
||||
or any(not isinstance(tag, str) or not tag or len(tag) > 200 for tag in tags)
|
||||
):
|
||||
raise ValueError("Invalid library tags")
|
||||
untagged = _tag_mode(tag_mode, tags) == "untagged"
|
||||
mode = _tag_mode(tag_mode, tags)
|
||||
untagged = mode == "untagged"
|
||||
parent, pair, parent_params, pair_params = [], ["pp.archived=0"], [], []
|
||||
if text:
|
||||
parent.append("(p.text LIKE ? ESCAPE '\\' OR ls.title LIKE ? ESCAPE '\\')")
|
||||
@@ -168,17 +203,11 @@ def _filter(
|
||||
pair.append("pp.category=?")
|
||||
parent_params.append(category)
|
||||
pair_params.append(category)
|
||||
for tag in tags:
|
||||
parent.append(
|
||||
"EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.prompt_id=p.id AND t.name=?)"
|
||||
)
|
||||
pair.append(
|
||||
"EXISTS(SELECT 1 FROM prompt_pair_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.pair_id=pp.id AND t.name=?)"
|
||||
)
|
||||
parent_params.append(tag)
|
||||
pair_params.append(tag)
|
||||
parent_tags, pair_tags, tag_params = _tag_conditions(tags, mode == "any")
|
||||
parent.extend(parent_tags)
|
||||
pair.extend(pair_tags)
|
||||
parent_params.extend(tag_params)
|
||||
pair_params.extend(tag_params)
|
||||
condition = (
|
||||
"("
|
||||
+ (" AND ".join(parent) or "1=1")
|
||||
@@ -194,7 +223,13 @@ def _filter(
|
||||
# A whole-card condition: inside the parent-or-pair OR an untagged pair
|
||||
# would let a tagged parent through.
|
||||
condition = f"({condition}) AND {UNTAGGED}"
|
||||
return f"({_ELIGIBLE}) AND ({condition})", [*parent_params, *pair_params]
|
||||
params = [*parent_params, *pair_params]
|
||||
if folder_prefixes:
|
||||
# A whole-card condition around both arms: images link to the parent.
|
||||
folder, folder_params = folder_condition(folder_prefixes)
|
||||
condition = f"({condition}) AND {folder}"
|
||||
params.extend(folder_params)
|
||||
return f"({_ELIGIBLE}) AND ({condition})", params
|
||||
|
||||
|
||||
class LibraryOperations:
|
||||
@@ -242,6 +277,8 @@ class LibraryOperations:
|
||||
rows = conn.execute(
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,p.created_at,"
|
||||
"p.run_count,p.last_used_at,"
|
||||
+ ARCHIVED
|
||||
+ " AS archived,"
|
||||
+ (f"{sort_expression} AS _sort," if sort_expression else "")
|
||||
+ "(SELECT COUNT(*) FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
|
||||
"AND v.archived=0) AS variant_count,"
|
||||
@@ -265,6 +302,7 @@ class LibraryOperations:
|
||||
# Usage is shown, never compared: the token hashes owner fields only.
|
||||
row["run_count"] = row["run_count"] if type(row["run_count"]) is int else 0
|
||||
row["last_used_at"] = utc_iso(row["last_used_at"])
|
||||
row["archived"] = bool(row["archived"])
|
||||
row["edit_token"] = library_edit_token(epoch, row)
|
||||
return result
|
||||
|
||||
@@ -278,14 +316,20 @@ class LibraryOperations:
|
||||
category: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
tag_mode: str = "all",
|
||||
folder_prefixes: list[str] | None = None,
|
||||
sort: str = "newest",
|
||||
limit: int = 40,
|
||||
after: dict | None = None,
|
||||
shelf: str = "active",
|
||||
) -> dict:
|
||||
_limit(limit)
|
||||
if sort not in _ORDERS:
|
||||
raise ValueError("Unknown library sort order")
|
||||
where, params = _filter(text, category, [] if tags is None else tags, tag_mode)
|
||||
if not isinstance(shelf, str) or shelf not in PARENT_SCOPES:
|
||||
raise ValueError("shelf must be active or archived")
|
||||
where, params = _filter(
|
||||
text, category, [] if tags is None else tags, tag_mode, folder_prefixes
|
||||
)
|
||||
if after is not None:
|
||||
if (
|
||||
not isinstance(after, dict)
|
||||
@@ -297,15 +341,18 @@ class LibraryOperations:
|
||||
ceiling = after["ceiling"]
|
||||
where += " AND p.id<=?"
|
||||
params.append(ceiling)
|
||||
# One SELECT counts the filtered parents and their active variants.
|
||||
total, variant_total = conn.execute(
|
||||
"SELECT COUNT(*),COALESCE(SUM((SELECT COUNT(*) FROM prompt_pairs v "
|
||||
"WHERE v.positive_prompt_id=p.id AND v.archived=0)),0) FROM "
|
||||
+ _FROM
|
||||
+ " WHERE "
|
||||
+ where,
|
||||
# One SELECT counts the filtered parents of this scope, their active
|
||||
# variants, and the filtered archived parents (Show archived, #443).
|
||||
scope = PARENT_SCOPES[shelf]
|
||||
total, variant_total, archived_total = conn.execute(
|
||||
f"SELECT COALESCE(SUM({scope}),0),"
|
||||
f"COALESCE(SUM(CASE WHEN {scope} THEN (SELECT COUNT(*) "
|
||||
"FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
|
||||
"AND v.archived=0) ELSE 0 END),0),"
|
||||
f"COALESCE(SUM({ARCHIVED}),0) FROM " + _FROM + " WHERE " + where,
|
||||
params,
|
||||
).fetchone()
|
||||
where += f" AND {scope}"
|
||||
expression, direction = _ORDERS[sort]
|
||||
if after is not None:
|
||||
operator = ">" if direction == "ASC" else "<"
|
||||
@@ -338,6 +385,7 @@ class LibraryOperations:
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"variant_total": variant_total,
|
||||
"archived_total": archived_total,
|
||||
# Echoed so a client can tell this read applied its tag match.
|
||||
"tag_mode": tag_mode,
|
||||
"has_more": has_more,
|
||||
@@ -351,9 +399,11 @@ class LibraryOperations:
|
||||
category: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
tag_mode: str = "all",
|
||||
folder_prefixes: list[str] | None = None,
|
||||
sort: str = "newest",
|
||||
limit: int = 40,
|
||||
after: dict | None = None,
|
||||
shelf: str = "active",
|
||||
) -> dict:
|
||||
with self.library_snapshot(include_ceiling=True) as (conn, epoch, ceiling):
|
||||
return self._list_library(
|
||||
@@ -364,9 +414,11 @@ class LibraryOperations:
|
||||
category=category,
|
||||
tags=tags,
|
||||
tag_mode=tag_mode,
|
||||
folder_prefixes=folder_prefixes,
|
||||
sort=sort,
|
||||
limit=limit,
|
||||
after=after,
|
||||
shelf=shelf,
|
||||
)
|
||||
|
||||
def _is_library_positive(self, conn: sqlite3.Connection, positive_id: int) -> bool:
|
||||
@@ -494,9 +546,13 @@ class LibraryOperations:
|
||||
"categories": [row["category"] for row in rows],
|
||||
}
|
||||
|
||||
def get_library_options(self) -> dict:
|
||||
def get_library_options(self, folder_roots=None) -> dict:
|
||||
"""Categories; with *folder_roots* (root spellings) also ``folders``."""
|
||||
with self.library_snapshot() as (conn, epoch, _ceiling):
|
||||
return self._get_library_options(conn, epoch)
|
||||
result = self._get_library_options(conn, epoch)
|
||||
if folder_roots is not None:
|
||||
result["folders"] = list_library_folders(conn, folder_roots, _ELIGIBLE)
|
||||
return result
|
||||
|
||||
def get_library_previews(self, positive_ids: list[int]) -> dict[int, list[dict]]:
|
||||
"""Read at most two links per parent in one bounded page query."""
|
||||
@@ -607,7 +663,9 @@ class LibraryOperations:
|
||||
owners = {}
|
||||
order = []
|
||||
for row in conn.execute(
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes FROM "
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,"
|
||||
+ ARCHIVED
|
||||
+ " AS archived FROM "
|
||||
+ _FROM
|
||||
+ " WHERE "
|
||||
+ _ELIGIBLE
|
||||
@@ -620,6 +678,7 @@ class LibraryOperations:
|
||||
"tags": [],
|
||||
"rating": row["rating"],
|
||||
"notes": row["notes"],
|
||||
"archived": bool(row["archived"]),
|
||||
"variants": [],
|
||||
}
|
||||
order.append(row["id"])
|
||||
@@ -664,8 +723,9 @@ class LibraryOperations:
|
||||
"""System › Overview counts from one snapshot, writing nothing.
|
||||
|
||||
``positive_prompts`` and ``negative_variants`` are the unfiltered
|
||||
Library's ``total`` and ``variant_total``: eligible parents and active
|
||||
pairs (every pair's positive is eligible). ``images`` counts indexed
|
||||
Library's ``total`` and ``variant_total``: eligible parents on the
|
||||
shelf (not archived, #443) and their active pairs (every pair's
|
||||
positive is eligible). ``images`` counts indexed
|
||||
``generated_images`` rows, not files on disk. An image has no rating
|
||||
of its own (it shows its linked prompt's), so ``rated_images`` counts
|
||||
the indexed images whose linked prompt is rated. ``review_pending`` is
|
||||
@@ -673,8 +733,14 @@ class LibraryOperations:
|
||||
"""
|
||||
with self.library_snapshot() as (conn, _epoch, _ceiling):
|
||||
row = conn.execute(
|
||||
"SELECT (SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE + "),"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs WHERE archived=0),"
|
||||
"SELECT (SELECT COUNT(*) FROM prompts p WHERE "
|
||||
+ _ELIGIBLE
|
||||
+ " AND "
|
||||
+ SHELVED
|
||||
+ "),"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs pp WHERE pp.archived=0 AND "
|
||||
+ SHELVED_PAIR
|
||||
+ "),"
|
||||
"(SELECT COUNT(*) FROM generated_images),"
|
||||
"(SELECT COUNT(*) FROM generated_images gi JOIN prompts r "
|
||||
"ON r.id=gi.prompt_id WHERE r.rating IS NOT NULL),"
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
"""Delete selected Library prompts; archive the paired ones (#443, D152).
|
||||
|
||||
A selected Library parent is classified, in this order, as ``not_found``
|
||||
(missing, or not a Library parent), ``protected`` (tagged ``__protected__``;
|
||||
never changed), ``already_archived``, ``archive`` (the positive or negative of
|
||||
any saved pair, archived pair or not: archiving hides it from the shelf and
|
||||
removes nothing) or ``delete`` (in no pair: deleted exactly as the legacy
|
||||
delete does, through ``delete_prompt_rows``). Every command checks the
|
||||
storage epoch first and runs in one ``BEGIN IMMEDIATE`` transaction.
|
||||
|
||||
The write executes what the user reviewed: it is given the reviewed archive
|
||||
and delete lists, and deletes only reviewed deletes the route backed up
|
||||
(``candidates``) whose fingerprint is unchanged since that backup.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
from typing import Iterable
|
||||
|
||||
from .library import _ELIGIBLE
|
||||
from .library_commands import _validate_bulk_ids, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .prompt_archive import ARCHIVED as _ARCHIVED
|
||||
from .prompt_archive import TABLE as ARCHIVE
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
# Ids bound per statement: the pair check binds each id twice, which stays
|
||||
# under SQLite's historical 999-variable limit (maintenance.py ID_BATCH).
|
||||
ID_BATCH = 400
|
||||
_PROTECTED = (
|
||||
"EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.prompt_id=p.id AND t.name='__protected__')"
|
||||
)
|
||||
_PAIRED = (
|
||||
"EXISTS(SELECT 1 FROM prompt_pairs pp "
|
||||
"WHERE pp.positive_prompt_id=p.id OR pp.negative_prompt_id=p.id)"
|
||||
)
|
||||
# Classes in the order a prompt is tested; the first that holds wins.
|
||||
CLASSES = ("not_found", "protected", "already_archived", "archive", "delete")
|
||||
|
||||
|
||||
def _batches(ids: list) -> Iterable[list]:
|
||||
for start in range(0, len(ids), ID_BATCH):
|
||||
yield ids[start : start + ID_BATCH]
|
||||
|
||||
|
||||
def delete_prompt_rows(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
|
||||
"""Delete prompts and their image link rows; return the prompts deleted.
|
||||
|
||||
The legacy delete's statements (``delete_prompt``, ``bulk_delete_prompts``)
|
||||
in the caller's transaction: image link rows first (``image_prompt_pairs``
|
||||
cascades, review decisions keep a NULL image), then the prompts (tags and
|
||||
Library state cascade), then the workflow documents no row references.
|
||||
Image files are never touched. The caller checks pairs first.
|
||||
"""
|
||||
documents = documents_of_rows(conn, "prompt_id", prompt_ids)
|
||||
deleted = 0
|
||||
for chunk in _batches(list(prompt_ids)):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
conn.execute(
|
||||
f"DELETE FROM generated_images WHERE prompt_id IN ({marks})", # nosec
|
||||
chunk,
|
||||
)
|
||||
deleted += conn.execute(
|
||||
f"DELETE FROM prompts WHERE id IN ({marks})", chunk # nosec B608
|
||||
).rowcount
|
||||
release_documents(conn, documents)
|
||||
return deleted
|
||||
|
||||
|
||||
class LibraryArchiveOperations:
|
||||
"""PromptDatabase mixin: classify, archive or delete, and restore parents."""
|
||||
|
||||
def _classify_library_delete(
|
||||
self, conn: sqlite3.Connection, prompt_ids: list[int]
|
||||
) -> dict:
|
||||
found = {}
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for row in conn.execute(
|
||||
f"SELECT p.id,{_PROTECTED},{_ARCHIVED},{_PAIRED} " # nosec B608
|
||||
f"FROM prompts p WHERE p.id IN ({marks}) AND {_ELIGIBLE}",
|
||||
chunk,
|
||||
):
|
||||
protected, archived, paired = row[1], row[2], row[3]
|
||||
if protected:
|
||||
found[row[0]] = "protected"
|
||||
elif archived:
|
||||
found[row[0]] = "already_archived"
|
||||
else:
|
||||
found[row[0]] = "archive" if paired else "delete"
|
||||
classes = {name: [] for name in CLASSES}
|
||||
for pid in prompt_ids:
|
||||
classes[found.get(pid, "not_found")].append(pid)
|
||||
return classes
|
||||
|
||||
@staticmethod
|
||||
def _image_links(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
|
||||
total = 0
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
total += conn.execute(
|
||||
"SELECT COUNT(*) FROM generated_images " # nosec B608
|
||||
f"WHERE prompt_id IN ({marks})",
|
||||
chunk,
|
||||
).fetchone()[0]
|
||||
return total
|
||||
|
||||
@staticmethod
|
||||
def _fingerprints(conn: sqlite3.Connection, prompt_ids: list[int]) -> dict:
|
||||
"""What a safety backup holds of each prompt a delete would remove.
|
||||
|
||||
The prompt row (usage included), its tag ids, its Library title and
|
||||
its image link rows (count and newest id), as a JSON-safe list.
|
||||
"""
|
||||
prints = {}
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for row in conn.execute(
|
||||
"SELECT p.id,p.text,p.hash,p.category,p.rating,p.notes," # nosec
|
||||
"p.updated_at,p.run_count,p.last_used_at,"
|
||||
"(SELECT group_concat(tag_id) FROM (SELECT tag_id FROM prompt_tags "
|
||||
"WHERE prompt_id=p.id ORDER BY tag_id)),"
|
||||
"(SELECT title FROM prompt_library_state WHERE prompt_id=p.id),"
|
||||
"(SELECT COUNT(*) FROM generated_images WHERE prompt_id=p.id),"
|
||||
"(SELECT MAX(id) FROM generated_images WHERE prompt_id=p.id) "
|
||||
f"FROM prompts p WHERE p.id IN ({marks})",
|
||||
chunk,
|
||||
):
|
||||
prints[row[0]] = [
|
||||
value if isinstance(value, (int, str, type(None))) else repr(value)
|
||||
for value in tuple(row)[1:]
|
||||
]
|
||||
return prints
|
||||
|
||||
@staticmethod
|
||||
def _checked_ids(prompt_ids: list[int], expected_epoch: str) -> list[int]:
|
||||
_validate_epoch(expected_epoch)
|
||||
_validate_bulk_ids(prompt_ids)
|
||||
return list(dict.fromkeys(prompt_ids))
|
||||
|
||||
def classify_library_delete(
|
||||
self, prompt_ids: list[int], *, expected_epoch: str
|
||||
) -> dict:
|
||||
"""What a delete of ``prompt_ids`` would do now (reads only).
|
||||
|
||||
Returns the id lists per class, in request order, ``image_links``
|
||||
(the image link rows the ``delete`` class would remove) and
|
||||
``fingerprints`` of the ``delete`` class (``_fingerprints``).
|
||||
"""
|
||||
prompt_ids = self._checked_ids(prompt_ids, expected_epoch)
|
||||
with self.library_snapshot() as (conn, _epoch, _ceiling):
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
classes = self._classify_library_delete(conn, prompt_ids)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
**classes,
|
||||
"image_links": self._image_links(conn, classes["delete"]),
|
||||
"fingerprints": self._fingerprints(conn, classes["delete"]),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _repair_hash(conn: sqlite3.Connection, prompt_id: int) -> None:
|
||||
"""Give an archived prompt its canonical hash, so ingestion finds it.
|
||||
|
||||
Scans, the monitor and node saves resolve prompts by hash; a stale or
|
||||
missing one would make them insert a visible copy (Q3). When another
|
||||
row already holds the canonical hash, lookups find that row and no new
|
||||
copy is made, so nothing is written.
|
||||
"""
|
||||
text, stored = conn.execute(
|
||||
"SELECT text,hash FROM prompts WHERE id=?", (prompt_id,)
|
||||
).fetchone()
|
||||
canonical = generate_prompt_hash(text) if isinstance(text, str) else None
|
||||
if canonical is None or stored == canonical:
|
||||
return
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM prompts WHERE hash=? AND id!=?", (canonical, prompt_id)
|
||||
).fetchone():
|
||||
return
|
||||
conn.execute("UPDATE prompts SET hash=? WHERE id=?", (canonical, prompt_id))
|
||||
|
||||
def archive_or_delete_library_prompts(
|
||||
self,
|
||||
archive_ids: list[int],
|
||||
delete_ids: list[int],
|
||||
*,
|
||||
expected_epoch: str,
|
||||
candidates: dict | None,
|
||||
) -> dict:
|
||||
"""Archive and delete what the user reviewed, all or nothing.
|
||||
|
||||
``archive_ids`` and ``delete_ids`` are the reviewed classes.
|
||||
``candidates`` maps each reviewed delete the route backed up to its
|
||||
fingerprint then (None: every reviewed delete, unchecked; tests and
|
||||
callers without a backup). A reviewed delete is deleted only when it
|
||||
still classifies ``delete`` and its fingerprint is unchanged; one that
|
||||
is now paired, and a reviewed archive now unpaired, are archived (the
|
||||
restorable direction); anything else is kept and counted.
|
||||
"""
|
||||
_validate_epoch(expected_epoch)
|
||||
for ids in (archive_ids, delete_ids):
|
||||
if not isinstance(ids, list) or any(
|
||||
type(pid) is not int or not 1 <= pid <= 2**63 - 1 for pid in ids
|
||||
):
|
||||
raise ValueError("prompt ids must be lists of positive integers")
|
||||
reviewed_delete = set(delete_ids)
|
||||
if not archive_ids and not delete_ids:
|
||||
raise ValueError("Nothing to delete or archive")
|
||||
if reviewed_delete & set(archive_ids):
|
||||
raise ValueError("A prompt cannot be both archived and deleted")
|
||||
prompt_ids = list(dict.fromkeys([*archive_ids, *delete_ids]))
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
classes = self._classify_library_delete(conn, prompt_ids)
|
||||
now_prints = self._fingerprints(conn, classes["delete"])
|
||||
deletes, archives, changed = [], list(classes["archive"]), 0
|
||||
for pid in classes["delete"]:
|
||||
if pid not in reviewed_delete:
|
||||
archives.append(pid) # reviewed as archive, now unpaired
|
||||
elif candidates is None or candidates.get(pid) == now_prints[pid]:
|
||||
deletes.append(pid)
|
||||
else:
|
||||
changed += 1
|
||||
changed += len(classes["protected"])
|
||||
image_links = self._image_links(conn, deletes)
|
||||
now = utc_now_iso()
|
||||
for pid in archives:
|
||||
self._repair_hash(conn, pid)
|
||||
conn.executemany(
|
||||
f"INSERT INTO {ARCHIVE}(prompt_id, archived_at) VALUES (?, ?)",
|
||||
[(pid, now) for pid in archives],
|
||||
)
|
||||
# A changed row is the newest edit the restore fingerprint reads.
|
||||
conn.executemany(
|
||||
"UPDATE prompts SET updated_at=? WHERE id=?",
|
||||
[(now, pid) for pid in archives],
|
||||
)
|
||||
deleted = delete_prompt_rows(conn, deletes)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"deleted": deleted,
|
||||
"archived": len(archives),
|
||||
"already_archived": len(classes["already_archived"]),
|
||||
"not_found": len(classes["not_found"]),
|
||||
"changed": changed,
|
||||
"image_links": image_links,
|
||||
}
|
||||
|
||||
def restore_library_prompts(
|
||||
self, prompt_ids: list[int], *, expected_epoch: str
|
||||
) -> dict:
|
||||
"""Put archived prompts back on the shelf, all or nothing.
|
||||
|
||||
Every requested id with an archive row is restored, whatever its
|
||||
protection or Library parent state: restoring only un-hides it.
|
||||
"""
|
||||
prompt_ids = self._checked_ids(prompt_ids, expected_epoch)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
restored, existing = [], set()
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for pid, archived in conn.execute(
|
||||
f"SELECT p.id,{_ARCHIVED} FROM prompts p " # nosec B608
|
||||
f"WHERE p.id IN ({marks})",
|
||||
chunk,
|
||||
):
|
||||
existing.add(pid)
|
||||
if archived:
|
||||
restored.append(pid)
|
||||
now = utc_now_iso()
|
||||
conn.executemany(
|
||||
f"DELETE FROM {ARCHIVE} WHERE prompt_id=?", [(pid,) for pid in restored]
|
||||
)
|
||||
conn.executemany(
|
||||
"UPDATE prompts SET updated_at=? WHERE id=?",
|
||||
[(now, pid) for pid in restored],
|
||||
)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"restored": len(restored),
|
||||
"unchanged": len(existing) - len(restored),
|
||||
"not_found": len(prompt_ids) - len(existing),
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Library output-folder filter (#444): folder names, root prefixes and listing.
|
||||
|
||||
A folder is a ``/``-joined path relative to an output root, as Images' folder
|
||||
filter and the legacy ``/prompt_manager/subfolders`` name them. A parent
|
||||
matches a folder when one of its linked images (``generated_images.prompt_id``,
|
||||
the link its previews and image count read) lies in that folder or below it,
|
||||
under any spelling of any output root.
|
||||
|
||||
Matching compares ``generated_images.file_path``, which is always
|
||||
``normcase(normpath(abspath(path)))`` (``models.normalize_image_path``), with
|
||||
prefixes built by the same normalization here, so case folding is the
|
||||
platform's own on both sides (Windows folds non-ASCII too, unlike SQLite's
|
||||
``lower()``) and the comparison is an exact ``substr`` with no wildcards.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
_MAX_FOLDER = 1024
|
||||
_MAX_PREFIXES = 64
|
||||
|
||||
|
||||
def normalize_folder(value: object, pathmod=os.path) -> str:
|
||||
"""The folder as ``a/b``, outer slashes dropped; empty means no filter.
|
||||
|
||||
``/`` always separates. ``\\`` separates only on Windows (``ntpath``): on
|
||||
POSIX it is an ordinary character of a folder's name, and the listing
|
||||
(``relative_folder``) names such a folder with it, so both agree.
|
||||
``.``/``..``/empty segments, control characters and (on Windows) a drive
|
||||
are refused: the value only ever names a folder below a root.
|
||||
"""
|
||||
if not isinstance(value, str) or len(value) > _MAX_FOLDER:
|
||||
raise ValueError(f"folder must be text of at most {_MAX_FOLDER} characters")
|
||||
windows = pathmod.sep == "\\"
|
||||
folder = (value.replace("\\", "/") if windows else value).strip("/")
|
||||
if not folder:
|
||||
return ""
|
||||
parts = folder.split("/")
|
||||
if (
|
||||
any(part in ("", ".", "..") for part in parts)
|
||||
or (windows and ":" in parts[0])
|
||||
or any(ord(char) < 32 or ord(char) == 127 for char in folder)
|
||||
):
|
||||
raise ValueError("folder must be a folder below an output root")
|
||||
return folder
|
||||
|
||||
|
||||
def _base(root: object, pathmod) -> str:
|
||||
return pathmod.normcase(pathmod.normpath(str(root)))
|
||||
|
||||
|
||||
def folder_prefixes(roots, folder: str, pathmod=os.path) -> list[str]:
|
||||
"""One ``file_path`` prefix per root spelling for a normalized folder."""
|
||||
prefixes = []
|
||||
for root in roots:
|
||||
joined = pathmod.join(_base(root, pathmod), *folder.split("/"))
|
||||
prefix = pathmod.normcase(pathmod.normpath(joined)) + pathmod.sep
|
||||
if prefix not in prefixes:
|
||||
prefixes.append(prefix)
|
||||
if not prefixes or len(prefixes) > _MAX_PREFIXES:
|
||||
raise ValueError("No output folder to filter by")
|
||||
return prefixes
|
||||
|
||||
|
||||
def folder_condition(prefixes: list[str]) -> tuple[str, list]:
|
||||
"""A condition on alias ``p``: one linked image under any prefix."""
|
||||
tests = " OR ".join("substr(fg.file_path,1,?)=?" for _ in prefixes)
|
||||
params = [value for prefix in prefixes for value in (len(prefix), prefix)]
|
||||
return (
|
||||
"EXISTS(SELECT 1 FROM generated_images fg WHERE fg.prompt_id=p.id "
|
||||
f"AND ({tests}))",
|
||||
params,
|
||||
)
|
||||
|
||||
|
||||
def relative_folder(image_path: str, roots, pathmod=os.path) -> str | None:
|
||||
"""The folder of *image_path* below the first root spelling holding it.
|
||||
|
||||
The spelling of the stored path is kept (its case on Windows); ``None``
|
||||
for an image directly in a root or outside every root.
|
||||
"""
|
||||
directory = pathmod.normpath(pathmod.dirname(image_path))
|
||||
folded = pathmod.normcase(directory)
|
||||
for root in roots:
|
||||
base = _base(root, pathmod)
|
||||
start = base if base.endswith(pathmod.sep) else base + pathmod.sep
|
||||
if not folded.startswith(start):
|
||||
continue
|
||||
# normcase keeps the length for ordinary names; else show the folded form.
|
||||
rest = (directory if len(directory) == len(folded) else folded)[len(start) :]
|
||||
parts = rest.split(pathmod.sep)
|
||||
if any(part in ("", ".", "..") for part in parts):
|
||||
return None
|
||||
return "/".join(parts)
|
||||
return None
|
||||
|
||||
|
||||
def list_library_folders(
|
||||
conn: sqlite3.Connection, roots, eligible: str, pathmod=os.path
|
||||
) -> list[str]:
|
||||
"""Every folder (and ancestor) holding an image of an eligible parent."""
|
||||
folders = set()
|
||||
for (image_path,) in conn.execute(
|
||||
"SELECT DISTINCT g.image_path FROM generated_images g "
|
||||
"JOIN prompts p ON p.id=g.prompt_id WHERE " + eligible
|
||||
):
|
||||
if not isinstance(image_path, str) or not image_path:
|
||||
continue
|
||||
folder = relative_folder(image_path, roots, pathmod)
|
||||
if folder is None:
|
||||
continue
|
||||
parts = folder.split("/")
|
||||
folders.update("/".join(parts[:index]) for index in range(1, len(parts) + 1))
|
||||
return sorted(folders, key=lambda value: (value.casefold(), value))
|
||||
+163
-26
@@ -15,6 +15,8 @@ change:
|
||||
``cleanup_orphaned_images``), counted only under a source that can be read
|
||||
now, as the health check's ``missing_files`` judges them
|
||||
(``health.link_states``). It deletes database rows only, never a file.
|
||||
It also forgets the removal marks (#480) that hide nothing any more: the
|
||||
file is gone or changed, under a readable source (#501, ``_stale_marks``).
|
||||
|
||||
``preview_maintenance`` reads one snapshot on its own ``mode=ro`` connection
|
||||
(``health.readonly_uri``) and closes it in ``finally``; files are looked at
|
||||
@@ -41,9 +43,16 @@ from pathlib import Path
|
||||
|
||||
from .execution_contexts import LIVE_HINT_CTES
|
||||
from .health import BATCH_ROWS, link_states, readable_roots, readonly_uri
|
||||
from .image_removal_schema import TABLE as MARKS_TABLE
|
||||
from .image_removal_schema import clear_marks, has_marks_table
|
||||
from .pairs import PairDependencyError
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
try:
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
except ImportError:
|
||||
from utils.hashing import register_prompt_functions
|
||||
|
||||
MAINTENANCE_OPERATIONS = (
|
||||
"duplicate_prompts",
|
||||
"optimize",
|
||||
@@ -80,6 +89,12 @@ PROTECTIONS = (
|
||||
"with Library state (a title or an explicit parent)",
|
||||
"with Library state (a title or an explicit parent)",
|
||||
),
|
||||
(
|
||||
# #443: an archived prompt is restorable, never pruned.
|
||||
"EXISTS (SELECT 1 FROM prompt_archive pa WHERE pa.prompt_id = p.id)",
|
||||
"archived in the Library",
|
||||
"archived in the Library",
|
||||
),
|
||||
(
|
||||
"p.id IN (SELECT prompt_id FROM live_hint_prompts)",
|
||||
"used by recorded runs",
|
||||
@@ -88,7 +103,8 @@ PROTECTIONS = (
|
||||
)
|
||||
|
||||
# Exempt references: images, pairs, the __protected__ tag, Library parent
|
||||
# state, and live recorded-run hints (computed once, see LIVE_HINT_CTES).
|
||||
# state, the Library archive, and live recorded-run hints (computed once,
|
||||
# see LIVE_HINT_CTES).
|
||||
ORPHANED_PROMPTS_SQL = (
|
||||
"WITH "
|
||||
+ LIVE_HINT_CTES
|
||||
@@ -182,6 +198,8 @@ class MaintenanceDraft:
|
||||
protected: dict
|
||||
media: dict
|
||||
details: list = field(default_factory=list)
|
||||
# missing_links only: locators of removal marks whose file is gone (#501).
|
||||
marks: list = field(default_factory=list)
|
||||
|
||||
def public(self):
|
||||
"""The plan fields a response may carry (no fingerprint, no targets)."""
|
||||
@@ -284,11 +302,96 @@ def _delete_ids(conn, table, ids):
|
||||
return removed
|
||||
|
||||
|
||||
def _mark_rows(conn, only=None):
|
||||
"""``[(locator, size, mtime_ns)]`` of the removal marks (all, or those
|
||||
among *only*), sorted by locator."""
|
||||
if not has_marks_table(conn):
|
||||
return []
|
||||
sql = f"SELECT locator, size, mtime_ns FROM {MARKS_TABLE}" # nosec B608
|
||||
if only is None:
|
||||
return sorted(tuple(row) for row in conn.execute(sql))
|
||||
found = []
|
||||
for batch in _batches(sorted(only)):
|
||||
found.extend(
|
||||
tuple(row)
|
||||
for row in conn.execute(
|
||||
f"{sql} WHERE locator IN ({_marks(batch)})", batch # nosec B608
|
||||
)
|
||||
)
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def _lists_something(folder):
|
||||
"""Whether *folder* holds at least one entry (an unplugged drive's mount
|
||||
point, or an unbound volume, is an empty folder that reads fine)."""
|
||||
try:
|
||||
with os.scandir(folder) as entries:
|
||||
return next(entries, None) is not None
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _name_listed(path):
|
||||
"""Whether *path*'s folder lists its name, compared under ``normcase``.
|
||||
|
||||
A locator is ``normcase``d; on a case-sensitive NTFS folder ``lstat`` of
|
||||
the lower-cased spelling fails for a file that is there.
|
||||
"""
|
||||
folder, name = os.path.split(path)
|
||||
wanted = os.path.normcase(name)
|
||||
try:
|
||||
with os.scandir(folder) as entries:
|
||||
return any(os.path.normcase(entry.name) == wanted for entry in entries)
|
||||
except OSError:
|
||||
return True # cannot tell: keep the mark
|
||||
|
||||
|
||||
def _stale_mark(locator, size, mtime_ns):
|
||||
"""Whether the file of a mark is gone or has other bytes (blocking)."""
|
||||
try:
|
||||
info = os.lstat(locator)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return os.path.isdir(os.path.dirname(locator)) and not _name_listed(locator)
|
||||
except OSError:
|
||||
return False
|
||||
return (info.st_size, info.st_mtime_ns) != (size, mtime_ns)
|
||||
|
||||
|
||||
def _stale_marks(marks, roots):
|
||||
"""``[(locator, owner)]`` of the marks that hide nothing any more (#501).
|
||||
|
||||
*marks* are ``(locator, size, mtime_ns)`` rows. A mark is stale when its
|
||||
file was changed, or is gone from a folder that still exists. Only marks
|
||||
under a source that can be read now and lists something are judged (an
|
||||
unplugged drive's marks, or its empty mount point's, stay); any doubt
|
||||
keeps the mark. *owner* is the index of the longest source holding it.
|
||||
Locators are ``normcase(realpath)`` paths, so the sources are too.
|
||||
"""
|
||||
readable = readable_roots(roots)
|
||||
prefixes = [
|
||||
os.path.normcase(os.path.realpath(root)).rstrip("\\/") + os.sep
|
||||
for root in roots
|
||||
]
|
||||
judged = {}
|
||||
stale = []
|
||||
for locator, size, mtime_ns in marks:
|
||||
owners = [i for i, prefix in enumerate(prefixes) if locator.startswith(prefix)]
|
||||
if not owners:
|
||||
continue # under no source now (a source removed from the settings)
|
||||
owner = max(owners, key=lambda i: len(prefixes[i]))
|
||||
if owner not in judged:
|
||||
judged[owner] = readable[owner] and _lists_something(roots[owner])
|
||||
if judged[owner] and _stale_mark(locator, size, mtime_ns):
|
||||
stale.append((locator, owner))
|
||||
return stale
|
||||
|
||||
|
||||
def _read_snapshot(db_path, read):
|
||||
"""``read(conn)`` on one ``mode=ro`` snapshot; the connection always closes."""
|
||||
conn = sqlite3.connect(readonly_uri(db_path), uri=True)
|
||||
try:
|
||||
conn.row_factory = sqlite3.Row
|
||||
register_prompt_functions(conn) # LIVE_HINT_CTES calls them (#460)
|
||||
conn.execute("BEGIN") # one read snapshot for the whole plan
|
||||
return read(conn)
|
||||
finally:
|
||||
@@ -425,8 +528,11 @@ class MaintenanceOperations:
|
||||
if operation == "workflow_storage":
|
||||
return self._plan_workflow_storage()
|
||||
if operation == "missing_links":
|
||||
rows = _read_snapshot(self.model.db_path, self._link_rows)
|
||||
return self._plan_missing_links(rows, roots)
|
||||
rows, marks = _read_snapshot(
|
||||
self.model.db_path,
|
||||
lambda conn: (self._link_rows(conn), _mark_rows(conn)),
|
||||
)
|
||||
return self._plan_missing_links(rows, roots, _stale_marks(marks, roots))
|
||||
reader = (
|
||||
self._plan_duplicate_prompts
|
||||
if operation == "duplicate_prompts"
|
||||
@@ -557,13 +663,15 @@ class MaintenanceOperations:
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _links_fingerprint(candidates):
|
||||
def _links_fingerprint(candidates, stale=()):
|
||||
fingerprint = _Fingerprint("missing_links")
|
||||
for row, owner in candidates:
|
||||
fingerprint.add([row[0], row[1], row[3], owner])
|
||||
for locator, owner in stale:
|
||||
fingerprint.add(["mark", locator, owner])
|
||||
return fingerprint.hexdigest()
|
||||
|
||||
def _plan_missing_links(self, rows, roots):
|
||||
def _plan_missing_links(self, rows, roots, stale=()):
|
||||
readable = readable_roots(roots)
|
||||
candidates, unavailable, outside = [], 0, 0
|
||||
for row, owner, state in link_states(rows, roots, readable):
|
||||
@@ -591,29 +699,47 @@ class MaintenanceOperations:
|
||||
)
|
||||
)
|
||||
count = len(candidates)
|
||||
parts = []
|
||||
if count:
|
||||
parts.append(
|
||||
_plural(count, "link to a missing file", "links to missing files")
|
||||
)
|
||||
if stale:
|
||||
parts.append(
|
||||
_plural(
|
||||
len(stale),
|
||||
"removed image whose file is gone or changed",
|
||||
"removed images whose files are gone or changed",
|
||||
)
|
||||
)
|
||||
details = [
|
||||
{
|
||||
"label": _relative(row[1], roots[owner]),
|
||||
"detail": f"{_source_name(roots[owner], owner)} · the file is missing",
|
||||
}
|
||||
for row, owner in candidates
|
||||
] + [
|
||||
{
|
||||
"label": _relative(locator, os.path.realpath(roots[owner])),
|
||||
"detail": (
|
||||
f"{_source_name(roots[owner], owner)} · removed image, "
|
||||
"the file is gone or changed"
|
||||
),
|
||||
}
|
||||
for locator, owner in stale
|
||||
]
|
||||
return MaintenanceDraft(
|
||||
operation="missing_links",
|
||||
fingerprint=self._links_fingerprint(candidates),
|
||||
fingerprint=self._links_fingerprint(candidates, stale),
|
||||
targets=[row[0] for row, _owner in candidates],
|
||||
scope={
|
||||
"count": count,
|
||||
"label": (
|
||||
_plural(count, "link to a missing file", "links to missing files")
|
||||
if count
|
||||
else "No links to missing files"
|
||||
),
|
||||
"count": count + len(stale),
|
||||
"label": "; ".join(parts) or "No links to missing files",
|
||||
},
|
||||
protected={"count": unavailable + outside, "reasons": reasons},
|
||||
media={"deletes_files": False, "label": LINKS_ONLY},
|
||||
details=[
|
||||
{
|
||||
"label": _relative(row[1], roots[owner]),
|
||||
"detail": (
|
||||
f"{_source_name(roots[owner], owner)} · the file is missing"
|
||||
),
|
||||
}
|
||||
for row, owner in candidates[:MAX_DETAILS]
|
||||
],
|
||||
details=details[:MAX_DETAILS],
|
||||
marks=[locator for locator, _owner in stale],
|
||||
)
|
||||
|
||||
def _plan_optimize(self):
|
||||
@@ -663,14 +789,18 @@ class MaintenanceOperations:
|
||||
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
def apply_maintenance(self, operation, fingerprint, targets, roots=()) -> dict:
|
||||
def apply_maintenance(
|
||||
self, operation, fingerprint, targets, roots=(), marks=()
|
||||
) -> dict:
|
||||
"""Apply a reviewed plan exactly, or raise PlanChanged (blocking).
|
||||
|
||||
Inside ``BEGIN IMMEDIATE`` the fingerprint is computed again on the
|
||||
write connection; when it differs nothing is written. ``targets`` and
|
||||
``roots`` are the plan's. Returns ``{changed, rows}``: how many
|
||||
records changed and up to ``MAX_RESULT_ROWS`` descriptions of them
|
||||
(``{prompt_id[, kept_id]}`` or ``{image_path}``).
|
||||
(``{prompt_id[, kept_id]}`` or ``{image_path}``). ``marks`` are a
|
||||
missing-links plan's stale removal marks; that apply also returns
|
||||
``marks``, how many it forgot.
|
||||
"""
|
||||
if operation not in MAINTENANCE_OPERATIONS or operation == "workflow_storage":
|
||||
raise ValueError(f"Unknown maintenance operation {operation!r}")
|
||||
@@ -683,9 +813,12 @@ class MaintenanceOperations:
|
||||
appliers = {
|
||||
"duplicate_prompts": self._apply_duplicate_prompts,
|
||||
"unused_components": self._apply_unused_components,
|
||||
"missing_links": self._apply_missing_links,
|
||||
}
|
||||
with self._pair_transaction() as conn:
|
||||
if operation == "missing_links":
|
||||
return self._apply_missing_links(
|
||||
conn, fingerprint, targets, roots, list(marks)
|
||||
)
|
||||
return appliers[operation](conn, fingerprint, targets, roots)
|
||||
|
||||
def _apply_duplicate_prompts(self, conn, fingerprint, targets, _roots):
|
||||
@@ -714,7 +847,7 @@ class MaintenanceOperations:
|
||||
]
|
||||
return {"changed": changed, "rows": rows}
|
||||
|
||||
def _apply_missing_links(self, conn, fingerprint, targets, roots):
|
||||
def _apply_missing_links(self, conn, fingerprint, targets, roots, marks):
|
||||
rows = []
|
||||
for batch in _batches(sorted(targets)):
|
||||
rows.extend(
|
||||
@@ -731,10 +864,14 @@ class MaintenanceOperations:
|
||||
if state != "missing": # back, or its source went away
|
||||
raise PlanChanged("missing_links")
|
||||
candidates.append((row, owner))
|
||||
if self._links_fingerprint(candidates) != fingerprint:
|
||||
stale = _stale_marks(_mark_rows(conn, marks) if marks else [], roots)
|
||||
if [locator for locator, _owner in stale] != marks: # a mark went or is valid
|
||||
raise PlanChanged("missing_links")
|
||||
if self._links_fingerprint(candidates, stale) != fingerprint:
|
||||
raise PlanChanged("missing_links")
|
||||
changed = _delete_ids(conn, "generated_images", [row[0] for row in rows])
|
||||
return {
|
||||
"changed": changed,
|
||||
"marks": clear_marks(conn, marks),
|
||||
"rows": [{"image_path": row[1]} for row in rows[:MAX_RESULT_ROWS]],
|
||||
}
|
||||
|
||||
@@ -38,7 +38,14 @@ from .review_schema import (
|
||||
review_without_annotations_error,
|
||||
)
|
||||
from .saved_views import migrate_saved_views, saved_views_schema_error
|
||||
|
||||
try:
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
except ImportError:
|
||||
from utils.hashing import register_prompt_functions
|
||||
from .image_categories import image_categories_schema_error, migrate_image_categories
|
||||
from .image_removal_schema import image_removals_schema_error, migrate_image_removals
|
||||
from .prompt_archive import migrate_prompt_archive, prompt_archive_schema_error
|
||||
from .workflow_documents import (
|
||||
DOCUMENT_ID_COLUMNS,
|
||||
migrate_workflow_documents,
|
||||
@@ -153,12 +160,15 @@ def configure_connection(conn: sqlite3.Connection) -> None:
|
||||
(common on Windows) let other writers' polling busy handlers starve past
|
||||
BUSY_TIMEOUT_SECONDS and fail with "database is locked" (#242). Without
|
||||
WAL (for example an in-memory database) SQLite's FULL default is kept.
|
||||
It also registers the prompt-key SQL functions (register_prompt_functions).
|
||||
"""
|
||||
journal_mode = conn.execute("PRAGMA journal_mode = WAL").fetchone()[0]
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}")
|
||||
if str(journal_mode).lower() == "wal":
|
||||
conn.execute("PRAGMA synchronous = NORMAL")
|
||||
# History reads and the live-hint rule call Python's prompt key (#460)
|
||||
register_prompt_functions(conn)
|
||||
|
||||
|
||||
class PromptModel:
|
||||
@@ -358,6 +368,10 @@ class PromptModel:
|
||||
migrate_autotag_state(conn)
|
||||
# Image categories (#332): independent of every table above.
|
||||
migrate_image_categories(conn)
|
||||
# The Library parent archive (#443): independent of every table above.
|
||||
migrate_prompt_archive(conn)
|
||||
# Removal marks (#480): independent of every table above.
|
||||
migrate_image_removals(conn)
|
||||
# Last: the older generated_images rebuilds run only before file_path.
|
||||
migrate_workflow_documents(conn)
|
||||
|
||||
@@ -1234,6 +1248,16 @@ class PromptModel:
|
||||
)
|
||||
if categories_error:
|
||||
return False, categories_error
|
||||
archive_error = prompt_archive_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if archive_error:
|
||||
return False, archive_error
|
||||
removals_error = image_removals_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if removals_error:
|
||||
return False, removals_error
|
||||
views_error = saved_views_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
|
||||
+11
-8
@@ -18,12 +18,14 @@ from .models import PromptModel, normalize_image_path, utc_now_iso
|
||||
from .pairs import PairDependencyError, PairOperations # noqa: F401 (re-exported)
|
||||
from .library import LibraryOperations, titled
|
||||
from .image_favorites import ImageFavoriteOperations
|
||||
from .image_removal import ImageRemovalOperations, removed_from_library
|
||||
from .image_collections import ImageCollectionOperations
|
||||
from .annotations import AnnotationOperations
|
||||
from .review import ReviewOperations
|
||||
from .review_decisions import ReviewDecisionOperations
|
||||
from .saved_views import SavedViewOperations
|
||||
from .library_commands import LibraryCommandOperations
|
||||
from .library_archive import LibraryArchiveOperations, delete_prompt_rows
|
||||
from .vocabulary import VocabularyOperations
|
||||
from .execution_history import ExecutionHistoryOperations
|
||||
from .execution_contexts import ExecutionContextOperations
|
||||
@@ -448,6 +450,7 @@ class PromptDatabase(
|
||||
PairOperations,
|
||||
LibraryOperations,
|
||||
LibraryCommandOperations,
|
||||
LibraryArchiveOperations,
|
||||
VocabularyOperations,
|
||||
ExecutionHistoryOperations,
|
||||
ExecutionContextOperations,
|
||||
@@ -465,6 +468,7 @@ class PromptDatabase(
|
||||
WorkflowStorageOperations,
|
||||
CategoryImportOperations,
|
||||
ImageCategoryOperations,
|
||||
ImageRemovalOperations,
|
||||
):
|
||||
"""Database operations class for managing prompts."""
|
||||
|
||||
@@ -1806,6 +1810,11 @@ class PromptDatabase(
|
||||
"""
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
if removed_from_library(conn, image_path):
|
||||
# #480: the user removed this exact file from the library; its
|
||||
# bytes are unchanged, so nothing links it again.
|
||||
self.logger.debug("Not linking a file removed from the library")
|
||||
return 0
|
||||
filename = os.path.basename(image_path)
|
||||
file_path = normalize_image_path(image_path)
|
||||
file_size, width, height, image_format = _coerce_file_info(metadata)
|
||||
@@ -2344,16 +2353,10 @@ class PromptDatabase(
|
||||
Returns:
|
||||
Number of prompts actually deleted.
|
||||
"""
|
||||
count = 0
|
||||
with self._pair_transaction() as conn:
|
||||
self._ensure_prompts_unpaired(conn, prompt_ids)
|
||||
documents = documents_of_rows(conn, "prompt_id", prompt_ids)
|
||||
for pid in prompt_ids:
|
||||
conn.execute("DELETE FROM generated_images WHERE prompt_id = ?", (pid,))
|
||||
cursor = conn.execute("DELETE FROM prompts WHERE id = ?", (pid,))
|
||||
if cursor.rowcount > 0:
|
||||
count += 1
|
||||
release_documents(conn, documents)
|
||||
# The workspace Library's delete runs the same statements (#443).
|
||||
count = delete_prompt_rows(conn, list(dict.fromkeys(prompt_ids)))
|
||||
conn.commit()
|
||||
return count
|
||||
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Library parent archive (#443, D152): prompts hidden from the shelf, restorable.
|
||||
|
||||
A row means the Library parent is archived; no row means it is on the shelf.
|
||||
Nothing else about the prompt changes: its text, metadata, tags, run count,
|
||||
image links, pairs and recorded runs stay. Restore deletes the row. Deleting
|
||||
the prompt (any path) drops the row by cascade.
|
||||
|
||||
A table of its own, not a column: ``prompt_library_state`` is validated by its
|
||||
exact definition, so a new column there would make older builds refuse the
|
||||
upgraded database, while older builds ignore an unknown table (they show the
|
||||
archived prompts again and keep the rows). Optional in old backups, required
|
||||
once a database is current.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
TABLE = "prompt_archive"
|
||||
MAX_STAMP_BYTES = 64
|
||||
# What ``utc_now_iso`` writes starts with a date; anything else was not written here.
|
||||
_STAMP_GLOB = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T*"
|
||||
_DDL = (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
|
||||
"archived_at TEXT NOT NULL, "
|
||||
"FOREIGN KEY(prompt_id) REFERENCES prompts(id) ON DELETE CASCADE)"
|
||||
)
|
||||
_COLUMNS = (("prompt_id", "INTEGER", 0, 1), ("archived_at", "TEXT", 1, 0))
|
||||
_FOREIGN_KEYS = {("prompt_id", "prompts", "id", "CASCADE")}
|
||||
# A predicate on ``p`` (the prompts alias): the parent is not archived.
|
||||
SHELVED = f"NOT EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)"
|
||||
ARCHIVED = f"EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection, allow_missing: bool) -> str | None:
|
||||
objects = conn.execute(
|
||||
"SELECT type,sql FROM sqlite_master WHERE name=? "
|
||||
"OR (type IN ('index','trigger') AND tbl_name=?)",
|
||||
(TABLE, TABLE),
|
||||
).fetchall()
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing prompt archive schema"
|
||||
tables = [sql for kind, sql in objects if kind == "table"]
|
||||
if len(tables) != 1 or len(objects) != 1:
|
||||
return "Prompt archive name is reserved by an unsupported schema object"
|
||||
if _compact(tables[0]) != _compact(_DDL):
|
||||
return "Prompt archive has an invalid table definition"
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({TABLE})")
|
||||
)
|
||||
if columns != _COLUMNS:
|
||||
return "Prompt archive has invalid columns"
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({TABLE})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS:
|
||||
return "Prompt archive has invalid foreign key definitions"
|
||||
return None
|
||||
|
||||
|
||||
def prompt_archive_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> str | None:
|
||||
"""Reject an unsupported archive schema or values; reads only.
|
||||
|
||||
A missing table is acceptable for backups that predate it unless
|
||||
``allow_missing`` is false.
|
||||
"""
|
||||
error = _structure_error(conn, allow_missing)
|
||||
if error is not None:
|
||||
return error
|
||||
if not conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone():
|
||||
return None
|
||||
if conn.execute(f"PRAGMA foreign_key_check({TABLE})").fetchone():
|
||||
return "Prompt archive contains foreign key violations"
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {TABLE} WHERE typeof(archived_at)!='text' " # nosec B608
|
||||
f"OR length(CAST(archived_at AS BLOB))>{MAX_STAMP_BYTES} "
|
||||
f"OR archived_at NOT GLOB '{_STAMP_GLOB}' LIMIT 1"
|
||||
).fetchone():
|
||||
return "Prompt archive contains invalid values"
|
||||
return None
|
||||
|
||||
|
||||
def migrate_prompt_archive(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table atomically; validate before and after (no backfill)."""
|
||||
conn.execute("SAVEPOINT prompt_archive_schema")
|
||||
try:
|
||||
error = _structure_error(conn, True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
error = _structure_error(conn, False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE prompt_archive_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO prompt_archive_schema")
|
||||
conn.execute("RELEASE prompt_archive_schema")
|
||||
raise
|
||||
@@ -149,6 +149,7 @@ def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
|
||||
collections (#331).
|
||||
"""
|
||||
from .library import _ELIGIBLE
|
||||
from .prompt_archive import SHELVED, TABLE as ARCHIVE
|
||||
|
||||
has_state = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND "
|
||||
@@ -157,9 +158,15 @@ def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
|
||||
has_pairs = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='prompt_pairs'"
|
||||
).fetchone()
|
||||
has_archive = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (ARCHIVE,)
|
||||
).fetchone()
|
||||
if has_state and has_pairs:
|
||||
# Archived parents (#443) are off the shelf, as in library_counts.
|
||||
positives = conn.execute(
|
||||
"SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE
|
||||
"SELECT COUNT(*) FROM prompts p WHERE "
|
||||
+ _ELIGIBLE
|
||||
+ (" AND " + SHELVED if has_archive else "")
|
||||
).fetchone()[0]
|
||||
else:
|
||||
positives = _count(conn, "prompts")
|
||||
|
||||
@@ -33,8 +33,9 @@ MAX_TAG_LENGTH = 50
|
||||
_IMAGE_SORTS = ("newest", "oldest", "name", "name-desc", "rating")
|
||||
_LIBRARY_SORTS = ("newest", "oldest", "name", "rating")
|
||||
_MEDIA = ("all", "image", "video")
|
||||
# How a Library view's Tags match (#444): every tag, or parents without tags.
|
||||
_TAG_MODES = ("all", "untagged")
|
||||
# How a Library view's Tags match (#444): every tag, any of them, or parents
|
||||
# without tags ("untagged" never goes with tags; the listing refuses both).
|
||||
_TAG_MODES = ("all", "any", "untagged")
|
||||
_CONTROL = re.compile(r"[\x00-\x1f\x7f]")
|
||||
_SOURCE = re.compile(r"all|0|[1-9][0-9]{0,8}")
|
||||
# A collection id as the listing's ``collection`` query takes it (#331): none
|
||||
@@ -69,8 +70,9 @@ _STATE_FIELDS = {
|
||||
"category": ("text", 100, ""),
|
||||
"sort": ("choice", _LIBRARY_SORTS, "newest"),
|
||||
"tags": ("tags", None, ()),
|
||||
# #444: "untagged" never goes with tags (the listing refuses both).
|
||||
# #444: the Tags match and the output folder (relative, "/"-joined).
|
||||
"tag_mode": ("choice", _TAG_MODES, "all"),
|
||||
"folder": ("text", 1024, ""),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
+29
-12
@@ -20,7 +20,8 @@ counted, hashed into the token and renamed with the prompts and pairs.
|
||||
|
||||
import sqlite3
|
||||
|
||||
from .library import _ELIGIBLE, UNTAGGED, _escape, _limit
|
||||
from .library import _ELIGIBLE, SHELVED_PAIR, UNTAGGED, _escape, _limit
|
||||
from .prompt_archive import SHELVED
|
||||
from .library_commands import _TOKEN, LibraryCommandError, _validate_epoch
|
||||
from .library_tokens import canonical_token
|
||||
from .models import utc_now_iso
|
||||
@@ -40,17 +41,21 @@ _IMAGE_FILES = (
|
||||
|
||||
def _tag_rows(page: str, order: str = "page.id") -> str:
|
||||
"""Count rows for the registry rows selected by ``page`` (id, name)."""
|
||||
# Eligible parent components carrying a tag in the page, plus the
|
||||
# positive parent of every active pair carrying it (always eligible).
|
||||
# Eligible parent components on the shelf carrying a tag in the page,
|
||||
# plus the positive parent of every active pair carrying it (always
|
||||
# eligible) when that parent is on the shelf: archived parents (#443)
|
||||
# are hidden with their pairs, so Browse lists what was counted.
|
||||
return (
|
||||
"WITH page AS (" + page + "),"
|
||||
"owners(tag_id,pid) AS ("
|
||||
"SELECT pt.tag_id,p.id FROM prompts p "
|
||||
"JOIN prompt_tags pt ON pt.prompt_id=p.id "
|
||||
"WHERE pt.tag_id IN (SELECT id FROM page) AND " + _ELIGIBLE + " "
|
||||
"AND " + SHELVED + " "
|
||||
"UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id "
|
||||
"WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)),"
|
||||
"WHERE pp.archived=0 AND " + SHELVED_PAIR + " "
|
||||
"AND ppt.tag_id IN (SELECT id FROM page)),"
|
||||
"images(tag_id,n) AS (" + _IMAGE_FILES + " "
|
||||
"WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) "
|
||||
"SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count,"
|
||||
@@ -79,8 +84,10 @@ _RANKED_TAG_PAGE = (
|
||||
"WITH owners(tag_id,pid) AS ("
|
||||
"SELECT pt.tag_id,p.id FROM prompts p "
|
||||
"JOIN prompt_tags pt ON pt.prompt_id=p.id WHERE " + _ELIGIBLE + " "
|
||||
"AND " + SHELVED + " "
|
||||
"UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id WHERE pp.archived=0),"
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id WHERE pp.archived=0 "
|
||||
"AND " + SHELVED_PAIR + "),"
|
||||
"counted AS (SELECT t.id,t.name,COUNT(DISTINCT o.pid) AS prompt_count "
|
||||
"FROM tags t LEFT JOIN owners o ON o.tag_id=t.id WHERE {search} "
|
||||
"GROUP BY t.id),"
|
||||
@@ -94,7 +101,13 @@ _RANKED_TAG_PAGE = (
|
||||
"ORDER BY page.prompt_count {direction},page.id"
|
||||
)
|
||||
_UNTAGGED_COUNT = (
|
||||
"SELECT COUNT(*) FROM prompts p WHERE (" + _ELIGIBLE + ") AND " + UNTAGGED
|
||||
"SELECT COUNT(*) FROM prompts p WHERE ("
|
||||
+ _ELIGIBLE
|
||||
+ ") AND "
|
||||
+ UNTAGGED
|
||||
# Untagged prompts opens the shelf, which hides archived parents (#443).
|
||||
+ " AND "
|
||||
+ SHELVED
|
||||
)
|
||||
_TAG_ONE = _tag_rows("SELECT id,name FROM tags WHERE id=?")
|
||||
# Every owner kind a tag can sit on, for the delete confirmation.
|
||||
@@ -134,7 +147,9 @@ def _validate_tag_id(tag_id: object) -> None:
|
||||
|
||||
|
||||
# Category values in Tags & categories (#332 §7): the prompt-backed values
|
||||
# (nonblank values on eligible parents or on active pairs, grouped exactly, so
|
||||
# (nonblank values on eligible parents or on active pairs; ``prompt_count``
|
||||
# counts only parents on the shelf, so a value only archived parents (#443)
|
||||
# hold stays listed with 0, as a tag does; grouped exactly, so
|
||||
# whitespace and case variants stay distinct; /library/options lists the same
|
||||
# prompt-backed set with its own query) plus the values only image rows hold.
|
||||
# A prompt-backed value is ordered by its oldest owning positive; an image-only
|
||||
@@ -143,15 +158,17 @@ def _validate_tag_id(tag_id: object) -> None:
|
||||
# files missing on disk. No FULL OUTER JOIN, so it runs on SQLite 3.31+.
|
||||
_IMAGE_ONLY_FIRST_PID = 2**63 - 1
|
||||
_CATEGORY_ROWS = (
|
||||
"WITH owners(value,pid) AS ("
|
||||
"SELECT p.category,p.id FROM prompts p WHERE p.category IS NOT NULL "
|
||||
"AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT pp.category,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"WITH owners(value,pid,shelved) AS ("
|
||||
"SELECT p.category,p.id," + SHELVED + " FROM prompts p "
|
||||
"WHERE p.category IS NOT NULL AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT pp.category,pp.positive_prompt_id," + SHELVED_PAIR + " "
|
||||
"FROM prompt_pairs pp "
|
||||
"WHERE pp.archived=0 AND pp.category IS NOT NULL AND TRIM(pp.category)!=''),"
|
||||
"image_values(value,image_count) AS ("
|
||||
"SELECT category,COUNT(*) FROM image_categories GROUP BY category),"
|
||||
"grouped(value,first_pid,prompt_count) AS ("
|
||||
"SELECT value,MIN(pid),COUNT(DISTINCT pid) FROM owners GROUP BY value "
|
||||
"SELECT value,MIN(pid),COUNT(DISTINCT CASE WHEN shelved THEN pid END) "
|
||||
"FROM owners GROUP BY value "
|
||||
"UNION ALL SELECT value," + str(_IMAGE_ONLY_FIRST_PID) + ",0 FROM image_values "
|
||||
"WHERE value NOT IN (SELECT value FROM owners)),"
|
||||
"counted AS (SELECT g.value,g.first_pid,g.prompt_count,"
|
||||
|
||||
+39
-13
@@ -19,6 +19,7 @@ Split into domain-specific mixins for maintainability:
|
||||
- SystemRoutesMixin: System health check, Overview and usage statistics
|
||||
- LibraryLocationRoutesMixin: where the library lives, Move library, Keep it here
|
||||
- MaintenanceRoutesMixin: reviewed maintenance plans (preview, apply)
|
||||
- LibraryDeleteRoutesMixin: the Library's Delete… (archive paired) and Restore
|
||||
- WorkflowStorageRoutesMixin: store each workflow once, then Reclaim space
|
||||
- DuplicateFilesMixin: duplicate image files as find and removal jobs
|
||||
- LoraImportRoutesMixin: the reviewed, staged LoRA reference import job
|
||||
@@ -40,8 +41,10 @@ from aiohttp import web
|
||||
from PIL import Image
|
||||
|
||||
from .prompts import PromptRoutesMixin
|
||||
from .revalidation import revalidated_response
|
||||
from .pairs import PairRoutesMixin
|
||||
from .library import LibraryRoutesMixin
|
||||
from .library_delete import LibraryDeleteRoutesMixin
|
||||
from .vocabulary import VocabularyRoutesMixin
|
||||
from .annotations import AnnotationRoutesMixin
|
||||
from .review import ReviewRoutesMixin
|
||||
@@ -74,6 +77,7 @@ from .autotag_routes import AutotagRoutesMixin
|
||||
from .lora_integration import LoraIntegrationMixin
|
||||
from .lora_import_routes import LoraImportRoutesMixin
|
||||
from .image_categories import ImageCategoryRoutesMixin
|
||||
from .image_delete import ImageDeleteMixin
|
||||
from .source_rescope import SourceRescopeMixin
|
||||
|
||||
try:
|
||||
@@ -313,6 +317,7 @@ async def _gzip_middleware(request, handler):
|
||||
class PromptManagerAPI(
|
||||
PairRoutesMixin,
|
||||
LibraryRoutesMixin,
|
||||
LibraryDeleteRoutesMixin,
|
||||
VocabularyRoutesMixin,
|
||||
AnnotationRoutesMixin,
|
||||
ReviewRoutesMixin,
|
||||
@@ -343,6 +348,7 @@ class PromptManagerAPI(
|
||||
LoraIntegrationMixin,
|
||||
LoraImportRoutesMixin,
|
||||
ImageCategoryRoutesMixin,
|
||||
ImageDeleteMixin,
|
||||
SourceRescopeMixin,
|
||||
):
|
||||
"""REST API handler for PromptManager operations and web interface.
|
||||
@@ -423,6 +429,12 @@ class PromptManagerAPI(
|
||||
# Reviewed maintenance plans under single-use tokens; a restart
|
||||
# forgets them (py/api/maintenance.py).
|
||||
self._maintenance_plans = PlanStore()
|
||||
# #480: the removal marks the workspace listing hides, read once now so
|
||||
# a listing page reads no table until a mark or the database changes.
|
||||
try:
|
||||
self._removal_marks()
|
||||
except Exception:
|
||||
self.logger.warning("Removal marks could not be read", exc_info=True)
|
||||
|
||||
# A safety backup interrupted by a crash leaves its temp copy behind.
|
||||
try:
|
||||
@@ -653,40 +665,45 @@ class PromptManagerAPI(
|
||||
|
||||
@routes.get("/prompt_manager/workspace")
|
||||
async def serve_workspace_ui(request: web.Request) -> web.Response:
|
||||
return self._serve_html_page("workspace.html", "Workspace")
|
||||
return self._serve_html_page(request, "workspace.html", "Workspace")
|
||||
|
||||
@routes.get("/prompt_manager/web")
|
||||
async def serve_web_ui(request):
|
||||
return self._serve_html_page("index.html", "Web UI", cache=False)
|
||||
return self._serve_html_page(request, "index.html", "Web UI", cache=False)
|
||||
|
||||
@routes.get("/prompt_manager/gallery.html")
|
||||
async def serve_gallery_ui(request):
|
||||
return self._serve_html_page("metadata.html", "Gallery")
|
||||
return self._serve_html_page(request, "metadata.html", "Gallery")
|
||||
|
||||
@routes.get("/prompt_manager/admin")
|
||||
async def serve_admin_ui(request):
|
||||
return self._serve_html_page("admin.html", "Admin UI")
|
||||
return self._serve_html_page(request, "admin.html", "Admin UI")
|
||||
|
||||
@routes.get("/prompt_manager/gallery")
|
||||
async def serve_gallery_admin_ui(request):
|
||||
return self._serve_html_page("gallery.html", "Gallery")
|
||||
return self._serve_html_page(request, "gallery.html", "Gallery")
|
||||
|
||||
# ── Static file serving ───────────────────────────────────────
|
||||
|
||||
@routes.get("/prompt_manager/lib/{filepath:.*}")
|
||||
async def serve_lib_static(request):
|
||||
"""Serve static library files (JS, CSS) from web/lib directory."""
|
||||
return self._serve_static_file("lib", request.match_info.get("filepath"))
|
||||
return self._serve_static_file(
|
||||
request, "lib", request.match_info.get("filepath")
|
||||
)
|
||||
|
||||
@routes.get("/prompt_manager/js/{filepath:.*}")
|
||||
async def serve_js_static(request):
|
||||
"""Serve static JavaScript files from web/js directory."""
|
||||
return self._serve_static_file("js", request.match_info.get("filepath"))
|
||||
return self._serve_static_file(
|
||||
request, "js", request.match_info.get("filepath")
|
||||
)
|
||||
|
||||
# ── Register domain-specific routes from mixins ───────────────
|
||||
|
||||
self._register_pair_routes(routes)
|
||||
self._register_library_routes(routes)
|
||||
self._register_library_delete_routes(routes)
|
||||
self._register_vocabulary_routes(routes)
|
||||
self._register_annotation_routes(routes)
|
||||
self._register_review_routes(routes)
|
||||
@@ -710,6 +727,7 @@ class PromptManagerAPI(
|
||||
self._register_lora_routes(routes)
|
||||
self._register_lora_import_routes(routes)
|
||||
self._register_image_category_routes(routes)
|
||||
self._register_image_delete_routes(routes)
|
||||
self._register_rescope_routes(routes)
|
||||
|
||||
# Register gzip compression middleware (once)
|
||||
@@ -726,8 +744,11 @@ class PromptManagerAPI(
|
||||
|
||||
self.logger.info("All routes registered with decorator pattern")
|
||||
|
||||
def _serve_static_file(self, subdir, filepath):
|
||||
"""Serve ``web/<subdir>/<filepath>``; 403 outside it, 404 when missing."""
|
||||
def _serve_static_file(self, request, subdir, filepath):
|
||||
"""Serve ``web/<subdir>/<filepath>``; 403 outside it, 404 when missing.
|
||||
|
||||
A served file is revalidated on every reuse (#478, ``revalidation``).
|
||||
"""
|
||||
root = os.path.join(_get_project_root(), "web", subdir)
|
||||
exact = STATIC_EXACT_FILES.get((subdir, filepath))
|
||||
# An exact file passes the same containment check, its extension only.
|
||||
@@ -742,10 +763,13 @@ class PromptManagerAPI(
|
||||
return web.Response(text="Not Found", status=404)
|
||||
with open(file_path, "rb") as f:
|
||||
content = f.read()
|
||||
return web.Response(body=content, content_type=content_type)
|
||||
return revalidated_response(request, content, content_type)
|
||||
|
||||
def _serve_html_page(self, filename, title, cache=True):
|
||||
"""Serve ``web/<filename>``; 404 when missing, 500 when unreadable."""
|
||||
def _serve_html_page(self, request, filename, title, cache=True):
|
||||
"""Serve ``web/<filename>``; 404 when missing, 500 when unreadable.
|
||||
|
||||
A served page is revalidated on every reuse (#478, ``revalidation``).
|
||||
"""
|
||||
html_path = os.path.join(_get_project_root(), "web", filename)
|
||||
try:
|
||||
if cache and html_path in self._html_cache:
|
||||
@@ -762,7 +786,9 @@ class PromptManagerAPI(
|
||||
content_type="text/html",
|
||||
status=404,
|
||||
)
|
||||
return web.Response(text=content, content_type="text/html", charset="utf-8")
|
||||
return revalidated_response(
|
||||
request, content.encode("utf-8"), "text/html", charset="utf-8"
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception(f"Failed to load {title}")
|
||||
return web.Response(
|
||||
|
||||
+22
-7
@@ -36,10 +36,13 @@ except ImportError:
|
||||
|
||||
try:
|
||||
from ...utils import exit_signal
|
||||
from ...utils.media_folders import is_skipped_folder
|
||||
except ImportError:
|
||||
from utils import exit_signal
|
||||
from utils.media_folders import is_skipped_folder
|
||||
|
||||
try:
|
||||
from ...database.image_removal import mark_hides
|
||||
from ...database.pair_ingestion import SCAN_MARKS, ingest_resolved_pair
|
||||
from ...database.run_attribution import (
|
||||
RecordedOutput,
|
||||
@@ -60,6 +63,7 @@ try:
|
||||
decide_file_prompt,
|
||||
)
|
||||
except ImportError:
|
||||
from database.image_removal import mark_hides
|
||||
from database.pair_ingestion import SCAN_MARKS, ingest_resolved_pair
|
||||
from database.run_attribution import (
|
||||
RecordedOutput,
|
||||
@@ -118,10 +122,11 @@ DUPLICATE_FIND_LEGACY_TITLE = "Find duplicate image files (gallery page)"
|
||||
def _collect_media_files(output_dirs, extensions):
|
||||
"""Media files under ``output_dirs`` (blocking), bounded and symlink-free.
|
||||
|
||||
Walks each root without following symlinks, skips ``thumbnails``
|
||||
directories and symlinked files, never descends more than MAX_SCAN_DEPTH
|
||||
levels and stops after MAX_SCAN_FILES files in total. Extensions match
|
||||
case-insensitively and each path is listed once.
|
||||
Walks each root without following symlinks, skips ``thumbnails`` and
|
||||
trash folders (``utils.media_folders``) and symlinked files, never
|
||||
descends more than MAX_SCAN_DEPTH levels and stops after MAX_SCAN_FILES
|
||||
files in total. Extensions match case-insensitively and each path is
|
||||
listed once.
|
||||
"""
|
||||
wanted = {ext.lower() for ext in extensions}
|
||||
found = []
|
||||
@@ -136,7 +141,7 @@ def _collect_media_files(output_dirs, extensions):
|
||||
dirnames[:] = sorted(
|
||||
d
|
||||
for d in dirnames
|
||||
if d != "thumbnails"
|
||||
if not is_skipped_folder(d)
|
||||
and depth < MAX_SCAN_DEPTH
|
||||
and not os.path.islink(os.path.join(current, d))
|
||||
)
|
||||
@@ -883,6 +888,8 @@ class AdminRoutesMixin:
|
||||
extensions = IMAGE_EXTENSIONS + VIDEO_EXTENSIONS
|
||||
media_files = []
|
||||
seen = set()
|
||||
# #480: a file removed from the library is never a copy or a keeper.
|
||||
marks = self._removal_marks()
|
||||
for slot, root in enumerate(roots):
|
||||
for path in _collect_media_files([root], extensions):
|
||||
try:
|
||||
@@ -891,6 +898,9 @@ class AdminRoutesMixin:
|
||||
stat = os.stat(path)
|
||||
except OSError:
|
||||
continue
|
||||
locator = os.path.normcase(os.path.realpath(str(path)))
|
||||
if locator in marks and mark_hides(marks, locator, path):
|
||||
continue
|
||||
key = _physical_file_key(path, stat)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
@@ -1551,8 +1561,13 @@ class AdminRoutesMixin:
|
||||
if job.cancelled:
|
||||
return
|
||||
batch = media_files[batch_start : batch_start + SCAN_BATCH_SIZE]
|
||||
skipped = [f for f in batch if str(f) in known]
|
||||
to_read = [f for f in batch if str(f) not in known]
|
||||
# #480: a file removed from the library is not read or linked
|
||||
# again while its bytes are unchanged.
|
||||
removed = await self._run_in_executor(self._removed_paths, batch)
|
||||
skipped = [f for f in batch if str(f) in known or str(f) in removed]
|
||||
to_read = [
|
||||
f for f in batch if str(f) not in known and str(f) not in removed
|
||||
]
|
||||
outcomes = [_scan_outcome(f, roots, "skipped") for f in skipped]
|
||||
counts["processed"] += len(skipped)
|
||||
counts["unchanged"] += len(skipped)
|
||||
|
||||
@@ -23,6 +23,11 @@ from .prompts import thumbnail_url_for
|
||||
from . import thumbnail_settings
|
||||
from .. import model_gate
|
||||
|
||||
try:
|
||||
from ...utils.media_folders import is_skipped_folder
|
||||
except ImportError:
|
||||
from utils.media_folders import is_skipped_folder
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
_INTERNAL_ERROR = "An internal error occurred. Check server logs for details."
|
||||
@@ -1319,8 +1324,9 @@ class AutotagRoutesMixin:
|
||||
def _collect_scan_files(output_path: Path):
|
||||
"""Media files under ``output_path`` from a bounded, symlink-free walk.
|
||||
|
||||
Descends at most ``_SCAN_MAX_DEPTH`` levels, skips ``thumbnails``
|
||||
trees and symlinked files, and stops after ``_SCAN_MAX_FILES`` files.
|
||||
Descends at most ``_SCAN_MAX_DEPTH`` levels, skips ``thumbnails`` and
|
||||
trash folders (``utils.media_folders``) and symlinked files, and stops
|
||||
after ``_SCAN_MAX_FILES`` files.
|
||||
Returns ``(paths, truncated)``.
|
||||
"""
|
||||
found = []
|
||||
@@ -1330,7 +1336,7 @@ class AutotagRoutesMixin:
|
||||
dirnames[:] = sorted(
|
||||
name
|
||||
for name in dirnames
|
||||
if name != "thumbnails" and depth + 1 <= _SCAN_MAX_DEPTH
|
||||
if not is_skipped_folder(name) and depth + 1 <= _SCAN_MAX_DEPTH
|
||||
)
|
||||
for name in sorted(filenames):
|
||||
if Path(name).suffix.lower() not in _MEDIA_EXTENSIONS:
|
||||
@@ -1357,6 +1363,10 @@ class AutotagRoutesMixin:
|
||||
files, truncated = await self._run_in_executor(
|
||||
self._collect_scan_files, output_path
|
||||
)
|
||||
# #501: a file removed from the library is not offered for tagging.
|
||||
files = await self._run_in_executor(
|
||||
self._without_removed, output_path, files
|
||||
)
|
||||
has_thumbnails = (output_path / "thumbnails").is_dir()
|
||||
|
||||
images = []
|
||||
|
||||
+13
-1
@@ -27,7 +27,11 @@ from .history_current import (
|
||||
resolve_current_candidate,
|
||||
snapshot_current_output_scope,
|
||||
)
|
||||
from .inspection_service import CurrentFileChanged, inspect_current_file
|
||||
from .inspection_service import (
|
||||
CurrentFileChanged,
|
||||
image_tags_reader,
|
||||
inspect_current_file,
|
||||
)
|
||||
from .images import image_category_key
|
||||
|
||||
try:
|
||||
@@ -778,6 +782,14 @@ class HistoryRoutesMixin:
|
||||
),
|
||||
own,
|
||||
),
|
||||
# The file's own image tags, keyed as the Tag dialog
|
||||
# keys them: the normcased realpath (#457).
|
||||
image_tags=image_tags_reader(
|
||||
db,
|
||||
str(resolved.canonical_target),
|
||||
self.logger,
|
||||
close=True,
|
||||
),
|
||||
recorded=recorded_output_lookup(
|
||||
db, close=True, reference=receipt["reference"]
|
||||
),
|
||||
|
||||
@@ -0,0 +1,492 @@
|
||||
"""Delete selected images: remove them from the library, or trash them (#480).
|
||||
|
||||
``POST /prompt_manager/images/delete/preview {items | filter}`` reviews the
|
||||
selection for both modes at once and answers a reviewed plan (the System ›
|
||||
Maintenance contract, ``py/api/maintenance.py``): the token, the counts of
|
||||
the records each file loses, the refused files with a coded reason (both
|
||||
modes, and the trash-only ones), the prompts that lose their last image and
|
||||
the safety backup. It writes nothing. ``items`` are Images listing identities
|
||||
(at most 500); ``filter`` is "Select all N matching". Each target is matched
|
||||
to its entry in the current listing (``not-listed`` otherwise), so every
|
||||
later step uses the walked spelling, never the request's text.
|
||||
|
||||
``POST /prompt_manager/maintenance/apply {token, mode}`` runs the plan as the
|
||||
``image_delete`` job (admission ``records``, not cancellable): the sources
|
||||
are checked, the promised safety backup is taken, then each file is checked
|
||||
again (containment and fingerprint) and
|
||||
|
||||
- ``mode="library"``: its records are deleted and its removal mark written
|
||||
in one transaction; the file stays on disk, hidden;
|
||||
- ``mode="trash"``: it is moved to the trash first (``utils/trash.py``); a
|
||||
refused move leaves the file and its records alone; after a move its
|
||||
records are deleted.
|
||||
|
||||
No path here unlinks a media file. A failed records write stops the job; the
|
||||
files after it are not attempted.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat as stat_module
|
||||
from pathlib import Path
|
||||
|
||||
from .admin import DELETABLE_MEDIA_EXTENSIONS, _remove_thumbnail
|
||||
from .duplicate_review import (
|
||||
CHANGED_CONTENT,
|
||||
GONE,
|
||||
LINK,
|
||||
OUTSIDE_SOURCE,
|
||||
UNREADABLE,
|
||||
file_problem,
|
||||
)
|
||||
from .images import _favorite_relative_key, _favorite_root_key
|
||||
from .jobs import IMAGE_DELETE_GROUP_NAMES, IMAGE_DELETE_JOB_KIND, JobBusy
|
||||
from .job_routes import busy_response
|
||||
from .maintenance import MAINTENANCE_PREVIEW_LEASE, _fail_job, _failure, backup_note
|
||||
|
||||
try:
|
||||
from ...database import image_removal_schema
|
||||
from ...database.image_removal import mark_hides
|
||||
from ...database.annotation_schema import MAX_LOCATOR_LENGTH
|
||||
from ...utils import trash
|
||||
from ...utils.media_folders import is_trash_folder
|
||||
except ImportError:
|
||||
from database import image_removal_schema
|
||||
from database.image_removal import mark_hides
|
||||
from database.annotation_schema import MAX_LOCATOR_LENGTH
|
||||
from utils import trash
|
||||
from utils.media_folders import is_trash_folder
|
||||
|
||||
IMAGE_DELETE = "delete_images"
|
||||
MODES = ("library", "trash")
|
||||
TITLES = {
|
||||
"library": "Delete images (remove from the library)",
|
||||
"trash": "Delete images (move to the trash)",
|
||||
}
|
||||
MAX_DETAIL_ROWS = 50
|
||||
|
||||
# Reason codes (design §4.4) and the sentence each shows.
|
||||
REASONS = {
|
||||
"not-listed": "No longer listed in Images.",
|
||||
"outside": "Outside the output folders.",
|
||||
"link": "A link, not a file.",
|
||||
"gone": "The file no longer exists.",
|
||||
"unreadable": "The file can't be read.",
|
||||
"thumbnail": "A generated preview, not an image.",
|
||||
"not-media": "Not a media file.",
|
||||
"trash-folder": "Inside a trash folder.",
|
||||
"path-too-long": "The path is too long.",
|
||||
"changed": "The file changed after the review.",
|
||||
trash.NO_TRASH: "This drive has no trash.",
|
||||
trash.NETWORK: "Network drives can't use the trash.",
|
||||
trash.TRASH_IN_ROOT: "This drive's trash is inside an output folder.",
|
||||
trash.LOCKED: "The file is open in another program. Close it and try again.",
|
||||
trash.FAILED: "The file could not be moved to the trash.",
|
||||
trash.WINDOWS_RECYCLE: (
|
||||
"Moving files to the Recycle Bin isn't available on Windows: files larger "
|
||||
"than the bin's capacity would be deleted permanently."
|
||||
),
|
||||
"not-attempted": "Not attempted: the job stopped after an error.",
|
||||
}
|
||||
_PROBLEM_CODES = {
|
||||
OUTSIDE_SOURCE: "outside",
|
||||
LINK: "link",
|
||||
GONE: "gone",
|
||||
UNREADABLE: "unreadable",
|
||||
CHANGED_CONTENT: "changed",
|
||||
}
|
||||
SOURCES_CHANGED = "The sources changed since the review; nothing was changed."
|
||||
RECORDS_FAILED = "The image records could not be removed; the job stopped."
|
||||
|
||||
|
||||
def lexical_problem(relative: str) -> str | None:
|
||||
"""The refusal code a root-relative POSIX path earns before any file access."""
|
||||
parts = relative.split("/")
|
||||
if "thumbnails" in parts[:-1]:
|
||||
return "thumbnail"
|
||||
if any(is_trash_folder(part) for part in parts[:-1]):
|
||||
return "trash-folder"
|
||||
if os.path.splitext(parts[-1])[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||||
return "not-media"
|
||||
return None
|
||||
|
||||
|
||||
def file_check(path, root, relative, *, expected=None):
|
||||
"""``(code, lstat)``: why *path* may not be deleted, or ``(None, lstat)``.
|
||||
|
||||
Text first (inside the root, not a preview or trash folder, a media
|
||||
file), then ``duplicate_review.file_problem``: a regular file, not a
|
||||
link, whose resolved path is still inside the resolved root. With
|
||||
*expected* (``{size, mtime_ns, inode}`` from the review) the file must
|
||||
still be those bytes. Every handle is closed when this returns.
|
||||
"""
|
||||
code = lexical_problem(relative)
|
||||
if code:
|
||||
return code, None
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
return "gone", None
|
||||
except OSError:
|
||||
return "unreadable", None
|
||||
if getattr(info, "st_file_attributes", 0) & getattr(
|
||||
stat_module, "FILE_ATTRIBUTE_REPARSE_POINT", 0
|
||||
):
|
||||
return "link", info
|
||||
if expected is None:
|
||||
problem = file_problem(path, root, None, info.st_size, hash_check=False)
|
||||
else:
|
||||
problem = file_problem(
|
||||
path,
|
||||
root,
|
||||
None,
|
||||
expected["size"],
|
||||
hash_check=False,
|
||||
mtime_ns=expected["mtime_ns"],
|
||||
inode=expected["inode"],
|
||||
)
|
||||
if problem:
|
||||
return _PROBLEM_CODES.get(problem, "outside"), info
|
||||
if len(os.path.normcase(os.path.realpath(path))) > MAX_LOCATOR_LENGTH:
|
||||
return "path-too-long", info
|
||||
return None, info
|
||||
|
||||
|
||||
def _row(target, code):
|
||||
return {
|
||||
"path": target["rel"],
|
||||
"root_label": target["root_label"],
|
||||
"code": code,
|
||||
"detail": REASONS[code],
|
||||
}
|
||||
|
||||
|
||||
def _summary(rows):
|
||||
counts = {}
|
||||
for row in rows:
|
||||
counts[row["code"]] = counts.get(row["code"], 0) + 1
|
||||
return {"count": len(rows), "counts": counts, "rows": rows[:MAX_DETAIL_ROWS]}
|
||||
|
||||
|
||||
def _fingerprint(targets) -> str:
|
||||
material = [
|
||||
(t["root_index"], t["rel"], t["size"], t["mtime_ns"], t["inode"])
|
||||
for t in targets
|
||||
]
|
||||
return hashlib.sha256(json.dumps(material).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class ImageDeleteMixin:
|
||||
"""The Images Delete… review and its apply job (#480)."""
|
||||
|
||||
def _register_image_delete_routes(self, routes):
|
||||
# A fixed path beside the dynamic /images/{image_id} routes: aiohttp
|
||||
# (3.10+) looks fixed paths up first (see image_categories.py).
|
||||
@routes.post("/prompt_manager/images/delete/preview")
|
||||
async def preview_image_delete_route(request):
|
||||
return await self.preview_image_delete(request)
|
||||
|
||||
# ── review ────────────────────────────────────────────────────────
|
||||
|
||||
async def preview_image_delete(self, request):
|
||||
"""Review the selection under the maintenance preview lease."""
|
||||
try:
|
||||
async with self._jobs.lease(MAINTENANCE_PREVIEW_LEASE):
|
||||
return await self._preview_image_delete(request)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def _preview_image_delete(self, request):
|
||||
_data, targets, roots, error = await self._category_request(
|
||||
request, frozenset()
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
db = self.db
|
||||
identity = self._database_identity(db)
|
||||
try:
|
||||
files = {}
|
||||
for index in dict.fromkeys(t[0] for t in targets):
|
||||
files[index] = await self._get_gallery_files(roots[index])
|
||||
review = await self._run_in_executor(
|
||||
self._review_image_delete, db, roots, targets, files
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Image delete review failed")
|
||||
return _failure("The review failed. Check server logs for details.", 500)
|
||||
return await self._plan_response(
|
||||
db,
|
||||
identity,
|
||||
IMAGE_DELETE,
|
||||
_fingerprint(review["targets"]),
|
||||
review["public"],
|
||||
{"targets": review["targets"]},
|
||||
)
|
||||
|
||||
def _review_image_delete(self, db, roots, targets, files):
|
||||
"""Every target checked now (blocking); ``{targets, public}``."""
|
||||
listings, seen = {}, set()
|
||||
acted, refused, trash_refused = [], [], []
|
||||
spellings = self._source_spellings()
|
||||
root_texts = [str(root) for root in roots]
|
||||
for index, _scope, relative in targets:
|
||||
root = roots[index]
|
||||
if index not in listings:
|
||||
listings[index] = self._favorite_listing(root, files[index])
|
||||
spelled = listings[index].get(_favorite_relative_key(relative))
|
||||
target = {
|
||||
"root_index": index,
|
||||
"root_scope": _scope,
|
||||
"root": str(root),
|
||||
"root_label": self._public_path(root),
|
||||
"rel": spelled or relative,
|
||||
}
|
||||
if spelled is None:
|
||||
refused.append(_row(target, "not-listed"))
|
||||
continue
|
||||
absolute = os.path.join(str(root), *spelled.split("/"))
|
||||
code, info = file_check(absolute, str(root), spelled)
|
||||
if code:
|
||||
refused.append(_row(target, code))
|
||||
continue
|
||||
identity = (info.st_dev, info.st_ino)
|
||||
if identity in seen:
|
||||
continue # one physical file named twice (nested roots, hardlinks)
|
||||
seen.add(identity)
|
||||
target.update(self._removal_of(absolute, info, spellings))
|
||||
target["trash"] = trash.trash_available(absolute, root_texts)
|
||||
if target["trash"]:
|
||||
trash_refused.append(_row(target, target["trash"]))
|
||||
acted.append(target)
|
||||
counted = db.count_removal_records([t["removal"] for t in acted])
|
||||
library_count = len(acted)
|
||||
public = {
|
||||
"operation": IMAGE_DELETE,
|
||||
"scope": {"count": library_count, "label": "images"},
|
||||
"library": {"count": library_count},
|
||||
"trash": {"count": library_count - len(trash_refused)},
|
||||
"protected": {"count": len(refused), "reasons": []},
|
||||
"refused": _summary(refused),
|
||||
"trash_refused": _summary(trash_refused),
|
||||
"records": counted["records"],
|
||||
"orphaned_prompts": counted["orphaned_prompts"],
|
||||
"media": {"deletes_files": False, "label": "Chosen in the review"},
|
||||
"details": [],
|
||||
}
|
||||
return {"targets": acted, "public": public}
|
||||
|
||||
@staticmethod
|
||||
def _removal_of(absolute, info, spellings):
|
||||
favorites = [
|
||||
(_favorite_root_key(root), _favorite_relative_key(relative))
|
||||
for root, relative in spellings.favorites(absolute).items()
|
||||
]
|
||||
return {
|
||||
"abs": absolute,
|
||||
"size": info.st_size,
|
||||
"mtime_ns": info.st_mtime_ns,
|
||||
"inode": [info.st_dev, info.st_ino],
|
||||
"removal": {
|
||||
"spellings": list(spellings.of(absolute).values()),
|
||||
"locator": os.path.normcase(os.path.realpath(absolute)),
|
||||
"favorites": favorites,
|
||||
"category": spellings.category_key(absolute),
|
||||
},
|
||||
}
|
||||
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
def _start_image_delete(self, db, plan, mode, backup_on):
|
||||
"""Register the delete job for a reviewed plan (on the loop)."""
|
||||
return self._jobs.start(
|
||||
IMAGE_DELETE_JOB_KIND,
|
||||
TITLES[mode],
|
||||
admission="records",
|
||||
cancellable=False,
|
||||
scope={"operation": IMAGE_DELETE, "mode": mode},
|
||||
groups=IMAGE_DELETE_GROUP_NAMES,
|
||||
runner=lambda job: self._run_image_delete(job, db, plan, mode, backup_on),
|
||||
)
|
||||
|
||||
async def _run_image_delete(self, job, db, plan, mode, backup_on):
|
||||
targets = plan.payload["targets"]
|
||||
unchanged = await self._run_in_executor(self._sources_unchanged, targets)
|
||||
if not unchanged:
|
||||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return
|
||||
try:
|
||||
await self._delete_images(job, db, targets, mode, backup)
|
||||
finally:
|
||||
if backup:
|
||||
await self._prune_backup_pool(job, db)
|
||||
|
||||
async def _delete_images(self, job, db, targets, mode, backup):
|
||||
job.update(total=len(targets), status_line=TITLES[mode])
|
||||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||||
done_group = "trashed" if mode == "trash" else "removed"
|
||||
changed, failed = 0, False
|
||||
for done, target in enumerate(targets, start=1):
|
||||
if failed:
|
||||
self._record_delete(job, "not_changed", target, "not-attempted")
|
||||
continue
|
||||
group, code, label = await self._run_in_executor(
|
||||
self._delete_one_image, db, target, mode, roots
|
||||
)
|
||||
if group == "errors":
|
||||
failed = True
|
||||
job.record(
|
||||
"errors",
|
||||
{
|
||||
"path": target["rel"],
|
||||
"root": target["root_label"],
|
||||
"root_index": target["root_index"],
|
||||
"root_scope": target["root_scope"],
|
||||
"detail": RECORDS_FAILED,
|
||||
},
|
||||
f"! {target['root_label']}/{target['rel']} · {RECORDS_FAILED}",
|
||||
)
|
||||
elif group == done_group:
|
||||
changed += 1
|
||||
detail = (
|
||||
f"Moved to the {label}" if label else "Removed from the library"
|
||||
)
|
||||
self._record_delete(job, group, target, None, detail)
|
||||
else:
|
||||
self._record_delete(job, group, target, code)
|
||||
job.update(processed=done, progress=int(done / len(targets) * 100))
|
||||
if changed:
|
||||
self.invalidate_gallery_cache()
|
||||
verb = "Moved" if mode == "trash" else "Removed"
|
||||
where = "to the trash" if mode == "trash" else "from the library"
|
||||
line = f"{verb} {changed:,} of {len(targets):,} images {where}"
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
if failed:
|
||||
job.scope["failure"] = "records-failed"
|
||||
job.publish({"type": "error", "code": "records-failed", "message": line})
|
||||
job.finish("failed")
|
||||
|
||||
@staticmethod
|
||||
def _record_delete(job, group, target, code, detail=None):
|
||||
text = detail or REASONS[code]
|
||||
row = {
|
||||
"path": target["rel"],
|
||||
"root": target["root_label"],
|
||||
"root_index": target["root_index"],
|
||||
"root_scope": target["root_scope"],
|
||||
"code": code,
|
||||
"detail": text,
|
||||
}
|
||||
mark = {"removed": "-", "trashed": "-"}.get(group, "=")
|
||||
job.record(
|
||||
group, row, f"{mark} {target['root_label']}/{target['rel']} · {text}"
|
||||
)
|
||||
|
||||
def _delete_one_image(self, db, target, mode, roots):
|
||||
"""``(group, code, trash label)`` for one reviewed file (blocking)."""
|
||||
code, _info = file_check(
|
||||
target["abs"],
|
||||
target["root"],
|
||||
target["rel"],
|
||||
expected={
|
||||
"size": target["size"],
|
||||
"mtime_ns": target["mtime_ns"],
|
||||
"inode": target["inode"],
|
||||
},
|
||||
)
|
||||
if code:
|
||||
return "not_changed", code, None
|
||||
label = None
|
||||
if mode == "trash":
|
||||
try:
|
||||
label = trash.move_to_trash(target["abs"], roots)
|
||||
except trash.TrashRefused as refused:
|
||||
code = refused.code if refused.code in REASONS else trash.FAILED
|
||||
return "not_changed", code, None
|
||||
mark = None
|
||||
else:
|
||||
mark = (target["size"], target["mtime_ns"])
|
||||
try:
|
||||
db.remove_image_records(target["removal"], mark=mark)
|
||||
except Exception:
|
||||
self.logger.exception("Removing an image's records failed")
|
||||
return "errors", None, label
|
||||
_remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
|
||||
return ("trashed" if mode == "trash" else "removed"), None, label
|
||||
|
||||
# ── the workspace listing ─────────────────────────────────────────
|
||||
|
||||
def _removal_marks(self) -> dict:
|
||||
"""The removal marks, cached until a mark changes or the database does.
|
||||
|
||||
A listing page reads no table while nothing changed: the cache follows
|
||||
this process's marks generation and the database identity (a restore).
|
||||
"""
|
||||
key = (self._database_identity(self.db), image_removal_schema.generation())
|
||||
cached = getattr(self, "_marks_cache", None)
|
||||
if cached is None or cached[0] != key:
|
||||
cached = (key, self.db.removal_marks())
|
||||
self._marks_cache = cached
|
||||
return cached[1]
|
||||
|
||||
def _hidden_entries(self, listings) -> set:
|
||||
"""Normalized paths of listed files a removal mark hides (blocking).
|
||||
|
||||
Only marked locators are looked at, and only those files are
|
||||
``lstat``'ed: a mark hides a file while its size and ``mtime_ns`` are
|
||||
still the ones it was removed with.
|
||||
"""
|
||||
marks = self._removal_marks()
|
||||
if not marks:
|
||||
return set()
|
||||
located = self._workspace_locators(listings)
|
||||
return {
|
||||
entry.normalized
|
||||
for locator, entry in ((locator, located.get(locator)) for locator in marks)
|
||||
if entry is not None and mark_hides(marks, locator, entry.path)
|
||||
}
|
||||
|
||||
def _removed_paths(self, paths) -> set:
|
||||
"""The absolute paths among *paths* a removal mark hides (blocking)."""
|
||||
marks = self._removal_marks()
|
||||
if not marks:
|
||||
return set()
|
||||
return {
|
||||
str(path)
|
||||
for path in paths
|
||||
if mark_hides(marks, os.path.normcase(os.path.realpath(str(path))), path)
|
||||
}
|
||||
|
||||
def _removal_check_under(self, root):
|
||||
"""``hidden(path) -> bool`` for files a walk of *root* found (blocking).
|
||||
|
||||
A walk never follows a link below *root*, so a file's locator is
|
||||
``normcase(join(realpath(root), relative))``: no ``realpath`` per
|
||||
file, and an ``lstat`` only for a marked one. Without marks it
|
||||
answers False at once.
|
||||
"""
|
||||
marks = self._removal_marks()
|
||||
if not marks:
|
||||
return lambda _path: False
|
||||
root = os.fspath(root)
|
||||
real = os.path.realpath(root)
|
||||
|
||||
def hidden(path):
|
||||
relative = os.path.relpath(os.fspath(path), root)
|
||||
locator = os.path.normcase(os.path.join(real, relative))
|
||||
return mark_hides(marks, locator, path)
|
||||
|
||||
return hidden
|
||||
|
||||
def _without_removed(self, root, paths) -> list:
|
||||
"""*paths* found by a walk of *root*, less the hidden ones (blocking)."""
|
||||
removed = self._removal_check_under(root)
|
||||
return [path for path in paths if not removed(path)]
|
||||
|
||||
|
||||
def parse_mode(value):
|
||||
"""The apply's ``mode``, or None when it is not one of MODES."""
|
||||
return value if isinstance(value, str) and value in MODES else None
|
||||
@@ -488,7 +488,11 @@ class ImageTagJobsMixin:
|
||||
_still_targets, self, index, root, files, seen
|
||||
)
|
||||
)
|
||||
return found
|
||||
# #480: a file removed from the library is never tagged by a batch.
|
||||
hidden = await self._run_in_executor(
|
||||
self._removed_paths, [target.path for target in found]
|
||||
)
|
||||
return [target for target in found if target.path not in hidden]
|
||||
|
||||
async def _start_autotag(
|
||||
self, *, model_type, existing, options, keep_in_memory, title, owner_kind
|
||||
|
||||
+19
-27
@@ -55,6 +55,7 @@ from .inspection_service import (
|
||||
IMAGE_EXTENSIONS as _IMAGE_EXTENSIONS,
|
||||
SERVABLE_EXTENSIONS,
|
||||
VIDEO_EXTENSIONS,
|
||||
image_tags_reader,
|
||||
inspect_current_file,
|
||||
)
|
||||
|
||||
@@ -65,6 +66,7 @@ try:
|
||||
from ...database.models import normalize_image_path
|
||||
from ...database.run_attribution import recorded_output_lookup
|
||||
from ...utils.card_seed import card_seed, header_params, parsed_positive
|
||||
from ...utils.media_folders import in_skipped_folder, is_skipped_folder
|
||||
from ...utils.parallel import map_parallel
|
||||
from ...utils.json_strict import dumps_strict
|
||||
from ...utils.inspection_response import (
|
||||
@@ -90,6 +92,7 @@ except ImportError:
|
||||
from database.models import normalize_image_path
|
||||
from database.run_attribution import recorded_output_lookup
|
||||
from utils.card_seed import card_seed, header_params, parsed_positive
|
||||
from utils.media_folders import in_skipped_folder, is_skipped_folder
|
||||
from utils.parallel import map_parallel
|
||||
from utils.json_strict import dumps_strict
|
||||
from utils.inspection_response import (
|
||||
@@ -914,7 +917,8 @@ def _name_suffix(name):
|
||||
def _walk_media(output_path, max_files):
|
||||
"""Yield the ``os.DirEntry`` of each media file under *output_path*.
|
||||
|
||||
Symlinks are never followed, ``thumbnails/`` is skipped, descent stops
|
||||
Symlinks are never followed, ``thumbnails/`` and trash folders are
|
||||
skipped (``utils.media_folders``, #501), descent stops
|
||||
at MAX_SCAN_DEPTH and at most *max_files* entries are produced. The order
|
||||
is ``os.walk``'s (top-down, subfolders in ``os.scandir`` order, files by
|
||||
name), which the gallery's ties in mtime keep. Each entry's type comes
|
||||
@@ -926,8 +930,6 @@ def _walk_media(output_path, max_files):
|
||||
produced = 0
|
||||
while stack:
|
||||
top, rel_parts = stack.pop()
|
||||
if "thumbnails" in rel_parts:
|
||||
continue
|
||||
dirs, files = [], []
|
||||
try:
|
||||
with os.scandir(top) as entries:
|
||||
@@ -951,7 +953,7 @@ def _walk_media(output_path, max_files):
|
||||
if len(rel_parts) >= MAX_SCAN_DEPTH:
|
||||
continue
|
||||
for entry in reversed(dirs):
|
||||
if not entry.is_symlink():
|
||||
if not entry.is_symlink() and not is_skipped_folder(entry.name):
|
||||
stack.append((entry.path, rel_parts + (entry.name,)))
|
||||
|
||||
|
||||
@@ -975,7 +977,7 @@ def _settled_entry(root, relative):
|
||||
*folders, name = relative.split(os.sep)
|
||||
if _name_suffix(name).lower() not in MEDIA_EXTENSIONS:
|
||||
return None
|
||||
if "thumbnails" in folders or len(folders) > MAX_SCAN_DEPTH:
|
||||
if in_skipped_folder(folders) or len(folders) > MAX_SCAN_DEPTH:
|
||||
return None
|
||||
folder = os.fspath(root)
|
||||
for part in ("", *folders):
|
||||
@@ -2449,6 +2451,10 @@ class ImageRoutesMixin:
|
||||
def key(entry):
|
||||
return _workspace_key(entry, order)
|
||||
|
||||
# #480: files removed from the library stay on disk, hidden here.
|
||||
hidden = self._hidden_entries(listings)
|
||||
if hidden:
|
||||
entries = [entry for entry in entries if entry.normalized not in hidden]
|
||||
return (
|
||||
entries,
|
||||
key,
|
||||
@@ -3070,7 +3076,9 @@ class ImageRoutesMixin:
|
||||
no symlinks), each preview folder is listed once, and at most
|
||||
MAX_THUMBNAIL_FILES targets are queued per batch (a job plans again
|
||||
after each batch); ``remaining`` counts the targets beyond that cap.
|
||||
Sources in *exclude* (worked on by an earlier batch) are left out.
|
||||
Sources in *exclude* (worked on by an earlier batch) are left out, and
|
||||
so are files removed from the library (#501: a removal mark hides
|
||||
them, and their previews went with their records).
|
||||
``stop()`` true ends the walk before the next file.
|
||||
"""
|
||||
settings = writer_settings(settings)
|
||||
@@ -3079,11 +3087,14 @@ class ImageRoutesMixin:
|
||||
plan = {key: 0 for key in _PLAN_COUNTS}
|
||||
plan.update(targets=[], fresh_files=[])
|
||||
folders = {}
|
||||
removed = self._removal_check_under(output_path)
|
||||
for src in _iter_media_files(output_path, MAX_GALLERY_FILES):
|
||||
if stop is not None and stop():
|
||||
break
|
||||
if exclude and src in exclude:
|
||||
continue
|
||||
if removed(src):
|
||||
continue
|
||||
rel_no_ext = src.relative_to(output_path).with_suffix("")
|
||||
stem = f"{rel_no_ext.name}{thumbnail_settings.THUMB_SUFFIX}"
|
||||
dst = thumbnails_dir / rel_no_ext.parent / (stem + settings.extension)
|
||||
@@ -3962,28 +3973,9 @@ class ImageRoutesMixin:
|
||||
)
|
||||
|
||||
def _inspection_image_tags(self, root: Path, parts: tuple) -> Callable:
|
||||
"""The inspector's ``image_tags`` reader for one rooted file (S1).
|
||||
|
||||
The returned function takes the inspection's own stable stat and
|
||||
answers ``current`` only for a byte record verified at exactly that
|
||||
``(size, mtime_ns)``; a failed read is ``unavailable``, never an error.
|
||||
"""
|
||||
"""The inspector's ``image_tags`` reader for one rooted file (S1)."""
|
||||
locator = os.path.normcase(os.path.join(os.path.realpath(root), *parts))
|
||||
|
||||
def read(size: int, mtime_ns: int) -> dict:
|
||||
try:
|
||||
found = self.db.current_image_tags({locator: (size, mtime_ns)})
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"Inspection image tags could not be read", exc_info=True
|
||||
)
|
||||
return {"state": "unavailable", "tags": []}
|
||||
tags = found.get(locator)
|
||||
if tags is None:
|
||||
return {"state": "none", "tags": []}
|
||||
return {"state": "current", "tags": list(tags)}
|
||||
|
||||
return read
|
||||
return image_tags_reader(self.db, locator, self.logger)
|
||||
|
||||
def _read_own_category(self, sources, path):
|
||||
"""One file's own category, None, or OWN_CATEGORY_UNREAD (blocking).
|
||||
|
||||
@@ -93,6 +93,35 @@ def _in_host_temp(path: Path) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def image_tags_reader(
|
||||
db: Any, locator: str, logger: Any, *, close: bool = False
|
||||
) -> Callable[[int, int], dict]:
|
||||
"""The inspector's ``image_tags`` reader for one file *locator* (S1).
|
||||
|
||||
The returned function takes the inspection's own stable stat and answers
|
||||
``current`` only for a byte record verified at exactly that
|
||||
``(size, mtime_ns)``; a failed read is ``unavailable``, never an error.
|
||||
``close`` closes the calling thread's connection afterwards, for routes
|
||||
that close theirs after reading (History).
|
||||
"""
|
||||
|
||||
def read(size: int, mtime_ns: int) -> dict:
|
||||
try:
|
||||
found = db.current_image_tags({locator: (size, mtime_ns)})
|
||||
except Exception:
|
||||
logger.warning("Inspection image tags could not be read", exc_info=True)
|
||||
return {"state": "unavailable", "tags": []}
|
||||
finally:
|
||||
if close:
|
||||
db.model.close()
|
||||
tags = found.get(locator)
|
||||
if tags is None:
|
||||
return {"state": "none", "tags": []}
|
||||
return {"state": "current", "tags": list(tags)}
|
||||
|
||||
return read
|
||||
|
||||
|
||||
def inspect_current_file(
|
||||
path: Path,
|
||||
*,
|
||||
|
||||
@@ -123,6 +123,11 @@ MODEL_DOWNLOAD_JOB_KIND = "model_download"
|
||||
DUPLICATES_JOB_KIND = "duplicates"
|
||||
DUPLICATE_GROUP_NAMES = ("groups_found",)
|
||||
DUPLICATE_CLEANUP_GROUP_NAMES = ("removed", "kept", "errors")
|
||||
# The workspace Library's Delete… (#443, py/api/library_delete.py): a
|
||||
# ``records`` hold around its safety backup and write.
|
||||
PROMPT_DELETE_JOB_KIND = "prompt_delete"
|
||||
IMAGE_DELETE_JOB_KIND = "image_delete" # #480
|
||||
IMAGE_DELETE_GROUP_NAMES = ("trashed", "removed", "not_changed", "errors")
|
||||
# The System health check (py/api/system.py): a read-only ``read`` job with
|
||||
# no file outcomes, so it may run beside a scan.
|
||||
HEALTH_JOB_KIND = "health"
|
||||
|
||||
+61
-2
@@ -21,9 +21,11 @@ from .pairs import pair_response
|
||||
try:
|
||||
from ...database.library import ARCHIVE_SCOPES
|
||||
from ...database.library_commands import LibraryCommandError
|
||||
from ...database.library_folders import folder_prefixes, normalize_folder
|
||||
except ImportError:
|
||||
from database.library import ARCHIVE_SCOPES
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.library_folders import folder_prefixes, normalize_folder
|
||||
|
||||
_SECRET = os.urandom(32)
|
||||
|
||||
@@ -155,6 +157,13 @@ def _archive_scope(request: web.Request) -> str:
|
||||
return value
|
||||
|
||||
|
||||
def _shelf(request: web.Request) -> str:
|
||||
value = request.query.get("shelf", "active")
|
||||
if value not in ("active", "archived"):
|
||||
raise ValueError("shelf must be active or archived")
|
||||
return value
|
||||
|
||||
|
||||
def _missing() -> web.Response:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Positive prompt not found"}, status=404
|
||||
@@ -283,37 +292,87 @@ class LibraryRoutesMixin:
|
||||
outputs = list(self._get_all_output_dirs())
|
||||
return self._media_roots(outputs), len(outputs)
|
||||
|
||||
def _library_root_spellings(self) -> list[str]:
|
||||
"""Every spelling of each output root (#444 folder filter): the
|
||||
validated, resolved root and each configured or detected path that
|
||||
resolves to it, so images saved through a symlinked root match."""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||||
candidates = [
|
||||
*GalleryConfig.MONITORING_DIRECTORIES,
|
||||
self._find_comfyui_output_dir(),
|
||||
*self._comfyui_output_dir_list(),
|
||||
]
|
||||
spellings = []
|
||||
for root in roots:
|
||||
for value in (root, *candidates):
|
||||
if not value or not isinstance(value, (str, os.PathLike)):
|
||||
continue
|
||||
value = str(value)
|
||||
same = value == root or os.path.normcase(
|
||||
os.path.realpath(value)
|
||||
) == os.path.normcase(root)
|
||||
if same and value not in spellings:
|
||||
spellings.append(value)
|
||||
return spellings
|
||||
|
||||
@pair_response
|
||||
async def get_library_options(self, request: web.Request) -> web.Response:
|
||||
result = await self._run_in_executor(self.db.get_library_options)
|
||||
include = request.query.get("include", "")
|
||||
if include not in ("", "folders"):
|
||||
raise ValueError("include must be empty or 'folders'")
|
||||
if include:
|
||||
# Folders are read only when Refine opens (a walk of linked paths).
|
||||
roots = await self._run_in_executor(self._library_root_spellings)
|
||||
result = await self._run_in_executor(self.db.get_library_options, roots)
|
||||
else:
|
||||
result = await self._run_in_executor(self.db.get_library_options)
|
||||
return web.json_response({"success": True, **result})
|
||||
|
||||
@pair_response
|
||||
async def list_library(self, request: web.Request) -> web.Response:
|
||||
folder = normalize_folder(request.query.get("folder", ""))
|
||||
options = {
|
||||
"text": request.query.get("text", ""),
|
||||
"category": request.query.get("category"),
|
||||
"tags": sorted(set(request.query.getall("tag", []))),
|
||||
"tag_mode": request.query.get("tag_mode", "all"),
|
||||
"sort": request.query.get("sort", "newest"),
|
||||
# #443: which shelf; part of the cursor scope, so a cursor never
|
||||
# crosses shelves. Not the variant ``archive`` scope of /{id}.
|
||||
"shelf": _shelf(request),
|
||||
}
|
||||
limit = _page_limit(request)
|
||||
cursor = request.query.get("cursor")
|
||||
roots = (
|
||||
await self._run_in_executor(self._library_root_spellings) if folder else []
|
||||
)
|
||||
prefixes = folder_prefixes(roots, folder) if folder else None
|
||||
|
||||
def read() -> tuple[dict, dict]:
|
||||
with self.db.library_snapshot(include_ceiling=True) as snapshot:
|
||||
conn, epoch, ceiling = snapshot
|
||||
# The folder and the roots it was resolved under join the scope.
|
||||
scope = _scope(
|
||||
{"kind": "parents", "projection": 2, "epoch": epoch, **options}
|
||||
{
|
||||
"kind": "parents",
|
||||
"projection": 2,
|
||||
"epoch": epoch,
|
||||
**options,
|
||||
**({"folder": folder, "roots": roots} if folder else {}),
|
||||
}
|
||||
)
|
||||
result = self.db._list_library(
|
||||
conn,
|
||||
epoch,
|
||||
ceiling,
|
||||
**options,
|
||||
folder_prefixes=prefixes,
|
||||
limit=limit,
|
||||
after=_decode(cursor, scope),
|
||||
)
|
||||
result["folder"] = folder
|
||||
records = self.db.get_library_previews(
|
||||
[parent["id"] for parent in result["items"]]
|
||||
)
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Delete or archive selected Library prompts, and restore archived ones (#443).
|
||||
|
||||
The workspace Prompt library's Delete… (D152). The legacy ``POST
|
||||
/prompt_manager/bulk/delete`` and ``DELETE /prompt_manager/delete/{id}`` are
|
||||
separate and unchanged.
|
||||
|
||||
- ``POST /prompt_manager/library/delete/preview {prompt_ids, expected_epoch}``
|
||||
classifies the selection (reads only, under a lease): the ids to ``delete``
|
||||
(in no saved pair), to ``archive`` (paired), the ``protected`` ones (tagged
|
||||
``__protected__``, kept), counts of ``already_archived`` and ``not_found``,
|
||||
the ``image_links`` a delete removes, the safety ``backup`` policy and,
|
||||
when a backup would be taken, the ``disk`` it needs.
|
||||
- ``POST /prompt_manager/library/delete {archive_ids, delete_ids,
|
||||
expected_epoch, backup}`` executes that review under a ``records`` hold
|
||||
(refused 409 ``busy`` beside a scan, restore, maintenance, move or
|
||||
Optimize). When a delete remains and a backup was promised (the reviewed
|
||||
``backup`` was "on" or the policy is on now) it checks the disk (507
|
||||
``insufficient-space``) and takes a verified safety backup (500
|
||||
``backup-failed``); then one transaction deletes the reviewed deletes whose
|
||||
fingerprint is unchanged since the backup and archives the paired ones.
|
||||
- ``POST /prompt_manager/library/restore {prompt_ids, expected_epoch}``
|
||||
removes the archive mark (under a lease).
|
||||
|
||||
A changed storage epoch answers 409 ``storage-changed``; nothing is written.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
try:
|
||||
from ...database.library_commands import LibraryCommandError
|
||||
from ...database.safety_backup import SafetyBackupFailed
|
||||
except ImportError:
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.safety_backup import SafetyBackupFailed
|
||||
|
||||
from . import maintenance as _maintenance
|
||||
from .admin import _prune_safety_backups
|
||||
from .job_routes import busy_response
|
||||
from .jobs import PROMPT_DELETE_JOB_KIND, JobBusy
|
||||
from .prompts import parse_bulk_prompt_ids
|
||||
|
||||
PREVIEW_URL = "/prompt_manager/library/delete/preview"
|
||||
DELETE_URL = "/prompt_manager/library/delete"
|
||||
RESTORE_URL = "/prompt_manager/library/restore"
|
||||
DELETE_TITLE = "Delete prompts (Library)"
|
||||
_SELECTION_KEYS = frozenset({"prompt_ids", "expected_epoch"})
|
||||
_DELETE_KEYS = frozenset({"archive_ids", "delete_ids", "expected_epoch", "backup"})
|
||||
STORAGE_CHANGED = "Prompt storage was restored; refresh the library"
|
||||
|
||||
|
||||
def _failure(status, code, error, **extra):
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": error, **extra}, status=status
|
||||
)
|
||||
|
||||
|
||||
def _invalid(message):
|
||||
return _failure(400, "invalid-request", message)
|
||||
|
||||
|
||||
def _storage_changed(exc):
|
||||
return _failure(409, exc.code, STORAGE_CHANGED, library_epoch=exc.epoch)
|
||||
|
||||
|
||||
async def _body(request, keys):
|
||||
"""The JSON object with exactly ``keys``, or raise ValueError."""
|
||||
try:
|
||||
data = await request.json()
|
||||
except ValueError:
|
||||
raise ValueError("Expected a JSON object") from None
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("Expected a JSON object")
|
||||
if set(data) != keys:
|
||||
raise ValueError("Request must contain exactly " + ", ".join(sorted(keys)))
|
||||
return data
|
||||
|
||||
|
||||
def _reviewed_ids(value, name):
|
||||
"""A reviewed id list: possibly empty, else as ``parse_bulk_prompt_ids``."""
|
||||
if value == []:
|
||||
return []
|
||||
try:
|
||||
return parse_bulk_prompt_ids(value)
|
||||
except ValueError as exc:
|
||||
raise ValueError(str(exc).replace("prompt_ids", name)) from None
|
||||
|
||||
|
||||
class LibraryDeleteRoutesMixin:
|
||||
"""Mixin providing the workspace Library's delete, archive and restore."""
|
||||
|
||||
def _register_library_delete_routes(self, routes):
|
||||
routes.post(PREVIEW_URL)(self.preview_library_delete)
|
||||
routes.post(DELETE_URL)(self.delete_library_prompts)
|
||||
routes.post(RESTORE_URL)(self.restore_library_prompts)
|
||||
|
||||
async def preview_library_delete(self, request):
|
||||
try:
|
||||
data = await _body(request, _SELECTION_KEYS)
|
||||
ids = parse_bulk_prompt_ids(data["prompt_ids"])
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
db = self.db
|
||||
policy = self._safety_backup_policy()
|
||||
try:
|
||||
async with self._jobs.lease("Library delete review"):
|
||||
review = await self._run_in_executor(
|
||||
db.classify_library_delete,
|
||||
ids,
|
||||
expected_epoch=data["expected_epoch"],
|
||||
)
|
||||
disk = None
|
||||
if review["delete"] and policy == "on":
|
||||
public, short, _size = await self._run_in_executor(
|
||||
_maintenance.disk_plan,
|
||||
db.model.db_path,
|
||||
backup=True,
|
||||
vacuum=False,
|
||||
)
|
||||
disk = {
|
||||
**public,
|
||||
"message": (
|
||||
None
|
||||
if short is None
|
||||
else _maintenance.short_space_message(short)
|
||||
),
|
||||
}
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
except LibraryCommandError as exc:
|
||||
return _storage_changed(exc)
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Library delete review failed")
|
||||
return _failure(500, "review-failed", "The delete could not be reviewed")
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"library_epoch": review["library_epoch"],
|
||||
"delete": review["delete"],
|
||||
"archive": review["archive"],
|
||||
"protected": review["protected"],
|
||||
"already_archived": len(review["already_archived"]),
|
||||
"not_found": len(review["not_found"]),
|
||||
"image_links": review["image_links"],
|
||||
"backup": policy,
|
||||
"disk": disk,
|
||||
}
|
||||
)
|
||||
|
||||
async def delete_library_prompts(self, request):
|
||||
try:
|
||||
data = await _body(request, _DELETE_KEYS)
|
||||
archive_ids = _reviewed_ids(data["archive_ids"], "archive_ids")
|
||||
delete_ids = _reviewed_ids(data["delete_ids"], "delete_ids")
|
||||
if not archive_ids and not delete_ids:
|
||||
raise ValueError("Nothing to delete or archive")
|
||||
if len(archive_ids) + len(delete_ids) > 1000:
|
||||
raise ValueError("At most 1000 prompts can be changed at once")
|
||||
if set(archive_ids) & set(delete_ids):
|
||||
raise ValueError("A prompt cannot be both archived and deleted")
|
||||
if data["backup"] not in ("on", "off"):
|
||||
raise ValueError("backup must be on or off")
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
busy = self._jobs.blocker(PROMPT_DELETE_JOB_KIND, admission="records")
|
||||
if busy is not None:
|
||||
return busy_response(JobBusy(busy))
|
||||
try:
|
||||
async with self._jobs.hold(
|
||||
PROMPT_DELETE_JOB_KIND, DELETE_TITLE, admission="records"
|
||||
) as job:
|
||||
response = await self._held_library_delete(
|
||||
self.db, data, archive_ids, delete_ids
|
||||
)
|
||||
if response.status >= 400:
|
||||
job.finish("failed")
|
||||
return response
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def _held_library_delete(self, db, data, archive_ids, delete_ids):
|
||||
epoch = data["expected_epoch"]
|
||||
promised = data["backup"] == "on" or self._safety_backup_policy() == "on"
|
||||
backup = None
|
||||
try:
|
||||
candidates = None
|
||||
if delete_ids:
|
||||
review = await self._run_in_executor(
|
||||
db.classify_library_delete, delete_ids, expected_epoch=epoch
|
||||
)
|
||||
candidates = review["fingerprints"]
|
||||
if candidates and promised:
|
||||
_disk, short, _size = await self._run_in_executor(
|
||||
_maintenance.disk_plan,
|
||||
db.model.db_path,
|
||||
backup=True,
|
||||
vacuum=False,
|
||||
)
|
||||
if short is not None:
|
||||
return _failure(
|
||||
507,
|
||||
"insufficient-space",
|
||||
_maintenance.short_space_message(short),
|
||||
)
|
||||
backup = await self._run_in_executor(
|
||||
_maintenance.take_safety_backup, db.model
|
||||
)
|
||||
result = await self._run_in_executor(
|
||||
db.archive_or_delete_library_prompts,
|
||||
archive_ids,
|
||||
delete_ids,
|
||||
expected_epoch=epoch,
|
||||
candidates=candidates if candidates is not None else {},
|
||||
)
|
||||
except LibraryCommandError as exc:
|
||||
return _storage_changed(exc)
|
||||
except SafetyBackupFailed as error:
|
||||
self.logger.error("Safety backup before a Library delete failed")
|
||||
return _failure(
|
||||
500, "backup-failed", _maintenance.backup_failed_message(error)
|
||||
)
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Library delete failed")
|
||||
return _failure(500, "delete-failed", "The prompts could not be changed")
|
||||
finally:
|
||||
if backup is not None:
|
||||
await self._prune_after_delete(db)
|
||||
return web.json_response({"success": True, **result, "backup": backup})
|
||||
|
||||
async def _prune_after_delete(self, db):
|
||||
try:
|
||||
await self._run_in_executor(_prune_safety_backups, db.model.db_path)
|
||||
except Exception: # pruning old copies never fails the delete
|
||||
self.logger.exception("Could not prune old safety backups")
|
||||
|
||||
async def restore_library_prompts(self, request):
|
||||
try:
|
||||
data = await _body(request, _SELECTION_KEYS)
|
||||
ids = parse_bulk_prompt_ids(data["prompt_ids"])
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
try:
|
||||
async with self._jobs.lease("Library restore"):
|
||||
result = await self._run_in_executor(
|
||||
self.db.restore_library_prompts,
|
||||
ids,
|
||||
expected_epoch=data["expected_epoch"],
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
except LibraryCommandError as exc:
|
||||
return _storage_changed(exc)
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Library restore failed")
|
||||
return _failure(500, "restore-failed", "The prompts could not be restored")
|
||||
return web.json_response({"success": True, **result})
|
||||
+21
-4
@@ -281,7 +281,7 @@ def short_space_message(short):
|
||||
)
|
||||
|
||||
|
||||
def _changed_line(operation, count):
|
||||
def _changed_line(operation, count, marks=0):
|
||||
if operation == "optimize":
|
||||
return "Optimized the database"
|
||||
if operation == "duplicate_prompts":
|
||||
@@ -291,7 +291,11 @@ def _changed_line(operation, count):
|
||||
noun = "unused prompt component" if count == 1 else "unused prompt components"
|
||||
return f"Removed {count:,} {noun}"
|
||||
noun = "link to a missing file" if count == 1 else "links to missing files"
|
||||
return f"Removed {count:,} {noun}"
|
||||
line = f"Removed {count:,} {noun}"
|
||||
if marks:
|
||||
gone = "removed image" if marks == 1 else "removed images"
|
||||
line += f"; forgot {marks:,} {gone} whose files are gone or changed"
|
||||
return line
|
||||
|
||||
|
||||
def take_safety_backup(model, label="backup"):
|
||||
@@ -419,6 +423,7 @@ class MaintenanceRoutesMixin:
|
||||
draft.public(),
|
||||
{
|
||||
"targets": draft.targets,
|
||||
"marks": draft.marks,
|
||||
"roots": roots,
|
||||
"protected": draft.protected["count"],
|
||||
"rows": draft.scope["count"],
|
||||
@@ -488,7 +493,9 @@ class MaintenanceRoutesMixin:
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
async def apply_maintenance_plan(self, request):
|
||||
data, error = await self._read_maintenance_body(request, ("token",))
|
||||
from .image_delete import IMAGE_DELETE, parse_mode
|
||||
|
||||
data, error = await self._read_maintenance_body(request, ("token", "mode"))
|
||||
if error is not None:
|
||||
return error
|
||||
token = data.get("token")
|
||||
@@ -497,6 +504,13 @@ class MaintenanceRoutesMixin:
|
||||
plan, refused = self._peek_plan(token)
|
||||
if refused is not None:
|
||||
return refused
|
||||
mode = None
|
||||
if plan.operation == IMAGE_DELETE:
|
||||
mode = parse_mode(data.get("mode"))
|
||||
if mode is None:
|
||||
return _invalid('mode must be "library" or "trash"')
|
||||
elif "mode" in data:
|
||||
return _invalid("Unknown field: mode")
|
||||
store = self._maintenance_plans
|
||||
if plan.operation == ROTATED_LOGS:
|
||||
# Log files only: no database identity, backup or disk check.
|
||||
@@ -544,6 +558,8 @@ class MaintenanceRoutesMixin:
|
||||
try:
|
||||
if plan.operation == DUPLICATE_FILES:
|
||||
job = self._start_duplicate_cleanup(db, plan, backup)
|
||||
elif plan.operation == IMAGE_DELETE:
|
||||
job = self._start_image_delete(db, plan, mode, backup)
|
||||
elif plan.operation == WORKFLOW_STORAGE:
|
||||
job = self._start_workflow_storage(db, backup)
|
||||
else:
|
||||
@@ -642,6 +658,7 @@ class MaintenanceRoutesMixin:
|
||||
plan.fingerprint,
|
||||
payload["targets"],
|
||||
payload["roots"],
|
||||
payload.get("marks", ()),
|
||||
)
|
||||
except PlanChanged:
|
||||
return _fail_job(job, "plan-changed", PLAN_CHANGED)
|
||||
@@ -653,7 +670,7 @@ class MaintenanceRoutesMixin:
|
||||
)
|
||||
job.record_counted("changed", result["changed"], rows)
|
||||
job.record_counted("protected", payload["protected"])
|
||||
line = _changed_line(plan.operation, result["changed"])
|
||||
line = _changed_line(plan.operation, result["changed"], result.get("marks", 0))
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Revalidated responses for the pages and static files PromptManager serves.
|
||||
|
||||
``Cache-Control: no-cache`` lets the browser keep a copy but makes it ask the
|
||||
server before every reuse, so a PromptManager update reaches the browser on a
|
||||
normal reload (#478). The ETag lets an unchanged file answer 304 without a
|
||||
body, which keeps that reload cheap.
|
||||
|
||||
The tag hashes the bytes rather than the file's mtime and size: an update can
|
||||
keep both (an archive extract, a same-length edit within one clock tick), and
|
||||
the handlers read the whole file anyway. It is weak because the gzip
|
||||
middleware may send the same content in a different encoding.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
REVALIDATE = "no-cache"
|
||||
|
||||
|
||||
def entity_tag(body: bytes) -> str:
|
||||
"""Weak ETag for ``body``."""
|
||||
return f'W/"{hashlib.sha256(body).hexdigest()[:32]}"'
|
||||
|
||||
|
||||
def etag_matches(if_none_match, etag: str) -> bool:
|
||||
"""True when an If-None-Match header names ``etag`` (weak comparison)."""
|
||||
if not if_none_match:
|
||||
return False
|
||||
if if_none_match.strip() == "*":
|
||||
return True
|
||||
opaque = etag.removeprefix("W/")
|
||||
return any(
|
||||
candidate.strip().removeprefix("W/") == opaque
|
||||
for candidate in if_none_match.split(",")
|
||||
)
|
||||
|
||||
|
||||
def revalidated_response(request, body: bytes, content_type: str, charset=None):
|
||||
"""200 with ``body``, or 304 when the request already holds this version."""
|
||||
etag = entity_tag(body)
|
||||
headers = {"Cache-Control": REVALIDATE, "ETag": etag}
|
||||
if etag_matches(request.headers.get("If-None-Match"), etag):
|
||||
return web.Response(status=304, headers=headers)
|
||||
return web.Response(
|
||||
body=body, content_type=content_type, charset=charset, headers=headers
|
||||
)
|
||||
@@ -149,8 +149,8 @@ test('panel matches the concept order and copy',async({page,request})=>{
|
||||
await expect(panel.locator('.meta-grid dt')).toHaveText(['Seed','Steps','CFG','Sampler','Scheduler','Model']);
|
||||
await expect(panel.locator('.meta-grid dd')).toHaveText(['9007199254740993','28','6.5','dpmpp_2m','karras','fixture-model.safetensors']);
|
||||
const footer=panel.locator('.detail-footer');
|
||||
// #331 D139: every listed file's footer ends with Collections… (folder icon).
|
||||
await expect(footer.locator('button')).toHaveText(['Send to ComfyUI','Prompt history','Collections…']);
|
||||
// #331 D139: every listed file's footer has Collections… (folder icon); #480 D159: Delete… ends it.
|
||||
await expect(footer.locator('button')).toHaveText(['Send to ComfyUI','Prompt history','Collections…','Delete…']);
|
||||
await expect(footer.locator('button.send')).toHaveAccessibleName(`Send prompt pair to ComfyUI: ${POSITIVE_TITLE}`);
|
||||
await expect(footer.locator('button.send svg.icon')).toHaveCount(1);await expect(footer.locator('button:not(.send) svg.icon')).toHaveCount(2);
|
||||
await expect(panel).not.toContainText(/prompt evidence|Positive prompt tags/);
|
||||
@@ -162,10 +162,10 @@ test('panel matches the concept order and copy',async({page,request})=>{
|
||||
await selectFile(page,'ambiguous.png');await expect(panel.locator('.prompt-section.positive .prompt-text')).toHaveText('embedded positive');
|
||||
await expect(panel.locator('span.chip.blue')).toHaveText('Embedded parameters');
|
||||
await expect(panel.locator('.panel-heading .chip')).toHaveText('File extension PNG');await expect(panel.locator('.panel-heading .chip .ca-visually-hidden')).toHaveText('File extension ');
|
||||
await expect(panel.locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Collections…']);
|
||||
// Unlinked: no badge, category, tags, pair or history; the footer holds only Collections… (D139).
|
||||
await expect(panel.locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Collections…','Delete…']);
|
||||
// Unlinked: no badge, category, tags, pair or history; the footer holds Collections… (D139) and Delete… (D159).
|
||||
reads.patch=null;await selectFile(page,'empty.png');await expect(panel.locator('.panel-heading h2')).toHaveText('empty.png');
|
||||
await expect(panel.locator('.detail-footer button')).toHaveText(['Collections…']);
|
||||
await expect(panel.locator('.detail-footer button')).toHaveText(['Collections…','Delete…']);
|
||||
expect(await panelStructure(panel)).toEqual({outer:['img.detail-cover','div.detail-inner','div.detail-footer'],
|
||||
inner:['div.panel-heading','p.small','section.prompt-section.positive','section.prompt-section.negative','dl.meta-grid']});
|
||||
await expect(panel.locator('.meta-grid dd')).toHaveText(Array(6).fill('Not recorded'));
|
||||
@@ -266,7 +266,7 @@ test('pair send and history appear only with real roles and link',async({page,re
|
||||
await expect(panel.getByRole('button',{name:'Send prompt pair to ComfyUI: explicit-empty.png',exact:true})).toBeVisible();
|
||||
await expect(panel.getByRole('button',{name:'Prompt history',exact:true})).toHaveCount(0);
|
||||
await selectFile(page,'empty.png');await expect(panel.locator('.panel-heading h2')).toHaveText('empty.png');
|
||||
await expect(panel.locator('.detail-footer button')).toHaveText(['Collections…']);await expect(panel.getByRole('button',{name:/^Send prompt pair/})).toHaveCount(0);
|
||||
await expect(panel.locator('.detail-footer button')).toHaveText(['Collections…','Delete…']);await expect(panel.getByRole('button',{name:/^Send prompt pair/})).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('no provenance, raw or download controls in the side panel',async({page,request})=>{
|
||||
@@ -627,6 +627,10 @@ test('viewer matches the concept shell and toolbar order',async({page,request})=
|
||||
expect(modePaths.map(path=>path.split(' > ').slice(2).join(' > '))).toEqual(['button#viewer-mode.icon-btn','button#viewer-mode.icon-btn > svg.icon','button#viewer-mode.icon-btn > svg.icon > path']);
|
||||
expect(production.paths[production.paths.indexOf(modePaths[2])+1]).toBe('dialog#viewer-dialog > div.dialog-header > button#viewer-close');
|
||||
production.paths=production.paths.filter(path=>!modePaths.includes(path));
|
||||
// #480 D159: the Images viewer's Delete… (`danger`) follows Download; the concept has none.
|
||||
expect(production.toolbar[8].text).toBe('Delete…');
|
||||
production.toolbar.splice(8,1);
|
||||
production.paths=production.paths.filter(path=>!path.includes('button.danger'));
|
||||
// #249 D34: after Next → production appends its own status lines (position, then the transform and slideshow
|
||||
// reasons), and Download stays the original-file link; every other control keeps the concept's text and order.
|
||||
const appended=production.toolbar.slice(9);
|
||||
|
||||
@@ -31,13 +31,15 @@ function captionDay(seconds,now=new Date()) {
|
||||
}
|
||||
|
||||
/**
|
||||
* D8 (amended by #249 PR D and #331): the selection bar's real actions only; Remove links… is left out. Add to
|
||||
* collection compares in place (D138).
|
||||
* D8 (amended by #249 PR D and #331): the selection bar's real actions only. Add to collection compares in place
|
||||
* (D138). #480 D159: Remove links… returns as Delete…; OMITTED_BULK names the concept button it replaces.
|
||||
*/
|
||||
const OMITTED_BULK=Object.freeze(['Remove links…']);
|
||||
const OMITTED_BULK_SELECTORS=Object.freeze(['bulk-remove'].map(action=>`#bulkbar [data-action="${action}"]`));
|
||||
/** D85: Auto-tag selected follows the concept's Add tags, a concept button of its own text. */
|
||||
const AUTO_TAG_CLONE=Object.freeze([{selector:'#bulkbar [data-action="bulk-tags"]',text:'Auto-tag selected'}]);
|
||||
/** #480 D159: Delete… takes the place of Remove links… (the concept's `danger` button, under another text). */
|
||||
const DELETE_CLONE=Object.freeze([{selector:'#bulkbar [data-action="bulk-remove"]',text:'Delete…'}]);
|
||||
|
||||
/**
|
||||
* `withoutItems` for a row with padding and a border (the `.bulkbar`): its items flow again inside the content box,
|
||||
@@ -452,10 +454,12 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`images selecti
|
||||
const restore=await mutate(page,{remove:[...OMITTED_BULK_SELECTORS]});
|
||||
const removed=(await inventoryOf(page,'#main')).bulkbar;
|
||||
await restore();
|
||||
// The oracle for production's bar: the concept page itself without the omitted actions and with D85's button.
|
||||
const restoreProduction=await mutate(page,{remove:[...OMITTED_BULK_SELECTORS],clone:AUTO_TAG_CLONE});
|
||||
// The oracle for production's bar: the concept page itself with D85's button and D159's Delete… in place of
|
||||
// Remove links….
|
||||
const restoreClones=await mutate(page,{clone:[...AUTO_TAG_CLONE,...DELETE_CLONE]});
|
||||
const restoreRemoved=await mutate(page,{remove:[...OMITTED_BULK_SELECTORS]});
|
||||
const oracle=await inventoryOf(page,'#main');
|
||||
await restoreProduction();
|
||||
await restoreRemoved();await restoreClones();
|
||||
await captureCanonicalState(page,testInfo,`concept-images-selected-${width}-${theme}`);
|
||||
await openProductionLoaded(page,theme,'no-preference');
|
||||
const emptyActual=await inventoryOf(page,'#workspace-main');
|
||||
@@ -468,7 +472,7 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`images selecti
|
||||
'Add to collection','Export metadata','Remove links…','Clear']);
|
||||
expect.soft(diffInventory(stripExact(mapBulkbar(reference.bulkbar,0).bulkbar.bar),stripExact(removed.bar)),'bar without the omitted actions').toEqual([]);
|
||||
expect.soft(actual.bulkbar.bar.items.map(item=>item.text),'production bar').toEqual(['1 selected','Add tags','Auto-tag selected',
|
||||
'Set category','Add to collection','Export metadata','Clear']);
|
||||
'Set category','Add to collection','Export metadata','Delete…','Clear']);
|
||||
expect.soft(compare(empty,emptyActual,manifest,['bulkbar','cards','panel']),'nothing checked').toEqual([]);
|
||||
expect.soft(compare(oracle,actual,manifest,['bulkbar','cards','panel']),'one card checked').toEqual([]);
|
||||
// The bar comparison notices a change to the production bar.
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {SIZES,THEMES,seedConceptAt,openConcept,openProduction,diffInventory,mutate} from './concept-inventory.mjs';
|
||||
|
||||
// #480 (D159): the Images selection bar's Delete… against the concept's "Remove links…" (app.js a432, `danger`) and
|
||||
// its review against the concept's `maintenancePlan` (a953–957): the bar button's place and style, the review's
|
||||
// `dl.meta-grid` rows and its actions, on the concept-shaped fixture, at 1600x1100 and 390x844 in both themes.
|
||||
// Paired screenshots are artifacts only; there are no PNG baselines.
|
||||
test.use({locale:'en-US',timezoneId:'UTC'});
|
||||
|
||||
const CARDS='.gallery-body > .image-grid > .media-card';
|
||||
|
||||
/** Runs in the page: the selection bar's danger button and where it sits among the bar's buttons. */
|
||||
function measureBar() {
|
||||
const bar=document.querySelector('#bulkbar > .bulkbar');
|
||||
const buttons=[...bar.querySelectorAll(':scope > button')];
|
||||
const danger=bar.querySelector(':scope > button.danger');
|
||||
const css=(node,...keys)=>Object.fromEntries(keys.map(key=>[key,getComputedStyle(node)[key]]));
|
||||
return {
|
||||
// The danger action sits right before Clear, the last button.
|
||||
dangerBeforeClear:buttons.indexOf(danger)===buttons.length-2,
|
||||
lastText:buttons.at(-1).textContent.trim(),
|
||||
danger:css(danger,'color','backgroundColor','borderTopColor','fontSize','fontWeight','paddingTop','paddingLeft',
|
||||
'borderTopLeftRadius','height'),
|
||||
};
|
||||
}
|
||||
|
||||
/** Runs in the page: the open review's grid and actions (the parts production shares with the concept). */
|
||||
function measureReview() {
|
||||
const dialog=document.querySelector('dialog#modal[open]');
|
||||
const css=(node,...keys)=>node?Object.fromEntries(keys.map(key=>[key,getComputedStyle(node)[key]])):null;
|
||||
const grid=dialog.querySelector('#modal-body dl.meta-grid');
|
||||
const entry=grid.querySelector(':scope > div');
|
||||
const actions=dialog.querySelector('#modal-body > .dialog-actions');
|
||||
const [cancel]=actions.children;
|
||||
return {
|
||||
grid:css(grid,'display','gridTemplateColumns','columnGap','rowGap'),
|
||||
dt:css(entry.querySelector('dt'),'fontSize','color','textTransform','letterSpacing'),
|
||||
dd:css(entry.querySelector('dd'),'fontSize','color','marginLeft'),
|
||||
terms:[...grid.querySelectorAll('dt')].map(dt=>dt.textContent.trim()),
|
||||
actions:css(actions,'display','justifyContent','columnGap','marginTop'),
|
||||
cancel:{text:cancel.textContent.trim(),...css(cancel,'backgroundColor','color','fontSize','paddingTop','paddingLeft')},
|
||||
};
|
||||
}
|
||||
|
||||
async function checkFirst(page) {
|
||||
await page.locator(CARDS).first().locator('.image-overlays > label > input[type="checkbox"]').check();
|
||||
}
|
||||
|
||||
async function concept(page,theme) {
|
||||
await openConcept(page,'images',theme);
|
||||
await checkFirst(page);
|
||||
const bar=await page.evaluate(measureBar);
|
||||
await page.locator('#main #bulkbar').getByRole('button',{name:'Remove links…',exact:true}).click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review maintenance plan');
|
||||
await page.mouse.move(0,0);
|
||||
return {bar,review:await page.evaluate(measureReview)};
|
||||
}
|
||||
|
||||
async function production(page,theme) {
|
||||
await openProduction(page,'images',theme);
|
||||
await expect(page.locator(CARDS).first()).toBeVisible();
|
||||
await checkFirst(page);
|
||||
const bar=await page.evaluate(measureBar);
|
||||
await page.locator('#workspace-main #bulkbar').getByRole('button',{name:'Delete…',exact:true}).click();
|
||||
const dialog=page.locator('dialog#modal[open]');
|
||||
await expect(dialog.locator('#modal-title')).toHaveText('Delete 1 image');
|
||||
await dialog.getByRole('radio',{name:/^Remove from the library only/}).check();
|
||||
await page.mouse.move(0,0);
|
||||
return {bar,review:await page.evaluate(measureReview)};
|
||||
}
|
||||
|
||||
/**
|
||||
* D159: production's rows follow the chosen mode (Scope, then what the files lose, Backup and Media on disk; the
|
||||
* concept's sample has Scope, Protected, Backup and Media on disk). The geometry, tokens and the Cancel button match.
|
||||
*/
|
||||
function mapConcept(reference,actual) {
|
||||
return {...reference,review:{...reference.review,terms:actual.review.terms}};
|
||||
}
|
||||
|
||||
test.beforeEach(async({page,request},testInfo)=>{testInfo.manifest=await seedConceptAt(page,request);});
|
||||
|
||||
for(const theme of THEMES)for(const [width,height] of SIZES) {
|
||||
test(`delete matches the concept's remove links ${theme} ${width}x${height}`,async({page},testInfo)=>{
|
||||
await page.setViewportSize({width,height});
|
||||
const reference=await concept(page,theme);
|
||||
await page.screenshot({path:testInfo.outputPath(`concept-remove-links-${theme}-${width}.png`)});
|
||||
const actual=await production(page,theme);
|
||||
await page.screenshot({path:testInfo.outputPath(`production-delete-${theme}-${width}.png`)});
|
||||
expect(reference.review.terms).toEqual(['Scope','Protected','Backup','Media on disk']);
|
||||
expect(actual.review.terms[0]).toBe('Scope');
|
||||
expect(actual.review.terms.slice(-2)).toEqual(['Backup','Media on disk']);
|
||||
expect(diffInventory(mapConcept(reference,actual),actual)).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
test('delete comparison detects mutations',async({page})=>{
|
||||
await page.setViewportSize({width:1600,height:1100});
|
||||
const reference=await concept(page,'dark');
|
||||
const actual=await production(page,'dark');
|
||||
const expected=mapConcept(reference,actual);
|
||||
expect(diffInventory(expected,actual)).toEqual([]);
|
||||
const undo=await mutate(page,{css:'dialog#modal dl.meta-grid { column-gap: 31px; } dialog#modal .dialog-actions { gap: 29px; }'});
|
||||
const paths=diffInventory(expected,{...actual,review:await page.evaluate(measureReview)}).map(item=>item.path);
|
||||
expect(paths).toEqual(expect.arrayContaining(['review.grid.columnGap','review.actions.columnGap']));
|
||||
await undo();
|
||||
});
|
||||
@@ -56,7 +56,8 @@ function withCategoryScope(inventory) {
|
||||
* equal production. `reference.viewerPlain` is the concept's own full details
|
||||
* with its tag chips made plain (D20), D117's summary button added and, under a coarse pointer, D34's targets, all at
|
||||
* runtime: the browser lays out what follows them, as the Images spec's removed toolbar items do. D139's
|
||||
* "Collections…" is drawn after Prompt history in both the concept's panel footer and its full details.
|
||||
* "Collections…" is drawn after Prompt history in both the concept's panel footer and its full details; #480's
|
||||
* D159 "Delete…" follows it in the panel footer.
|
||||
*/
|
||||
export function mapInspectorConcept(reference,{viewport}) {
|
||||
const {panel,viewerPlain,fileControls}=reference;
|
||||
@@ -125,6 +126,13 @@ async function withCollectionsButton(page,selector) {
|
||||
for(const key of Object.keys(button.dataset))delete button.dataset[key];
|
||||
button.append(document.querySelector('.filter-row .chip > svg.icon').cloneNode(true),' Collections…');
|
||||
history.after(button);(window.__collectionsButtons??=[]).push(button);
|
||||
// #480 D159: Delete… ends the panel's actions, drawn as the concept's own `danger` button (its Remove links…).
|
||||
if(selector.startsWith('#image-inspector')) {
|
||||
const remove=history.cloneNode(false);
|
||||
for(const key of Object.keys(remove.dataset))delete remove.dataset[key];
|
||||
remove.className='danger';remove.textContent='Delete…';
|
||||
button.after(remove);window.__collectionsButtons.push(remove);
|
||||
}
|
||||
},selector);
|
||||
await settle(page);
|
||||
return async()=>{await page.evaluate(()=>{for(const button of window.__collectionsButtons??[])button.remove();delete window.__collectionsButtons;});};
|
||||
@@ -373,7 +381,7 @@ test('a recipe-linked image without recipe metadata shows its positive prompt me
|
||||
await expect(tags.locator('.role-label')).toHaveText('Prompt tags · 3');
|
||||
await expect(tags.locator('.tags > button.chip')).toHaveText(['minimal','warm_light','editorial']);
|
||||
await expect(tags.locator('span.chip')).toHaveCount(0);
|
||||
await expect(panelOf(page).locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Prompt history','Collections…']);
|
||||
await expect(panelOf(page).locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Prompt history','Collections…','Delete…']);
|
||||
});
|
||||
|
||||
// #391: a tag chip replaces the collection; until its listing answers the caption says it is loading, never that the
|
||||
|
||||
@@ -44,7 +44,12 @@ export function mapLibraryConcept(reference) {
|
||||
// "4 runs" and "Used today". The two-line clamp is
|
||||
// applied below through `clampedCards`, measured on the concept with `CLAMP_CSS` (its samples wrap at 390px).
|
||||
footer:card.footer}));
|
||||
return {...reference,library:{...library,toolbar,
|
||||
// D152 (#443): Show archived prompts (N) comes first in the heading actions, a plain button like Export (the concept
|
||||
// fixture has nothing archived, so N is 0).
|
||||
const actions=reference.heading.actions;
|
||||
const plain=actions.buttons.find(button=>!button.primary);
|
||||
const heading={...reference.heading,actions:{...actions,buttons:[{...plain,text:'Show archived prompts (0)'},...actions.buttons]}};
|
||||
return {...reference,heading,library:{...library,toolbar,
|
||||
// D35: the caption uses the singular for one.
|
||||
caption:{...library.caption,top:library.caption.top-rise,spans:library.caption.spans.map(span=>({...span,text:countCopy(span.text)}))},
|
||||
...(({grid,cards:clamped})=>({grid:{...grid,top:grid.top-rise},cards:clamped}))(clampedCards({grid:library.grid,cards})),
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
|
||||
// #443 D152: the Library's Delete… review against the concept's own reviewed archive: the recipe editor's "Archive…"
|
||||
// (app.js 976), which opens `maintenancePlan("archive <title>")` (953–958, 1309–1310). The concept's shell, grid tokens,
|
||||
// `p.small` and actions are kept; its title, demo note, rows, note text and `ca-primary` start are replaced by D152's,
|
||||
// and the start takes the concept's `danger` class, measured on the editor's own "Archive…". The bar's Delete… is
|
||||
// compared with the concept Images bar's danger action, "Remove links…" (app.js 432).
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
page.conceptErrors = [];
|
||||
page.on('pageerror', error => page.conceptErrors.push(error.message));
|
||||
});
|
||||
test.afterEach(async ({ page, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.conceptErrors).toEqual([]);
|
||||
});
|
||||
|
||||
/** Runs in the page: the open review dialog's title, body tree, grid, notes and actions with the concept's tokens. */
|
||||
function measureReview() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const body = dialog.querySelector('#modal-body');
|
||||
const grid = body.querySelector(':scope > dl.meta-grid');
|
||||
const notes = [...body.querySelectorAll(':scope > p.small')];
|
||||
const actions = body.querySelector(':scope > .dialog-actions');
|
||||
return {
|
||||
title: text(dialog.querySelector('#modal-title')),
|
||||
tree: [...body.children].map(name).filter(item => item !== 'p.demo-note' && item !== 'h3'),
|
||||
grid: { rows: [...grid.children].map(row => [text(row.querySelector('dt')), text(row.querySelector('dd'))]),
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
dt: css(grid.querySelector('dt'), 'fontSize', 'color', 'textTransform'), dd: css(grid.querySelector('dd'), 'fontSize', 'color') },
|
||||
notes: notes.map(note => ({ text: text(note), ...css(note, 'fontSize', 'color', 'marginTop') })),
|
||||
actions: { tree: [...actions.children].map(name), ...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(item => ({ text: text(item), className: item.className,
|
||||
...css(item, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor') })) },
|
||||
};
|
||||
}
|
||||
|
||||
/** Runs in the page: a button's class and tokens (the concept's danger button, or production's). */
|
||||
function measureButton(selector) {
|
||||
const node = document.querySelector(selector);
|
||||
const style = getComputedStyle(node);
|
||||
return { className: node.className, ...Object.fromEntries(['backgroundColor', 'color', 'fontSize', 'fontWeight', 'paddingTop',
|
||||
'paddingLeft', 'borderTopColor', 'borderTopLeftRadius', 'height'].map(key => [key, key === 'height'
|
||||
? `${Math.round(node.getBoundingClientRect().height * 100) / 100}px` : style[key]])) };
|
||||
}
|
||||
|
||||
async function conceptReview(page, theme) {
|
||||
await openConcept(page, 'prompt/alpine', theme);
|
||||
await page.locator('#main .heading-actions').getByRole('button', { name: 'Edit metadata', exact: true }).click();
|
||||
const archive = page.locator('dialog#modal[open] .dialog-actions button', { hasText: 'Archive…' });
|
||||
await expect(archive).toBeVisible();await page.mouse.move(0, 0);
|
||||
const danger = await page.evaluate(measureButton, 'dialog#modal[open] .dialog-actions button.danger');
|
||||
await archive.click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review maintenance plan');
|
||||
await page.mouse.move(0, 0);
|
||||
return { danger, review: await page.evaluate(measureReview) };
|
||||
}
|
||||
|
||||
async function seedMixed(request) {
|
||||
// One unpaired prompt beside the concept's paired ones, so the review deletes one and archives one.
|
||||
await expect(await request.post('/prompt_manager/save', { data: { text: 'Unpaired study', category: 'Studies' } })).toBeOK();
|
||||
const items = (await (await request.get('/prompt_manager/library?limit=100')).json()).items;
|
||||
return { unpaired: items.find(item => item.text === 'Unpaired study'), paired: items.find(item => item.variant_count > 0) };
|
||||
}
|
||||
|
||||
async function productionReview(page, request, theme) {
|
||||
const { unpaired, paired } = await seedMixed(request);
|
||||
await openProduction(page, 'library', theme);
|
||||
for (const item of [unpaired, paired]) await page.locator(`article[data-prompt-id="${item.id}"] .image-overlays input`).check();
|
||||
await page.locator('#library-bulkbar .bulkbar').getByRole('button', { name: 'Delete…', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Delete or archive selected prompts');
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureReview);
|
||||
}
|
||||
|
||||
/** D152's changes to the concept's review; everything else must equal the concept. */
|
||||
function mapDeleteConcept({ danger, review }) {
|
||||
const [cancel] = review.actions.buttons;
|
||||
const note = review.notes.at(-1);
|
||||
return {
|
||||
...review,
|
||||
title: 'Delete or archive selected prompts',
|
||||
tree: ['dl.meta-grid', 'p.small', 'p.small', 'div.dialog-actions'],
|
||||
grid: { ...review.grid, rows: [['Delete permanently', '1 prompt not in a saved pair'],
|
||||
['Archive', '1 prompt in a saved pair: hidden, restorable'],
|
||||
['Image links removed', '0 images will no longer show these prompts (files stay on disk)'],
|
||||
['Backup', 'Create and verify before deleting']] },
|
||||
notes: [
|
||||
{ ...note, text: 'Prompts in a saved pair, as its positive or its negative, are archived instead of deleted, so the '
|
||||
+ 'pair and its runs stay intact. Archived prompts cannot be deleted. To bring them back, press Show archived prompts, '
|
||||
+ 'select them and press Restore.' },
|
||||
// A second paragraph takes the concept's own `p + p` margin (style.css 93-95).
|
||||
{ ...note, marginTop: '10px', text: "Deleting removes each prompt's text, title, category, tags, rating, notes, run count, last use and "
|
||||
+ 'image links. Deleted prompts cannot be brought back one by one: the only way back is to restore the safety '
|
||||
+ 'backup taken first (System › Backup & export), which returns the whole library to that moment. It is kept '
|
||||
+ 'among the last 10 safety backups. Image files, favorites, collections and recorded runs stay. A later scan, a '
|
||||
+ 'new image or a node run with the same text adds it again as a new prompt.' },
|
||||
],
|
||||
actions: { ...review.actions, tree: [review.actions.tree[0], 'button.danger'], buttons: [cancel, { text: 'Delete 1 and archive 1 prompt', className: danger.className,
|
||||
backgroundColor: danger.backgroundColor, color: danger.color, fontSize: danger.fontSize, paddingTop: danger.paddingTop,
|
||||
paddingLeft: danger.paddingLeft, borderTopColor: danger.borderTopColor }] },
|
||||
};
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`the Delete… review matches the concept's reviewed archive ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const concept = await conceptReview(page, theme);
|
||||
expect(concept.review.tree).toEqual(['dl.meta-grid', 'p.small', 'div.dialog-actions']);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-archive-review-${theme}-${width}.png`) });
|
||||
const production = await productionReview(page, request, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-delete-review-${theme}-${width}.png`) });
|
||||
expect(diffInventory(mapDeleteConcept(concept), production)).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
test('the Delete… review comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const concept = await conceptReview(page, 'dark');
|
||||
await productionReview(page, request, 'dark');
|
||||
const expected = mapDeleteConcept(concept);
|
||||
const paths = async () => diffInventory(expected, await page.evaluate(measureReview)).map(item => item.path);
|
||||
expect(await paths()).toEqual([]);
|
||||
await mutate(page, { css: '#modal-body .dialog-actions .danger { padding-left: 2px }' });
|
||||
expect(await paths()).toContain('actions.buttons.1.paddingLeft');
|
||||
});
|
||||
|
||||
test('the bar\'s Delete… matches the concept Images bar\'s danger action', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await openConcept(page, 'images', 'dark');
|
||||
await page.locator('#image-grid .media-card').first().locator('input[data-select]').check();
|
||||
await expect(page.locator('#bulkbar .bulkbar')).toBeVisible();await page.mouse.move(0, 0);
|
||||
const reference = await page.evaluate(measureButton, '#bulkbar .bulkbar button.danger');
|
||||
const { paired } = await seedMixed(request);
|
||||
await openProduction(page, 'library', 'dark');
|
||||
await page.locator(`article[data-prompt-id="${paired.id}"] .image-overlays input`).check();
|
||||
await expect(page.locator('#library-bulkbar .bulkbar')).toBeVisible();await page.mouse.move(0, 0);
|
||||
expect(await page.evaluate(measureButton, '#library-bulkbar .bulkbar button.danger')).toEqual(reference);
|
||||
});
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Keep the browser test server's large frees going back to the OS (#516).
|
||||
|
||||
glibc serves an allocation of at least the mmap threshold (128 KiB at start) with its
|
||||
own mmap and unmaps it on free. Freeing such a block also raises the threshold, up to
|
||||
32 MiB, so the server's later image-sized buffers (PIL pixel data, PNG encoding,
|
||||
whole-file reads in the executor threads) come from per-thread malloc arenas instead.
|
||||
Those arenas fragment and keep their pages: a server grew from 13 MB to 0.6-1 GB RSS
|
||||
over one page-workflow shard while its Python heap stayed near 12 MB. Setting
|
||||
M_MMAP_THRESHOLD fixes the threshold at its start value and turns the raising off.
|
||||
"""
|
||||
|
||||
import ctypes
|
||||
import sys
|
||||
|
||||
M_MMAP_THRESHOLD = -3
|
||||
MMAP_THRESHOLD_BYTES = 128 * 1024
|
||||
|
||||
|
||||
def pin_mmap_threshold(libc=None) -> bool:
|
||||
"""Fix glibc's mmap threshold; False where there is no glibc ``mallopt``."""
|
||||
if not sys.platform.startswith("linux"):
|
||||
return False
|
||||
if libc is None:
|
||||
try:
|
||||
libc = ctypes.CDLL(None)
|
||||
except OSError:
|
||||
return False
|
||||
mallopt = getattr(libc, "mallopt", None)
|
||||
if mallopt is None:
|
||||
return False
|
||||
return bool(mallopt(M_MMAP_THRESHOLD, MMAP_THRESHOLD_BYTES))
|
||||
@@ -20,6 +20,7 @@ from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
from tests.browser.folder_trash import close_trash, replace_folder
|
||||
from tests.browser.malloc_tuning import pin_mmap_threshold
|
||||
from tests.browser.request_hold import register_request_holds
|
||||
from tests.browser import slow_reset_watch
|
||||
from tests.browser.fast_reset import SEED_ON_LIVE_ENV, ScratchSeeder
|
||||
@@ -220,6 +221,12 @@ def create_app(root: Path) -> web.Application:
|
||||
PromptManagerConfig.DEFAULT_DB_PATH = str(root / "browser.db")
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [str(root / "output"), str(root / "input")]
|
||||
|
||||
# #480: Delete… never reaches a real system trash from a page test; the fake
|
||||
# renames into this server's root, which goes when the server exits.
|
||||
from tests import fake_trash
|
||||
|
||||
fake_trash.install(root / "fake-trash")
|
||||
|
||||
from py.api import PromptManagerAPI
|
||||
from py import autotag
|
||||
from utils.hashing import generate_prompt_hash
|
||||
@@ -749,6 +756,8 @@ def main(argv: list[str] | None = None) -> None:
|
||||
help="shut down when stdin closes (the launcher's parent-death guard)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
# Before any thread allocates (#516): large frees then return to the OS.
|
||||
pin_mmap_threshold()
|
||||
listener = bind_listener(args.port)
|
||||
# One machine-readable line tells the launcher which port this process holds.
|
||||
print(f"{PORT_ANNOUNCEMENT}{listener.getsockname()[1]}", flush=True)
|
||||
|
||||
@@ -87,8 +87,8 @@ function measureBackups(mainSelector) {
|
||||
}
|
||||
|
||||
const SCHEDULE_REASON = 'Daily and Weekly are not available yet: back up with Back up whenever you choose.';
|
||||
const SAFETY_DESCRIPTION = 'Keep a verified copy of the database on disk before maintenance, cleanups and LoRA imports change ' +
|
||||
'records. A restore always keeps one.';
|
||||
const SAFETY_DESCRIPTION = 'Keep a verified copy of the database on disk before maintenance, cleanups, LoRA imports and ' +
|
||||
'prompt deletes change records. A restore always keeps one.';
|
||||
|
||||
/**
|
||||
* D103: "Back up" (the concept's "Simulate backup"), the status line from /backup/status, "Export JSON" (its "Export
|
||||
|
||||
@@ -527,7 +527,7 @@ test('Add to collection creates a collection and adds to it ignoring case (CUJ 1
|
||||
await expect(imageCards(page)).toHaveCount(21);
|
||||
for (const file of ['loose.png', 'landscape_01.png', 'experiment_01.png']) await check(page, file);
|
||||
await expect(bar(page).locator('button')).toHaveText(['Add tags', 'Auto-tag selected', 'Set category', 'Add to collection',
|
||||
'Export metadata', 'Clear']);
|
||||
'Export metadata', 'Delete…', 'Clear']);
|
||||
await barButton(page, 'Add to collection').click();
|
||||
// The concept's simpleInput with Add (D138) and the existing names as suggestions.
|
||||
await expect(addDialog(page).locator('label')).toHaveText('Collection name');
|
||||
@@ -578,7 +578,7 @@ test('Remove from collection takes the checked images off the open collection (C
|
||||
await check(page, 'landscape_01.png');
|
||||
await check(page, 'landscape_02.png');
|
||||
await expect(bar(page).locator('button')).toHaveText(['Add tags', 'Auto-tag selected', 'Set category', 'Add to collection',
|
||||
'Remove from collection', 'Export metadata', 'Clear']);
|
||||
'Remove from collection', 'Export metadata', 'Delete…', 'Clear']);
|
||||
await barButton(page, 'Remove from collection').click();
|
||||
await expect(toast(page)).toHaveText('Removed 2 images from “Landscapes”.');
|
||||
await expect(imageCards(page)).toHaveCount(10);
|
||||
@@ -715,7 +715,8 @@ test('Collections… in the side panel adds, removes and creates for one image (
|
||||
// The newest image, landscape_01, is the panel's.
|
||||
await expect(panel(page).getByRole('heading', { level: 2 })).toHaveText('landscape_01.png');
|
||||
const open = panel(page).getByRole('button', { name: 'Collections…', exact: true });
|
||||
await expect(panel(page).locator('.detail-footer > button').last()).toHaveText('Collections…');
|
||||
// #480 D159: Delete… follows Collections… at the end of the panel's actions.
|
||||
await expect(panel(page).locator('.detail-footer > button').nth(-2)).toHaveText('Collections…');
|
||||
await open.click();
|
||||
await expect(collectionsDialog(page)).toBeVisible();
|
||||
await expect(choice(page, 'Landscapes')).toBeChecked();
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
|
||||
import {saveImageTags} from './concept-inventory.mjs';
|
||||
import {readFile,writeFile} from 'node:fs/promises';
|
||||
import {createHash} from 'node:crypto';
|
||||
|
||||
@@ -495,23 +496,38 @@ test('the inline current file names its prompts apart from the frozen run fields
|
||||
expect(await exact(dialog(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'))).toBe(before);
|
||||
});
|
||||
|
||||
// #449 (D154): History's inline current file draws tags in the shared labelled sections, named with its context so they
|
||||
// stay apart from the frozen run's fields. The server sends History no image_tags facet today (D88), so the read is
|
||||
// given one here to prove the shared render in History.
|
||||
test('the inline current file labels its image tags after its negative prompt',async({page})=>{
|
||||
// #449 (D154), #457: History's inline current file draws the file's own image tags in the shared labelled section, named
|
||||
// with its context so they stay apart from the frozen run's fields. The tags are saved on the real file through the Tag
|
||||
// routes and come from History's own current-file read (its `image_tags` facet), as Images reads them.
|
||||
test('the inline current file labels its image tags after its negative prompt',async({page,request})=>{
|
||||
const reads=[];
|
||||
await page.route(url=>url.pathname.endsWith('/inspection'),async route=>{const response=await route.fetch();const body=await response.json();
|
||||
reads.push(body.image_tags);
|
||||
body.prompts={positive:{status:'resolved',text:'current file positive',source:'graph-inferred'},negative:{status:'resolved',text:'current file negative',source:'graph-inferred'}};
|
||||
body.image_tags={state:'current',tags:['history_tag']};
|
||||
await route.fulfill({response,json:body});});
|
||||
await open(page,page.fixture.detail_id);await marker(page,page.fixture.runs.shared).click();await expect(dialog(page).locator('[data-receipt]')).toHaveCount(5);
|
||||
await dialog(page).getByRole('button',{name:'Raw',exact:true}).click();
|
||||
await dialog(page).locator('[data-receipt]').first().getByRole('button',{name:'View current file',exact:true}).click();
|
||||
await expect(page.locator('dialog#viewer-dialog')).toBeVisible();await page.keyboard.press('Escape');await expect(dialog(page)).toBeVisible();
|
||||
const current=dialog(page).getByRole('region',{name:'Current reported file',exact:true});
|
||||
const tags=current.getByRole('region',{name:'Current file image tags',exact:true});
|
||||
await expect(tags.locator('.role-label')).toHaveText('Image tags · 1');await expect(tags.locator('.tags > .chip')).toHaveText(['history_tag']);
|
||||
const showCurrent=async()=>{
|
||||
await open(page,page.fixture.detail_id);await marker(page,page.fixture.runs.shared).click();await expect(dialog(page).locator('[data-receipt]')).toHaveCount(5);
|
||||
await dialog(page).getByRole('button',{name:'Raw',exact:true}).click();
|
||||
await dialog(page).locator('[data-receipt]').first().getByRole('button',{name:'View current file',exact:true}).click();
|
||||
await expect(page.locator('dialog#viewer-dialog')).toBeVisible();await page.keyboard.press('Escape');await expect(dialog(page)).toBeVisible();
|
||||
await expect(current.getByRole('region',{name:'Current file negative prompt',exact:true})).toBeVisible();
|
||||
};
|
||||
// An untagged file draws no image tags section.
|
||||
await showCurrent();await expect(tags).toHaveCount(0);expect(reads.at(-1)).toEqual({state:'none',tags:[]});
|
||||
await dialog(page).getByRole('button',{name:'Close recorded run',exact:true}).click();
|
||||
expect(await saveImageTags(request,'history/current.png',['history_tag','dusk'])).toEqual(['history_tag','dusk']);
|
||||
await showCurrent();expect(reads.at(-1)).toEqual({state:'current',tags:['history_tag','dusk']});
|
||||
await expect(tags.locator('.role-label')).toHaveText('Image tags · 2');await expect(tags.locator('.tags > .chip')).toHaveText(['history_tag','dusk']);
|
||||
const negative=current.getByRole('region',{name:'Current file negative prompt',exact:true});
|
||||
expect(await negative.evaluate((node,section)=>({after:Boolean(node.compareDocumentPosition(section)&Node.DOCUMENT_POSITION_FOLLOWING),inside:node.contains(section)}),await tags.elementHandle())).toEqual({after:true,inside:false});
|
||||
// The viewer opened from History shares the inspection: the same section, plain chips (no Tags filter there, D88).
|
||||
await dialog(page).locator('[data-receipt]').first().getByRole('button',{name:'View current file',exact:true}).click();
|
||||
const viewerTags=page.locator('dialog#viewer-dialog #viewer-details section.tag-section.image-tags');
|
||||
await expect(viewerTags.locator('.role-label')).toHaveText('Image tags · 2');
|
||||
await expect(viewerTags.locator('span.chip')).toHaveText(['history_tag','dusk']);await expect(viewerTags.locator('button')).toHaveCount(0);
|
||||
await page.keyboard.press('Escape');await expect(dialog(page)).toBeVisible();
|
||||
});
|
||||
|
||||
// #332 CUJ 5 (MC8, §5.3): History's current-file inspector reads the file's own category with the Images key rule, so
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConceptAt, openProduction } from './concept-inventory.mjs';
|
||||
|
||||
// #480 (D159): Images › Delete…: the review asks every time (nothing preselected), "Remove from the library only"
|
||||
// hides the file and keeps it on disk, "Also move the files to the trash" moves it (the server's fake trash here).
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
const CARDS = '#workspace-main article[data-image-key]';
|
||||
const bar = page => page.locator('#workspace-main #bulkbar');
|
||||
const review = page => page.getByRole('dialog', { name: /^Delete \d+ images?$/ });
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConceptAt(page, request);
|
||||
page.deleteErrors = [];
|
||||
page.on('pageerror', error => page.deleteErrors.push(error.message));
|
||||
page.on('dialog', dialog => { page.deleteErrors.push(`browser ${dialog.type()}`); dialog.dismiss().catch(() => {}); });
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.deleteErrors).toEqual([]);
|
||||
});
|
||||
|
||||
async function openImages(page) {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await openProduction(page, 'images', 'dark');
|
||||
await expect(page.locator(CARDS).first()).toBeVisible();
|
||||
return page.locator(CARDS).count();
|
||||
}
|
||||
|
||||
async function check(page, index) {
|
||||
const card = page.locator(CARDS).nth(index);
|
||||
const key = await card.getAttribute('data-image-key');
|
||||
await card.locator('.image-overlays input[type="checkbox"]').check();
|
||||
return key;
|
||||
}
|
||||
|
||||
async function deleteChecked(page, mode) {
|
||||
await bar(page).getByRole('button', { name: 'Delete…', exact: true }).click();
|
||||
const dialog = review(page);
|
||||
await expect(dialog).toBeVisible();
|
||||
const primary = dialog.locator('.dialog-actions button.danger');
|
||||
// Ask every time: no mode is preselected and the primary waits for one.
|
||||
await expect(dialog.getByRole('radio')).toHaveCount(2);
|
||||
for (const radio of await dialog.getByRole('radio').all()) await expect(radio).not.toBeChecked();
|
||||
await expect(primary).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(dialog.locator('dl.meta-grid')).toBeHidden();
|
||||
const label = mode === 'trash' ? 'Also move the files to the trash' : 'Remove from the library only';
|
||||
await dialog.getByRole('radio', { name: new RegExp(`^${label}`) }).check();
|
||||
await expect(dialog.locator('dl.meta-grid')).toBeVisible();
|
||||
await expect(primary).not.toHaveAttribute('aria-disabled', 'true');
|
||||
return { dialog, primary };
|
||||
}
|
||||
|
||||
for (const mode of ['library', 'trash']) {
|
||||
test(`${mode}: the deleted image leaves Images and stays gone after a reload`, async ({ page }) => {
|
||||
const before = await openImages(page);
|
||||
const key = await check(page, 0);
|
||||
const { dialog, primary } = await deleteChecked(page, mode);
|
||||
await expect(dialog.locator('dt', { hasText: 'Media on disk' })).toBeVisible();
|
||||
await expect(primary).toHaveText(mode === 'trash' ? 'Move 1 image to the trash' : 'Remove 1 image from the library');
|
||||
await primary.click();
|
||||
await expect(toast(page)).toContainText(mode === 'trash' ? 'Moved 1 image to the trash.' : 'Removed 1 image from the library.');
|
||||
await expect(page.locator(`${CARDS}[data-image-key="${key}"]`)).toHaveCount(0);
|
||||
await expect(page.locator(CARDS)).toHaveCount(before - 1);
|
||||
await expect(bar(page)).toBeEmpty();
|
||||
await page.reload();
|
||||
await expect(page.locator(CARDS).first()).toBeVisible();
|
||||
await expect(page.locator(`${CARDS}[data-image-key="${key}"]`)).toHaveCount(0);
|
||||
});
|
||||
}
|
||||
|
||||
// #501: the sidebar's Images count (read once at mount) follows a finished delete without a reload.
|
||||
test('a finished delete re-reads the sidebar Images count', async ({ page }) => {
|
||||
await openImages(page);
|
||||
const badge = page.locator('#primary-nav a[data-route="images"] .nav-count');
|
||||
await expect(badge).toHaveText(/^\d[\d,]*$/);
|
||||
const before = Number((await badge.textContent()).replace(/,/g, ''));
|
||||
await check(page, 0);
|
||||
const { primary } = await deleteChecked(page, 'library');
|
||||
await primary.click();
|
||||
await expect(toast(page)).toContainText('Removed 1 image from the library.');
|
||||
await expect(badge).toHaveText((before - 1).toLocaleString('en-US'));
|
||||
});
|
||||
|
||||
test('Cancel changes nothing', async ({ page }) => {
|
||||
const before = await openImages(page);
|
||||
await check(page, 0);
|
||||
const { dialog } = await deleteChecked(page, 'library');
|
||||
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog).toBeHidden();
|
||||
await expect(page.locator(CARDS)).toHaveCount(before);
|
||||
});
|
||||
|
||||
test('switching modes redraws the rows from one review', async ({ page }) => {
|
||||
await openImages(page);
|
||||
await check(page, 0);
|
||||
let previews = 0;
|
||||
page.on('request', request => { if (request.url().endsWith('/images/delete/preview')) previews += 1; });
|
||||
const { dialog, primary } = await deleteChecked(page, 'library');
|
||||
await expect(dialog.locator('dd', { hasText: 'Files stay on disk, hidden from Images.' })).toBeVisible();
|
||||
await dialog.getByRole('radio', { name: /^Also move the files to the trash/ }).check();
|
||||
await expect(dialog.locator('dd', { hasText: 'Nothing is deleted permanently' })).toBeVisible();
|
||||
await expect(primary).toHaveText('Move 1 image to the trash');
|
||||
expect(previews).toBe(1);
|
||||
});
|
||||
|
||||
test('Delete… in the inspector reviews that one image', async ({ page }) => {
|
||||
const before = await openImages(page);
|
||||
const card = page.locator(CARDS).first();
|
||||
await card.getByRole('button', { name: /^Inspect / }).click();
|
||||
const button = page.locator('#workspace-main .file-inspector button.danger', { hasText: 'Delete…' });
|
||||
await expect(button).toBeVisible();
|
||||
await button.click();
|
||||
const dialog = review(page);
|
||||
await expect(dialog).toBeVisible();
|
||||
await dialog.getByRole('radio', { name: /^Remove from the library only/ }).check();
|
||||
await dialog.locator('.dialog-actions button.danger').click();
|
||||
await expect(page.locator(CARDS)).toHaveCount(before - 1);
|
||||
});
|
||||
|
||||
test('Delete… in the viewer closes it after the delete', async ({ page }) => {
|
||||
const before = await openImages(page);
|
||||
await page.locator(CARDS).first().getByRole('button', { name: /^Open / }).click();
|
||||
const viewer = page.locator('dialog#viewer-dialog[open]');
|
||||
await expect(viewer).toBeVisible();
|
||||
await viewer.getByRole('button', { name: 'Delete…', exact: true }).click();
|
||||
const dialog = review(page);
|
||||
await expect(dialog).toBeVisible();
|
||||
await dialog.getByRole('radio', { name: /^Remove from the library only/ }).check();
|
||||
await dialog.locator('.dialog-actions button.danger').click();
|
||||
await expect(viewer).toBeHidden();
|
||||
await expect(page.locator(CARDS)).toHaveCount(before - 1);
|
||||
});
|
||||
|
||||
// The review never starts a delete without an explicit choice (live test 2026-10-29): every activation of the
|
||||
// aria-disabled primary is ignored, the dialog has no form to submit, and a chosen primary starts once.
|
||||
test('without a chosen mode nothing can start a delete', async ({ page }) => {
|
||||
const before = await openImages(page);
|
||||
await check(page, 0);
|
||||
const applies = [];
|
||||
page.on('request', request => { if (request.url().includes('/maintenance/apply')) applies.push(request.postDataJSON()); });
|
||||
await bar(page).getByRole('button', { name: 'Delete…', exact: true }).click();
|
||||
const dialog = review(page);
|
||||
await expect(dialog).toBeVisible();
|
||||
const primary = dialog.locator('.dialog-actions button.danger');
|
||||
await expect(primary).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(primary).toHaveText('Choose what happens to the files');
|
||||
// Visibly dimmed while no mode is chosen.
|
||||
await expect(primary).toHaveCSS('opacity', '0.55');
|
||||
await expect(primary).toHaveCSS('cursor', 'not-allowed');
|
||||
await primary.click({ force: true });
|
||||
await primary.dblclick({ force: true });
|
||||
await primary.focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await page.keyboard.press('Space');
|
||||
await dialog.locator('#modal-title').click();
|
||||
await page.keyboard.press('Enter');
|
||||
expect(await dialog.locator('form').count()).toBe(0);
|
||||
await page.evaluate(() => document.querySelector('dialog#modal[open]')?.dispatchEvent(new Event('submit', { bubbles: true })));
|
||||
await expect(dialog).toBeVisible();
|
||||
expect(applies).toEqual([]);
|
||||
for (const radio of await dialog.getByRole('radio').all()) await expect(radio).not.toBeChecked();
|
||||
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(page.locator(CARDS)).toHaveCount(before);
|
||||
expect(applies).toEqual([]);
|
||||
});
|
||||
|
||||
test('a chosen mode starts exactly one delete, with that mode, however often it is pressed', async ({ page }) => {
|
||||
const before = await openImages(page);
|
||||
await check(page, 0);
|
||||
const applies = [];
|
||||
page.on('request', request => { if (request.url().includes('/maintenance/apply')) applies.push(request.postDataJSON()); });
|
||||
const { primary } = await deleteChecked(page, 'library');
|
||||
await primary.dblclick();
|
||||
await expect(toast(page)).toContainText('Removed 1 image from the library.');
|
||||
await expect(page.locator(CARDS)).toHaveCount(before - 1);
|
||||
expect(applies.length).toBe(1);
|
||||
expect(applies[0].mode).toBe('library');
|
||||
expect(Object.keys(applies[0]).sort()).toEqual(['mode', 'token']);
|
||||
});
|
||||
@@ -1376,6 +1376,61 @@ async function expectStar(button, favorite) {
|
||||
await expect(button).toHaveAttribute('aria-pressed', String(favorite));
|
||||
}
|
||||
|
||||
// #494: the Favorites count is re-read 300 ms after a star changes. A page that has begun to leave starts no count read
|
||||
// (WebKit fails it with "Fetch API cannot load … due to access control checks.", reported as a page error; queue run
|
||||
// 40934368716). A leave that is called off counts again on the next input.
|
||||
for (const leave of ['beforeunload', 'pagehide']) {
|
||||
test(`a star changed just before ${leave} starts no Favorites count read while the page leaves`, async ({ page }) => {
|
||||
await page.clock.install();
|
||||
const countReads = [];
|
||||
page.on('request', item => {
|
||||
const url = new URL(item.url());
|
||||
if (url.pathname === '/prompt_manager/images/output' && url.searchParams.get('favorites') === '1' && url.searchParams.get('limit') === '1') countReads.push(url.search);
|
||||
});
|
||||
await page.goto(workspace);
|
||||
const card = page.locator('article[data-image-key="0:shared/same.png"]');
|
||||
await expectStar(star(card, 'same.png'), false);
|
||||
await expect.poll(() => countReads.length).toBe(1);
|
||||
await star(card, 'same.png').click();
|
||||
await expectStar(star(card, 'same.png'), true);
|
||||
await expect(toast(page)).toHaveText('Added to favorites.');
|
||||
await page.evaluate(type => window.dispatchEvent(new Event(type)), leave);
|
||||
await page.clock.runFor(1000);
|
||||
// A read made after the clock ran is listed, so the listener saw every count read.
|
||||
await page.evaluate(() => fetch('/prompt_manager/images/output?mode=workspace&favorites=1&limit=1&probe=1'));
|
||||
await expect.poll(() => countReads.length).toBe(2);
|
||||
expect(countReads.at(-1)).toContain('probe=1');
|
||||
if (leave === 'beforeunload') {
|
||||
// The leave was called off (the page is still here): the next input counts again, once, and the badge shows it.
|
||||
await page.keyboard.press('Shift');
|
||||
await expect.poll(() => countReads.length).toBe(3);
|
||||
expect(countReads.at(-1)).not.toContain('probe=1');
|
||||
await expect(page.locator('nav [data-route="favorites"] .nav-count')).toHaveText('1');
|
||||
await page.clock.runFor(1000);
|
||||
expect(countReads).toHaveLength(3);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('a count read in flight when the page starts to leave is dropped, then read again if the page stays (#494)', async ({ page }) => {
|
||||
const held = [];
|
||||
// The page's first Library count read is held at the server boundary while the page starts to leave.
|
||||
await page.route('**/prompt_manager/library?limit=1', route => { held.push(route); });
|
||||
await page.goto(workspace);
|
||||
await expect.poll(() => held.length).toBe(1);
|
||||
const failed = page.waitForEvent('requestfailed', item => item.url().includes('/prompt_manager/library?limit=1'));
|
||||
await page.evaluate(() => window.dispatchEvent(new Event('beforeunload')));
|
||||
await failed;
|
||||
await page.unroute('**/prompt_manager/library?limit=1');
|
||||
await held[0].continue().catch(() => {});
|
||||
await expect(page.locator('nav [data-route="library"] .nav-count')).toHaveCount(0);
|
||||
// The page stayed: the next input reads the dropped count again.
|
||||
const reread = page.waitForRequest(item => item.url().includes('/prompt_manager/library?limit=1'));
|
||||
await page.keyboard.press('Shift');
|
||||
await reread;
|
||||
await expect(page.locator('nav [data-route="library"] .nav-count')).toHaveText(/^\d+$/);
|
||||
});
|
||||
|
||||
test('favorite persists across reload', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/reset', { data: { include_video: true } })).toBeOK();
|
||||
const sent = [];
|
||||
@@ -1794,7 +1849,7 @@ test('card checkboxes select images into the concept bar and Clear empties it',
|
||||
await checkbox(first, 'same.png').check();
|
||||
await expect(bulkbar(page).locator('.bulkbar > strong')).toHaveText('1 selected');
|
||||
// D85: Add tags (the concept's) and Auto-tag selected lead the real actions.
|
||||
await expect(barButtons(page)).toHaveText(['Add tags', 'Auto-tag selected', 'Set category', 'Add to collection', 'Export metadata', 'Clear']);
|
||||
await expect(barButtons(page)).toHaveText(['Add tags', 'Auto-tag selected', 'Set category', 'Add to collection', 'Export metadata', 'Delete…', 'Clear']);
|
||||
// Toggling neither opens the viewer, nor inspects, nor moves focus.
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
await expect(checkbox(first, 'same.png')).toBeFocused();
|
||||
|
||||
@@ -704,6 +704,147 @@ test('a video replaced while still starting is reset once it settles', async ({
|
||||
}
|
||||
});
|
||||
|
||||
test('a new selection while the viewer\'s own video is still starting leaves it until it settles (#483)', async ({ context, browserName }) => {
|
||||
test.setTimeout(180000);
|
||||
// D155 opens the viewer once the inline preview has settled; the viewer then starts a video of its own. A new
|
||||
// selection from that video's own `loadstart` (readyState HAVE_NOTHING), or a few milliseconds after it, must not
|
||||
// reset it or take it off the page while it starts (WebKit, GStreamer: TESTING.md D62; merge-queue run
|
||||
// 40903238387). The viewer stays open until it reports metadata or an error, then closes; the URL goes last.
|
||||
for (let attempt = 0; attempt < 10; attempt++) {
|
||||
const page = await context.newPage();
|
||||
try {
|
||||
if (attempt === 0) await startCoverage(page, browserName);
|
||||
await page.addInitScript(() => {
|
||||
const load = HTMLMediaElement.prototype.load;
|
||||
HTMLMediaElement.prototype.load = function () { if (!this.hasAttribute('src')) { this.__resets = (this.__resets ?? 0) + 1; this.__resetReadyState ??= this.readyState; } return load.call(this); };
|
||||
const revoke = URL.revokeObjectURL;
|
||||
URL.revokeObjectURL = url => {
|
||||
const starting = window.__starting;
|
||||
if (url === starting?.url) starting.revoked = { src: starting.node.hasAttribute('src'), connected: starting.node.isConnected, resets: starting.node.__resets ?? 0 };
|
||||
return revoke(url);
|
||||
};
|
||||
});
|
||||
await openInspect(page);
|
||||
await page.evaluate(() => {
|
||||
const input = document.querySelector('#local-file');
|
||||
new MutationObserver((records, observer) => {
|
||||
const video = document.querySelector('#viewer-dialog video');
|
||||
if (!video) return;
|
||||
observer.disconnect();
|
||||
const replace = () => {
|
||||
window.__starting = { node: video, url: video.getAttribute('src'), readyState: video.readyState };
|
||||
input.files = window.__pngFiles; input.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
window.__starting.replaced = { connected: video.isConnected, src: video.hasAttribute('src'), resets: video.__resets ?? 0 };
|
||||
};
|
||||
video.addEventListener('loadstart', () => { if (window.__delay) setTimeout(replace, window.__delay); else replace(); }, { once: true });
|
||||
}).observe(document.body, { childList: true, subtree: true });
|
||||
});
|
||||
const png = parityPng('f02.png');
|
||||
await page.evaluate(bytes => {
|
||||
const transfer = new DataTransfer();
|
||||
transfer.items.add(new File([new Uint8Array(bytes)], 'f02.png', { type: 'image/png' }));
|
||||
window.__pngFiles = transfer.files;
|
||||
}, [...png.buffer]);
|
||||
await page.evaluate(delay => { window.__delay = delay; }, attempt % 5);
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible({ timeout: 8000 });
|
||||
await expect.poll(() => page.evaluate(() => window.__starting?.revoked ?? null)).toEqual({ src: false, connected: false, resets: 1 });
|
||||
const { readyState, replaced } = await page.evaluate(() => ({ readyState: window.__starting.readyState, replaced: window.__starting.replaced }));
|
||||
if (attempt % 5 === 0) expect(readyState).toBe(0);
|
||||
// Starting when replaced: still on the page with its URL, never reset; settled: released at once.
|
||||
expect(replaced).toEqual(readyState === 0 ? { connected: true, src: true, resets: 0 } : { connected: false, src: false, resets: 1 });
|
||||
expect(await page.evaluate(() => window.__starting.node.__resets)).toBe(1);
|
||||
// Its one reset came after it had metadata, never while it was starting.
|
||||
expect(await page.evaluate(() => window.__starting.node.__resetReadyState)).toBeGreaterThanOrEqual(1);
|
||||
// The new file's viewer opens after the old one closed, and the page still answers.
|
||||
await expect(viewer(page).getByRole('img', { name: 'f02.png', exact: true })).toBeVisible({ timeout: 8000 });
|
||||
await expect(viewer(page).locator('video')).toHaveCount(0);
|
||||
if (attempt === 0) await saveCoverage(page, browserName);
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
/** #483: holds the viewer's video in HAVE_NOTHING (its `loadedmetadata` never fires) so a hold can be driven by hand. */
|
||||
async function stallViewerVideo(page) {
|
||||
await page.addInitScript(() => {
|
||||
const describe = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'readyState');
|
||||
Object.defineProperty(HTMLMediaElement.prototype, 'readyState', { configurable: true, get() {
|
||||
return this.closest?.('#viewer-dialog') && window.__stallViewer ? 0 : describe.get.call(this);
|
||||
} });
|
||||
window.addEventListener('loadedmetadata', event => {
|
||||
if (window.__stallViewer && event.target.closest?.('#viewer-dialog')) event.stopImmediatePropagation();
|
||||
}, true);
|
||||
});
|
||||
}
|
||||
|
||||
test('a held viewer closed by hand, or replaced twice, opens only the latest file\'s viewer at once (#483)', async ({ page }) => {
|
||||
await stallViewerVideo(page);
|
||||
await openInspect(page);
|
||||
await page.evaluate(() => { window.__stallViewer = true; });
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
// Two selections while the viewer's video is (made to look) still starting: the viewer stays, untouched.
|
||||
await chooser(page).setInputFiles(parityPng('f02.png', 'first.png'));
|
||||
await chooser(page).setInputFiles(parityPng('f02.png', 'second.png'));
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
// Open file metadata pressed meanwhile (the page behind the viewer is inert, so by script): the latest request.
|
||||
await action(page).evaluate(node => node.click());
|
||||
// Closing it by hand ends the hold at once: the latest file's viewer opens, once, well before D62's 15 s.
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer(page).getByRole('img', { name: 'second.png', exact: true })).toBeVisible({ timeout: 5000 });
|
||||
await expect(viewer(page).getByRole('img', { name: 'first.png', exact: true })).toHaveCount(0);
|
||||
await expect(viewer(page).locator('video')).toHaveCount(0);
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(1);
|
||||
});
|
||||
|
||||
test('a held viewer revokes its URL only once it is gone, drops a stale open, and ends on page leave (#483)', async ({ page }) => {
|
||||
await stallViewerVideo(page);
|
||||
await openInspect(page);
|
||||
await trackReleases(page);
|
||||
await page.evaluate(() => { window.__stallViewer = true; });
|
||||
const videoUrl = () => page.evaluate(() => document.querySelector('#viewer-dialog video')?.getAttribute('src') ?? null);
|
||||
const released = url => page.evaluate(url => window.__released.filter(item => item.url === url), url);
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
const first = await videoUrl();
|
||||
await chooser(page).setInputFiles(parityPng('f02.png', 'held.png'));
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
// The session asked to revoke the video's URL; it waits while the held viewer still plays it.
|
||||
expect(await released(first)).toEqual([]);
|
||||
// An Open asked for during the hold, then an AVI (which never opens the viewer by itself): the open for the earlier
|
||||
// file is stale and is dropped when the hold ends.
|
||||
await action(page).evaluate(node => node.click());
|
||||
const avi = { name: 'clip.avi', mimeType: 'video/x-msvideo', buffer: Buffer.concat([Buffer.from('RIFF'), Buffer.alloc(4), Buffer.from('AVI LIST'), Buffer.alloc(64)]) };
|
||||
await chooser(page).setInputFiles(avi);
|
||||
await expect(page.locator('#workspace-toast')).toHaveText(TOAST);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
await expect.poll(() => released(first)).toEqual([{ url: first, live: 0 }]);
|
||||
// Page leave while held: the viewer goes and the deferred URL is revoked with nothing holding it.
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
const second = await videoUrl();
|
||||
await chooser(page).setInputFiles(parityPng('f02.png', 'leave.png'));
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
expect(await released(second)).toEqual([]);
|
||||
await page.evaluate(() => window.dispatchEvent(new PageTransitionEvent('pagehide', { persisted: true })));
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
await expect.poll(() => released(second)).toEqual([{ url: second, live: 0 }]);
|
||||
// Another route while held: the page's dispose ends the hold and revokes the URL with nothing holding it.
|
||||
await page.evaluate(() => window.dispatchEvent(new PageTransitionEvent('pageshow', { persisted: true })));
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
const third = await videoUrl();
|
||||
await chooser(page).setInputFiles(parityPng('f02.png', 'route.png'));
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
await page.evaluate(() => { window.location.hash = '#/library'; });
|
||||
await expect(main(page).getByRole('heading', { level: 1, name: 'Prompt library', exact: true })).toBeVisible();
|
||||
await expect.poll(() => released(third)).toEqual([{ url: third, live: 0 }]);
|
||||
});
|
||||
|
||||
test('leaving for another route while a video starts waits for it to settle', async ({ context, browserName }) => {
|
||||
test.setTimeout(180000);
|
||||
// A WebKit (GStreamer) video detached before its pipeline had started deadlocked the page (TESTING.md D62); a
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
|
||||
|
||||
// #443 D152: Delete… on the Library bar deletes unpaired prompts, archives paired ones (restorable), keeps protected ones.
|
||||
const route='/prompt_manager/workspace#/library';
|
||||
const PREVIEW='/prompt_manager/library/delete/preview';
|
||||
const DELETE='/prompt_manager/library/delete';
|
||||
const cards=page=>page.locator('article[data-prompt-id]');
|
||||
const card=(page,id)=>page.locator(`article[data-prompt-id="${id}"]`);
|
||||
const box=(page,id)=>card(page,id).locator('.image-overlays input[type="checkbox"]');
|
||||
const bar=page=>page.locator('#library-bulkbar > .bulkbar');
|
||||
const toast=page=>page.locator('.toast');
|
||||
const review=page=>page.getByRole('dialog');
|
||||
const showArchived=page=>page.locator('#workspace-main .heading-actions').getByRole('button',{name:/^Show archived prompts/});
|
||||
|
||||
async function items(request,shelf='active') {
|
||||
const query=shelf==='archived'?'&shelf=archived':'';
|
||||
return (await(await request.get(`/prompt_manager/library?limit=100${query}`)).json()).items;
|
||||
}
|
||||
const byText=(list,text)=>list.find(item=>item.text===text);
|
||||
function recordPosts(page,path) {
|
||||
const posts=[];
|
||||
page.on('request',item=>{if(item.method()==='POST'&&new URL(item.url()).pathname===path)posts.push(item.postDataJSON());});
|
||||
return posts;
|
||||
}
|
||||
async function openReview(page) {
|
||||
await bar(page).getByRole('button',{name:'Delete…',exact:true}).click();
|
||||
await expect(review(page)).toBeVisible();
|
||||
}
|
||||
|
||||
test.beforeEach(async({page,request,browserName})=>{
|
||||
await startCoverage(page,browserName);await expect(await request.post('/__test__/reset',{data:{native_library:true,extra_prompts:2}})).toBeOK();
|
||||
page.libraryErrors=[];page.on('pageerror',error=>page.libraryErrors.push(error.message));
|
||||
});
|
||||
test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.libraryErrors).toEqual([]);});
|
||||
|
||||
test('Delete… reviews the selection, deletes the unpaired, archives the paired and keeps the protected',async({page,request})=>{
|
||||
const before=await items(request);
|
||||
const dog=byText(before,'Dog landscape'),cat=byText(before,'Cat portrait'),guarded=byText(before,'Library extra 0000');
|
||||
await expect(await request.post('/prompt_manager/bulk/tags',{data:{prompt_ids:[guarded.id],tags:['__protected__']}})).toBeOK();
|
||||
const previews=recordPosts(page,PREVIEW),writes=recordPosts(page,DELETE);
|
||||
await page.goto(route);await expect(cards(page)).toHaveCount(before.length);
|
||||
for(const item of [dog,cat,guarded]) await box(page,item.id).check();
|
||||
await expect(bar(page).getByRole('button')).toHaveText([`Select all ${before.length} loaded`,'Add tags','Set category','Delete…','Clear']);
|
||||
await expect(bar(page).getByRole('button',{name:'Delete…',exact:true})).toHaveClass(/danger/);
|
||||
await openReview(page);
|
||||
await expect(review(page)).toHaveAccessibleName('Delete or archive selected prompts');
|
||||
const grid=review(page).locator('dl.meta-grid > div');
|
||||
await expect(grid.locator('dt')).toHaveText(['Delete permanently','Archive','Protected (kept)','Image links removed','Backup']);
|
||||
await expect(grid.locator('dd')).toHaveText(['1 prompt not in a saved pair','1 prompt in a saved pair: hidden, restorable',
|
||||
'1: Library extra 0000','0 images will no longer show these prompts (files stay on disk)','Create and verify before deleting']);
|
||||
await expect(review(page).locator('p.small')).toHaveCount(3);
|
||||
await expect(review(page).getByRole('button',{name:'Delete 1 and archive 1 prompt',exact:true})).toHaveClass(/danger/);
|
||||
// Cancel writes nothing and releases the bar.
|
||||
await review(page).getByRole('button',{name:'Cancel',exact:true}).click();
|
||||
await expect(review(page)).toHaveCount(0);expect(writes).toEqual([]);
|
||||
await expect(bar(page).getByRole('button',{name:'Delete…',exact:true})).not.toHaveAttribute('aria-disabled','true');
|
||||
await openReview(page);
|
||||
await review(page).getByRole('button',{name:'Delete 1 and archive 1 prompt',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText(/^Deleted 1 prompt\. Archived 1 prompt\. 1 protected prompt was kept\. Safety backup saved: \S+\.backup_\S+ \(System › Backup & export\)\.$/);
|
||||
await expect(page.locator('#library-bulkbar')).toBeEmpty();
|
||||
await expect(card(page,dog.id)).toHaveCount(0);await expect(card(page,cat.id)).toHaveCount(0);await expect(card(page,guarded.id)).toHaveCount(1);
|
||||
const epoch=(await(await request.get('/prompt_manager/library/options')).json()).library_epoch;
|
||||
// One preview per review opened (Cancel, then Delete…).
|
||||
const reviewed={prompt_ids:[dog.id,cat.id,guarded.id],expected_epoch:epoch};
|
||||
expect(previews).toEqual([reviewed,reviewed]);
|
||||
expect(writes).toEqual([{archive_ids:[cat.id],delete_ids:[dog.id],expected_epoch:epoch,backup:'on'}]);
|
||||
const after=await items(request);
|
||||
expect(byText(after,'Dog landscape')).toBeUndefined();expect(byText(after,'Cat portrait')).toBeUndefined();
|
||||
expect((await items(request,'archived')).map(item=>item.id)).toEqual([cat.id]);
|
||||
await expect(showArchived(page)).toHaveText('Show archived prompts (1)');
|
||||
});
|
||||
|
||||
test('Show archived prompts lists archived prompts, and Restore on its bar returns them',async({page,request})=>{
|
||||
const before=await items(request);const cat=byText(before,'Cat portrait'),dual=byText(before,'Dual role portrait');
|
||||
await page.goto(route);await expect(cards(page)).toHaveCount(before.length);
|
||||
await box(page,cat.id).check();await box(page,dual.id).check();
|
||||
await openReview(page);
|
||||
await expect(review(page)).toHaveAccessibleName('Archive selected prompts');
|
||||
// The archive reason is the saved pair, for one or many prompts (live note on #470): never negative variants.
|
||||
await expect(review(page).locator('dl.meta-grid > div').filter({hasText:'Archive'}).locator('dd')).toHaveText('2 prompts in a saved pair: hidden, restorable');
|
||||
await review(page).getByRole('button',{name:'Archive 2 prompts',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('Archived 2 prompts.');
|
||||
await expect(cards(page)).toHaveCount(before.length-2);
|
||||
await expect(showArchived(page)).toHaveAttribute('aria-pressed','false');
|
||||
await showArchived(page).click();
|
||||
await expect(showArchived(page)).toHaveAttribute('aria-pressed','true');
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
await expect(page.locator('.section-caption > span').first()).toHaveText(/^2 archived prompts · \d+ negative variants? hidden with them$/);
|
||||
await expect(card(page,cat.id).locator('.card-title .chip',{hasText:'Archived'})).toHaveCount(1);
|
||||
await box(page,cat.id).check();
|
||||
await expect(bar(page).getByRole('button')).toHaveText(['Select all 2 loaded','Restore','Clear']);
|
||||
await bar(page).getByRole('button',{name:'Restore',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('Restored 1 prompt to the Prompt library.');
|
||||
await expect(cards(page)).toHaveCount(1);
|
||||
await showArchived(page).click();
|
||||
await expect(cards(page)).toHaveCount(before.length-1);
|
||||
await expect(card(page,cat.id)).toHaveCount(1);
|
||||
expect((await items(request,'archived')).map(item=>item.id)).toEqual([dual.id]);
|
||||
// An empty archived view says so.
|
||||
await showArchived(page).click();await box(page,dual.id).check();
|
||||
await bar(page).getByRole('button',{name:'Restore',exact:true}).click();
|
||||
await expect(page.locator('.library-status')).toHaveText('No archived prompts.');
|
||||
});
|
||||
|
||||
test('an archived prompt\'s page says so and restores it from its notice',async({page,request})=>{
|
||||
const cat=byText(await items(request),'Cat portrait');
|
||||
const epoch=(await(await request.get('/prompt_manager/library/options')).json()).library_epoch;
|
||||
await expect(await request.post(DELETE,{data:{archive_ids:[cat.id],delete_ids:[],expected_epoch:epoch,backup:'off'}})).toBeOK();
|
||||
await page.goto(`/prompt_manager/workspace#/library/${cat.id}`);
|
||||
const notice=page.locator('.library-archived-notice');
|
||||
await expect(notice.locator('h2')).toHaveText('Archived');
|
||||
// The notice says why (the saved pair) and counts the prompt's negative variants (live note on #470).
|
||||
expect(cat.variant_count).toBeGreaterThan(1);
|
||||
await expect(notice.locator('p').first()).toHaveText(`This prompt is archived because it is in a saved pair. It and its ${cat.variant_count} negative variants are hidden from the Prompt library; nothing was removed.`);
|
||||
await notice.getByRole('button',{name:'Restore this prompt to the Prompt library'}).click();
|
||||
await expect(toast(page)).toHaveText('Restored 1 prompt to the Prompt library.');
|
||||
await expect(notice).toHaveCount(0);
|
||||
expect(byText(await items(request),'Cat portrait')?.id).toBe(cat.id);
|
||||
});
|
||||
|
||||
// #443: a starved renderer (headless WebKit under CPU load, a background tab) can hold animation frames for
|
||||
// seconds; a loaded page of prompts is shown without waiting for one, on first load and after a delete.
|
||||
async function holdAnimationFrames(page) {
|
||||
await page.addInitScript(()=>{
|
||||
const request=window.requestAnimationFrame.bind(window),cancel=window.cancelAnimationFrame.bind(window);
|
||||
const held=new Map(),real=new Map();let holding=true,next=0;
|
||||
window.requestAnimationFrame=callback=>{const id=--next;if(holding)held.set(id,callback);else real.set(id,request(callback));return id;};
|
||||
window.cancelAnimationFrame=id=>{held.delete(id);if(real.has(id))cancel(real.get(id));real.delete(id);};
|
||||
window.__releaseAnimationFrames=()=>{holding=false;for(const [id,callback] of held)real.set(id,request(callback));held.clear();};
|
||||
});
|
||||
}
|
||||
test('a held animation frame does not hide the loaded prompts, on first load or after a delete',async({page,request})=>{
|
||||
const before=await items(request);const dog=byText(before,'Dog landscape');
|
||||
await holdAnimationFrames(page);
|
||||
await page.goto(route);await expect(cards(page)).toHaveCount(before.length);
|
||||
await box(page,dog.id).check();await openReview(page);
|
||||
await review(page).getByRole('button',{name:'Delete 1 prompt',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText(/^Deleted 1 prompt\./);
|
||||
await expect(cards(page)).toHaveCount(before.length-1);await expect(card(page,dog.id)).toHaveCount(0);
|
||||
await page.evaluate(()=>window.__releaseAnimationFrames());
|
||||
await expect(cards(page)).toHaveCount(before.length-1);
|
||||
});
|
||||
|
||||
test('a double press sends one preview, and the bar holds while it is reviewed',async({page,request})=>{
|
||||
const dog=byText(await items(request),'Dog landscape');
|
||||
const previews=recordPosts(page,PREVIEW);
|
||||
let release;const held=new Promise(resolve=>{release=resolve;});
|
||||
await page.route(`**${PREVIEW}`,async item=>{await held;await item.fallback();});
|
||||
await page.goto(route);await box(page,dog.id).check();
|
||||
const remove=bar(page).getByRole('button',{name:/^(Delete…|Reviewing…)$/});
|
||||
await remove.click();
|
||||
await expect(remove).toHaveText('Reviewing…');await expect(remove).toHaveAttribute('aria-disabled','true');
|
||||
await expect(bar(page).getByRole('button',{name:'Clear',exact:true})).toHaveAttribute('aria-disabled','true');
|
||||
await remove.click({force:true});
|
||||
release();
|
||||
await expect(review(page)).toHaveAccessibleName('Delete selected prompts');
|
||||
expect(previews).toHaveLength(1);
|
||||
await review(page).getByRole('button',{name:'Cancel',exact:true}).click();
|
||||
await expect(bar(page).getByRole('button',{name:'Delete…',exact:true})).not.toHaveAttribute('aria-disabled','true');
|
||||
});
|
||||
|
||||
test('a restore since the page loaded, a failed backup, a busy job and an unconfirmed answer each say what happened',async({page,request})=>{
|
||||
const before=await items(request);const dog=byText(before,'Dog landscape');
|
||||
await armUnseenRestores(page);
|
||||
await page.goto(route);await expect(cards(page)).toHaveCount(before.length);
|
||||
// Failed backup: nothing changed, the selection stays.
|
||||
await page.route(`**${DELETE}`,item=>item.fulfill({status:500,contentType:'application/json',
|
||||
body:JSON.stringify({success:false,code:'backup-failed',error:'The safety backup could not be taken and verified; nothing was changed.'})}));
|
||||
await box(page,dog.id).check();await openReview(page);
|
||||
await review(page).getByRole('button',{name:'Delete 1 prompt',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('The safety backup failed. The safety backup could not be taken and verified; nothing was changed.');
|
||||
await expect(box(page,dog.id)).toBeChecked();
|
||||
// Busy: names the job, nothing changed.
|
||||
await page.unroute(`**${DELETE}`);
|
||||
await page.route(`**${DELETE}`,item=>item.fulfill({status:409,contentType:'application/json',
|
||||
body:JSON.stringify({success:false,code:'busy',error:'busy',running:{id:'abc',title:'Scan'}})}));
|
||||
await openReview(page);await review(page).getByRole('button',{name:'Delete 1 prompt',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('Scan is running. Nothing was changed; try again when it finishes.');
|
||||
// Unconfirmed: the shelf reloads.
|
||||
await page.unroute(`**${DELETE}`);
|
||||
await page.route(`**${DELETE}`,item=>item.abort());
|
||||
await openReview(page);await review(page).getByRole('button',{name:'Delete 1 prompt',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('The server did not confirm the change. The Library was reloaded: check the shelf and Show archived prompts.');
|
||||
await expect(page.locator('#library-bulkbar')).toBeEmpty();
|
||||
await page.unroute(`**${DELETE}`);
|
||||
// A restore the page has not seen: the review opened before it is refused on the server.
|
||||
await box(page,dog.id).check();await openReview(page);
|
||||
const backup=await request.get('/prompt_manager/backup');await expect(backup).toBeOK();const buffer=await backup.body();
|
||||
const restore=await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'library.db',mimeType:'application/octet-stream',buffer}}}));
|
||||
await expect(restore).toBeOK();
|
||||
await review(page).getByRole('button',{name:'Delete 1 prompt',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('The library was restored or moved since this page loaded. Nothing was deleted or archived.');
|
||||
expect(byText(await items(request),'Dog landscape')).toBeDefined();
|
||||
});
|
||||
|
||||
test('a failed review keeps the selection, and a selection nothing can change opens the close-only review',async({page,request})=>{
|
||||
const before=await items(request);const guarded=byText(before,'Library extra 0000');
|
||||
await expect(await request.post('/prompt_manager/bulk/tags',{data:{prompt_ids:[guarded.id],tags:['__protected__']}})).toBeOK();
|
||||
await page.goto(route);await expect(cards(page)).toHaveCount(before.length);
|
||||
await box(page,guarded.id).check();
|
||||
await page.route(`**${PREVIEW}`,item=>item.fulfill({status:500,contentType:'application/json',body:JSON.stringify({success:false,code:'review-failed',error:'x'})}),{times:1});
|
||||
await bar(page).getByRole('button',{name:'Delete…',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('The selected prompts could not be reviewed. Try again.');
|
||||
await expect(box(page,guarded.id)).toBeChecked();
|
||||
await expect(bar(page).getByRole('button',{name:'Delete…',exact:true})).not.toHaveAttribute('aria-disabled','true');
|
||||
await openReview(page);
|
||||
await expect(review(page)).toHaveAccessibleName('Delete selected prompts');
|
||||
await expect(review(page).locator('p.small').last()).toHaveText('None of the selected prompts can be deleted or archived.');
|
||||
await expect(review(page).locator('.dialog-actions > button')).toHaveText(['Close']);
|
||||
await review(page).getByRole('button',{name:'Close',exact:true}).click();
|
||||
await expect(bar(page).getByRole('button',{name:'Delete…',exact:true})).not.toHaveAttribute('aria-disabled','true');
|
||||
expect(byText(await items(request),'Library extra 0000')).toBeDefined();
|
||||
});
|
||||
@@ -493,13 +493,15 @@ test('a restore seen finishing keeps New prompt open and reloads the Library aft
|
||||
test('library heading offers Export then + New prompt, and New opens the saved prompt history',async({page,request})=>{
|
||||
await page.goto(`${workspace}#/library`);await expect(page.locator('article[data-prompt-id]').first()).toBeVisible();
|
||||
const actions=headingActions(page).locator('button');
|
||||
expect(await visibleNames(actions)).toEqual(['Export','+ New prompt']);
|
||||
// D152 (#443): Show archived prompts (N) comes first, a plain toggle.
|
||||
expect(await visibleNames(actions)).toEqual(['Show archived prompts (0)','Export','+ New prompt']);
|
||||
await expect(actions.nth(0)).toHaveAttribute('aria-pressed','false');await expect(actions.nth(0)).not.toHaveClass(/ca-primary/);
|
||||
// One image is counted in the singular.
|
||||
const f=page.fixture;await expect(page.locator(`article[data-prompt-id="${f.positive_id}"]`).getByText('1 image',{exact:true})).toHaveCount(1);
|
||||
await expect(actions.nth(0)).toHaveAccessibleName('Export full Library JSON');await expect(actions.nth(0)).not.toHaveClass(/ca-primary/);
|
||||
await expect(actions.nth(1)).toHaveClass(/\bca-primary\b/);
|
||||
for(const index of [0,1])expect((await actions.nth(index).boundingBox()).height).toBeGreaterThanOrEqual(32);
|
||||
await actions.nth(1).click();const dialog=editorDialog(page);await expect(dialog.getByRole('heading',{name:'New prompt',exact:true})).toBeVisible();await ready(dialog);
|
||||
await expect(actions.nth(1)).toHaveAccessibleName('Export full Library JSON');await expect(actions.nth(1)).not.toHaveClass(/ca-primary/);
|
||||
await expect(actions.nth(2)).toHaveClass(/\bca-primary\b/);
|
||||
for(const index of [0,1,2])expect((await actions.nth(index).boundingBox()).height).toBeGreaterThanOrEqual(32);
|
||||
await actions.nth(2).click();const dialog=editorDialog(page);await expect(dialog.getByRole('heading',{name:'New prompt',exact:true})).toBeVisible();await ready(dialog);
|
||||
await field(dialog,'Title').fill('Harbor at dawn');await field(dialog,'Positive prompt').fill('A quiet harbor at dawn');await field(dialog,'Negative prompt (optional)').fill('fog, noise');
|
||||
await button(dialog,'Save').click();await expect(dialog).toHaveCount(0);
|
||||
const created=(await(await request.get('/prompt_manager/library?text=quiet%20harbor')).json()).items[0];
|
||||
@@ -633,7 +635,11 @@ test('restored storage after a confirmed save or before an archive offers an exp
|
||||
const response=await route.fetch();await expect(await restore()).toBeOK();return route.fulfill({response});
|
||||
});
|
||||
await page.getByRole('button',{name:'Edit metadata',exact:true}).click();let dialog=editorDialog(page);await ready(dialog);
|
||||
await field(dialog,'Notes').fill('Lost to a restore');await button(dialog,'Save').click();
|
||||
await field(dialog,'Notes').fill('Lost to a restore');
|
||||
// The save's answer waits for a whole restore in the route above; wait for that answer, not the 5 s expect
|
||||
// default (#501 merge queue: under load the restore outlasted it on WebKit).
|
||||
const saved=page.waitForResponse(item=>item.request().method()==='PATCH'&&new URL(item.url()).pathname===`/prompt_manager/library/${f.positive_id}`);
|
||||
await button(dialog,'Save').click();await saved;
|
||||
await expect(note(dialog)).toHaveText('Saved; Library refresh failed because prompt storage was restored. Reload the Library to see current data.');
|
||||
expect(await actionNames(dialog)).toEqual(['Reload Library','Create text revision','Archive…','Cancel','Save']);
|
||||
for(const name of ['Create text revision','Archive…'])await expect(button(dialog,name)).toBeDisabled();
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
// #444 (D150): the Library's output-folder filter and multi-tag OR in "Refine this view", against the real server, and
|
||||
// the Refine dialog against the concept's Filters dialog (app.js:1345-1352).
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {inventoryOf,diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
const route='/prompt_manager/workspace#/library';
|
||||
const concept='/__test__/canonical/workspace';
|
||||
const cards=page=>page.locator('article[data-prompt-id]');
|
||||
const card=(page,text)=>cards(page).filter({has:page.getByText(text,{exact:true})});
|
||||
const toolbar=page=>page.locator('#workspace-main search.toolbar');
|
||||
const filtersButton=page=>toolbar(page).getByRole('button',{name:/^Filters(?: · \d)?$/});
|
||||
const refine=page=>page.getByRole('dialog',{name:'Refine this view',exact:true});
|
||||
const folder=page=>refine(page).getByRole('combobox',{name:'Folder',exact:true});
|
||||
const tags=page=>refine(page).getByRole('textbox',{name:'Tags',exact:true});
|
||||
const match=page=>refine(page).getByRole('combobox',{name:'Tag match',exact:true});
|
||||
const apply=page=>refine(page).getByRole('button',{name:'Apply filters',exact:true});
|
||||
const caption=page=>page.locator('#workspace-main .section-caption > span').first();
|
||||
|
||||
test.beforeEach(async({page,request,browserName})=>{
|
||||
await startCoverage(page,browserName);await expect(await request.post('/__test__/reset',{data:{native_library:true}})).toBeOK();
|
||||
page.libraryErrors=[];page.on('pageerror',error=>page.libraryErrors.push(error.message));
|
||||
});
|
||||
test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.libraryErrors).toEqual([]);});
|
||||
|
||||
async function open(page) {
|
||||
await page.goto(route);await expect(caption(page)).toHaveText(/^3 positive prompts · /);
|
||||
}
|
||||
/** Open Refine and wait for the folder list it reads on opening. */
|
||||
async function openRefine(page) {
|
||||
const folders=page.waitForResponse(item=>new URL(item.url()).pathname==='/prompt_manager/library/options'&&new URL(item.url()).searchParams.get('include')==='folders');
|
||||
await filtersButton(page).click();await expect(refine(page)).toBeVisible();await folders;
|
||||
}
|
||||
|
||||
test('Refine offers Folder, Tags and Tag match; a folder narrows the shelf to prompts with images there',async({page})=>{
|
||||
await open(page);await openRefine(page);
|
||||
await expect(refine(page).locator('.field label')).toHaveText(['Folder','Tags','Tag match']);
|
||||
// Both roots hold a "shared" folder; the root's own images name no folder.
|
||||
await expect(folder(page).locator('option')).toHaveText(['All output folders','shared']);
|
||||
await expect(match(page).locator('option')).toHaveText(['All of these tags','Any of these tags','Prompts without tags']);
|
||||
await folder(page).selectOption('shared');await apply(page).click();
|
||||
await expect(caption(page)).toHaveText(/^1 positive prompt · /);
|
||||
await expect(cards(page)).toHaveCount(1);await expect(card(page,'Cat portrait')).toHaveCount(1);
|
||||
await expect(filtersButton(page)).toHaveText('Filters · 1');
|
||||
await openRefine(page);await expect(folder(page)).toHaveValue('shared');
|
||||
await refine(page).getByRole('button',{name:'Clear filters',exact:true}).click();
|
||||
await expect(caption(page)).toHaveText(/^3 positive prompts · /);await expect(filtersButton(page)).toHaveText('Filters');
|
||||
});
|
||||
|
||||
test('Tag match Any shows every prompt carrying one of the tags; changing only the match applies it',async({page})=>{
|
||||
await open(page);await openRefine(page);
|
||||
await tags(page).fill('cats, dogs');await apply(page).click();
|
||||
await expect(caption(page)).toHaveText(/^0 positive prompts · /);await expect(filtersButton(page)).toHaveText('Filters · 1');
|
||||
await openRefine(page);await match(page).selectOption('any');await apply(page).click();
|
||||
await expect(caption(page)).toHaveText(/^2 positive prompts · /);
|
||||
await expect(cards(page)).toHaveCount(2);await expect(card(page,'Cat portrait')).toHaveCount(1);await expect(card(page,'Dog landscape')).toHaveCount(1);
|
||||
await expect(filtersButton(page)).toHaveText('Filters · 2');
|
||||
});
|
||||
|
||||
test('a saved view keeps its folder and tag match',async({page})=>{
|
||||
await open(page);await openRefine(page);
|
||||
await folder(page).selectOption('shared');await tags(page).fill('cats, dogs');await match(page).selectOption('any');await apply(page).click();
|
||||
await expect(caption(page)).toHaveText(/^1 positive prompt · /);
|
||||
await toolbar(page).getByRole('button',{name:'Save view',exact:true}).click();
|
||||
const save=page.getByRole('dialog',{name:'Save this view',exact:true});
|
||||
await save.getByRole('textbox',{name:'View name',exact:true}).fill('Shared cats or dogs');
|
||||
const created=page.waitForResponse(item=>new URL(item.url()).pathname==='/prompt_manager/saved-views'&&item.request().method()==='POST');
|
||||
await save.getByRole('button',{name:'Save',exact:true}).click();
|
||||
const body=(await created).request().postDataJSON();
|
||||
expect(body.state).toMatchObject({folder:'shared',tag_mode:'any',tags:['cats','dogs']});
|
||||
await page.goto('about:blank');await open(page);
|
||||
await page.getByRole('navigation',{name:'Saved views',exact:true}).getByRole('link',{name:'Shared cats or dogs',exact:true}).click();
|
||||
await expect(caption(page)).toHaveText(/^1 positive prompt · /);await expect(filtersButton(page)).toHaveText('Filters · 3');
|
||||
});
|
||||
|
||||
test('a chosen folder missing from the folder list stays listed and chosen in Refine',async({page})=>{
|
||||
await open(page);await openRefine(page);await folder(page).selectOption('shared');await apply(page).click();
|
||||
await expect(caption(page)).toHaveText(/^1 positive prompt · /);
|
||||
// The folder has no image any more as far as the list knows (or the read failed): Apply must not drop it.
|
||||
await page.route('**/prompt_manager/library/options?include=folders',route=>route.fulfill({json:{success:true,library_epoch:'x',categories:[],folders:[]}}));
|
||||
await openRefine(page);
|
||||
await expect(folder(page).locator('option')).toHaveText(['All output folders','shared']);await expect(folder(page)).toHaveValue('shared');
|
||||
await apply(page).click();await expect(caption(page)).toHaveText(/^1 positive prompt · /);await expect(filtersButton(page)).toHaveText('Filters · 1');
|
||||
});
|
||||
|
||||
// Live B1: the server's flat list sorts full paths as text ("-" and "_" before "/"); Refine shows each folder directly
|
||||
// under its own parent and keeps case variants apart.
|
||||
test('the Folder list nests each folder under its exact parent',async({page})=>{
|
||||
const folders=['CCC','CCC/mabes9sans-dataset','CCC/mabes9sans-dataset-v1','CCC/mabes9sans-dataset/mabes9sans-dataset_IMG_GEN',
|
||||
'FLUX','FLUX/2025-12-11','flux','flux/images','flux-klein','kiko','kiko-v6','kiko/char'].sort();
|
||||
await page.route('**/prompt_manager/library/options?include=folders',route=>route.fulfill({json:{success:true,library_epoch:'x',categories:[],folders}}));
|
||||
await open(page);await openRefine(page);
|
||||
const I='\u00a0\u00a0';
|
||||
await expect(folder(page).locator('option')).toHaveText(['All output folders','CCC',`${I}mabes9sans-dataset`,`${I}${I}mabes9sans-dataset_IMG_GEN`,
|
||||
`${I}mabes9sans-dataset-v1`,'FLUX',`${I}2025-12-11`,'flux',`${I}images`,'flux-klein','kiko',`${I}char`,'kiko-v6']);
|
||||
expect(await folder(page).locator('option').evaluateAll(nodes=>nodes.map(node=>node.value))).toEqual(['','CCC','CCC/mabes9sans-dataset',
|
||||
'CCC/mabes9sans-dataset/mabes9sans-dataset_IMG_GEN','CCC/mabes9sans-dataset-v1','FLUX','FLUX/2025-12-11','flux','flux/images','flux-klein',
|
||||
'kiko','kiko/char','kiko-v6']);
|
||||
});
|
||||
|
||||
test('an older server that ignores the folder shows an error with Retry, never the whole shelf',async({page})=>{
|
||||
await open(page);
|
||||
await page.route('**/prompt_manager/library?*',async route=>{
|
||||
const response=await route.fetch();const body=await response.json();delete body.folder;delete body.tag_mode;
|
||||
await route.fulfill({response,json:body});
|
||||
});
|
||||
await openRefine(page);await folder(page).selectOption('shared');await apply(page).click();
|
||||
await expect(page.locator('#workspace-main').getByText('Prompt library could not be loaded. Retry.')).toBeVisible();
|
||||
await expect(cards(page)).toHaveCount(0);
|
||||
});
|
||||
|
||||
// D150: the Refine dialog is the concept's Filters dialog: its Folder select and Tags input, plus Tag match drawn as the
|
||||
// concept's own select; the concept's "Prompt evidence" (no data yet) and its prototype note are not shown.
|
||||
function mapFiltersDialog(reference,{folders,matchOptions}) {
|
||||
const [conceptFolder,,conceptTags]=reference.fields;
|
||||
return {...reference,
|
||||
fields:[{...conceptFolder,options:folders},{...conceptTags},{...conceptFolder,label:'Tag match',options:matchOptions}],
|
||||
small:[],
|
||||
// Production's Apply reads "Apply filters" (Images and Library share it, #249).
|
||||
actions:{...reference.actions,buttons:reference.actions.buttons.map(item=>item.text==='Apply tag filter'?{...item,text:'Apply filters'}:item)}};
|
||||
}
|
||||
for(const [width,height] of [[1600,1100],[390,844]])for(const theme of ['dark','light'])test(`Refine dialog matches the concept Filters dialog ${width} ${theme} (D150)`,async({page},testInfo)=>{
|
||||
await page.setViewportSize({width,height});
|
||||
await page.goto('about:blank');await page.goto(`${concept}#library`);await page.getByRole('combobox',{name:'Theme',exact:true}).selectOption(theme);
|
||||
await page.locator('#main').getByRole('button',{name:'Filters',exact:true}).click();await expect(page.locator('#modal')).toBeVisible();
|
||||
const reference=await page.locator('#modal').evaluate(inventoryOf);
|
||||
await captureCanonicalState(page,testInfo,`concept-filters-${width}-${theme}`,'#modal');
|
||||
await open(page);await page.getByRole('combobox',{name:'Theme',exact:true}).selectOption(theme);await openRefine(page);
|
||||
await expect(folder(page).locator('option')).toHaveCount(2);
|
||||
const actual=await refine(page).evaluate(inventoryOf);
|
||||
await captureCanonicalState(page,testInfo,`production-refine-${width}-${theme}`,'dialog[open]');
|
||||
const expected=mapFiltersDialog(reference,{folders:[['','All output folders'],['shared','shared']],matchOptions:[['all','All of these tags'],['any','Any of these tags'],['untagged','Prompts without tags']]});
|
||||
const path=testInfo.outputPath('refine-inventory.json');await writeFile(path,JSON.stringify({reference,actual},null,2));
|
||||
await testInfo.attach('refine-inventory',{path,contentType:'application/json'});
|
||||
expect(reference.fields.map(item=>item.label)).toEqual(['Folder','Prompt evidence','Tags']);
|
||||
expect(diffInventory(expected,actual)).toEqual([]);
|
||||
await refine(page).getByRole('button',{name:'Close dialog',exact:true}).click();
|
||||
await openRefine(page);await folder(page).selectOption('shared');await tags(page).fill('cats, dogs');await match(page).selectOption('any');await apply(page).click();
|
||||
await expect(caption(page)).toHaveText(/^1 positive prompt · /);await page.mouse.move(0,0);
|
||||
await captureCanonicalState(page,testInfo,`production-library-folder-any-${width}-${theme}`,'#workspace-main');
|
||||
});
|
||||
@@ -62,7 +62,8 @@ test('Set category writes the selected prompts on the server, reloads the shelf
|
||||
await box(page,first.id).check();await expect(bar(page).locator('strong')).toHaveText('1 selected');
|
||||
await box(page,second.id).check();await expect(bar(page).locator('strong')).toHaveText('2 selected');
|
||||
// Two of three loaded are checked, so Select all N loaded follows the count (D151, #444 ruling).
|
||||
await expect(bar(page).getByRole('button')).toHaveText([`Select all ${items.length} loaded`,'Add tags','Set category','Clear']);
|
||||
// D152 (#443) adds Delete… before Clear.
|
||||
await expect(bar(page).getByRole('button')).toHaveText([`Select all ${items.length} loaded`,'Add tags','Set category','Delete…','Clear']);
|
||||
await setCategory(page,' Studio set ');
|
||||
await expect(toast(page)).toHaveText('Category “Studio set” set on 2 prompts.');
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
@@ -287,7 +288,7 @@ test('Ctrl/Cmd+A and Select all N loaded select every loaded prompt, windowed on
|
||||
let release;const held=new Promise(resolve=>{release=resolve;});
|
||||
await page.route('**/prompt_manager/bulk/tags',async route=>{await held;await route.continue();},{times:1});
|
||||
await addTags(page,'everyone');
|
||||
for(const name of ['Add tags','Set category',`Select all ${all} loaded`]) await expect(bar(page).getByRole('button',{name,exact:true})).toHaveAttribute('aria-disabled','true');
|
||||
for(const name of ['Add tags','Set category','Delete…','Clear',`Select all ${all} loaded`]) await expect(bar(page).getByRole('button',{name,exact:true})).toHaveAttribute('aria-disabled','true');
|
||||
release();
|
||||
await expect(toast(page)).toHaveText(`Added “everyone” to ${all-1} prompts.`);
|
||||
expect(writes).toHaveLength(1);expect(new Set(writes[0].prompt_ids).size).toBe(all-1);
|
||||
|
||||
@@ -155,7 +155,8 @@ test('Retry re-reads a failed category list with the shelf',async({page})=>{
|
||||
await page.locator('#workspace-main search.toolbar').getByRole('button',{name:'Filters',exact:true}).click();
|
||||
await refine(page).getByRole('button',{name:'Apply filters',exact:true}).click();await expect(refine(page)).toHaveCount(0);
|
||||
const search=page.getByRole('searchbox',{name:'Search workspace',exact:true});await search.fill(' ');
|
||||
await page.waitForTimeout(400);expect(reads).toEqual([]);await expect(cards(page)).toHaveCount(3);
|
||||
// Only the folder list Refine reads on opening (D150), never the shelf.
|
||||
await page.waitForTimeout(400);expect(reads).toEqual(['/prompt_manager/library/options']);await expect(cards(page)).toHaveCount(3);
|
||||
});
|
||||
|
||||
test('failed library reads retry while late search responses cannot overwrite newer results',async({page})=>{
|
||||
@@ -200,7 +201,7 @@ test('library toolbar matches the concept and tags live in Refine this view',asy
|
||||
// D44: the active Tags filter shows on the button; the category is the toolbar's own and is not counted.
|
||||
await expect(toolbar.getByRole('button')).toHaveText(['Save view','Filters · 1']);
|
||||
await toolbar.getByRole('button',{name:'Filters · 1',exact:true}).click();
|
||||
await expect(refine(page).locator('.field label')).toHaveText(['Tags','Tag match']);await expect(refine(page).getByRole('textbox',{name:'Tags',exact:true})).toHaveValue('studio');
|
||||
await expect(refine(page).locator('.field label')).toHaveText(['Folder','Tags','Tag match']);await expect(refine(page).getByRole('textbox',{name:'Tags',exact:true})).toHaveValue('studio');
|
||||
await refine(page).getByRole('button',{name:'Clear filters',exact:true}).click();await expect(refine(page)).toHaveCount(0);
|
||||
await expect(toolbar.getByRole('button',{name:'Filters',exact:true})).toBeFocused();
|
||||
await expect(page.getByRole('combobox',{name:'Filter by category',exact:true})).toHaveValue('');await expect(cards(page)).toHaveCount(3);
|
||||
|
||||
@@ -208,7 +208,7 @@ test('a view stored under older rules still lists and opens with its usable valu
|
||||
await raw({ page: 'library', name: '', state: JSON.stringify({ q: 'dog', sort: 'name-desc' }) });
|
||||
expect(await listed(request)).toEqual([
|
||||
{ id: imagesId, page: 'images', name: 'Old cats', state: { ...IMAGE_DEFAULTS, q: 'cat' } },
|
||||
expect.objectContaining({ page: 'library', name: 'Saved view', state: { q: 'dog', category: '', sort: 'newest', tags: [], tag_mode: 'all' } }),
|
||||
expect.objectContaining({ page: 'library', name: 'Saved view', state: { q: 'dog', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' } }),
|
||||
]);
|
||||
// A state that is no JSON object is never listed, and its link behaves like a missing view.
|
||||
const corruptId = await raw({ page: 'images', name: 'Corrupt', state: '{not json' });
|
||||
@@ -233,7 +233,7 @@ test('a view stored under older rules still lists and opens with its usable valu
|
||||
// D153 (#447): the workspace no longer offers "Highest rated"; the API still stores such a view as it was saved.
|
||||
test('a view saved under Highest rated opens on Recently added on Images and the Library', async ({ page, request }) => {
|
||||
const images = await create(request, { page: 'images', name: 'Rated cats', state: { ...IMAGE_DEFAULTS, q: 'cat', sort: 'rating' } });
|
||||
const library = await create(request, { page: 'library', name: 'Rated dogs', state: { q: 'dog', category: '', sort: 'rating', tags: [], tag_mode: 'all' } });
|
||||
const library = await create(request, { page: 'library', name: 'Rated dogs', state: { q: 'dog', category: '', sort: 'rating', tags: [], tag_mode: 'all', folder: '' } });
|
||||
expect((await listed(request)).map(view => view.state.sort)).toEqual(['rating', 'rating']);
|
||||
const listings = [];
|
||||
page.on('request', item => {
|
||||
@@ -257,7 +257,7 @@ test('a view saved under Highest rated opens on Recently added on Images and the
|
||||
|
||||
test('leaving a saved view returns Images and the Library to their own last state (#249 M2)', async ({ page, request }) => {
|
||||
const dogs = await create(request, { page: 'images', name: 'Dogs', state: { ...IMAGE_DEFAULTS, q: 'dogs' } });
|
||||
const cats = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all' } });
|
||||
const cats = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' } });
|
||||
await page.goto(`${workspace}#/images`);
|
||||
await search(page).fill('cat');
|
||||
await expect(imageCards(page)).toHaveCount(1);
|
||||
@@ -300,7 +300,7 @@ test('a toolbar change while sources load keeps the saved Source and Folder (#24
|
||||
});
|
||||
|
||||
test('a saved view left while its list or rename is pending leaves the next page alone', async ({ page, request }) => {
|
||||
const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all' } });
|
||||
const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' } });
|
||||
let releaseList = () => {};
|
||||
const holdList = async () => {
|
||||
const held = new Promise(resolve => { releaseList = resolve; });
|
||||
@@ -348,7 +348,7 @@ test('an unknown saved view opens Images, and a failed listing offers Retry', as
|
||||
await expect(page).toHaveURL(/#\/images$/);
|
||||
await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Images');
|
||||
|
||||
const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all' } });
|
||||
const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' } });
|
||||
let failing = true;
|
||||
await page.route(`**${API}`, route => failing && route.request().method() === 'GET'
|
||||
? route.fulfill({ status: 500, contentType: 'application/json', body: '{"success":false,"error":"down"}' })
|
||||
@@ -539,7 +539,7 @@ test('Edit renames an open view, and deletes it only after confirming inside the
|
||||
});
|
||||
|
||||
test('a deleted Library view returns to the Library, and failed edits say so', async ({ page, request }) => {
|
||||
const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all' } });
|
||||
const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' } });
|
||||
await page.goto(`${workspace}#/saved/${view.id}`);
|
||||
await expect(heading(page)).toHaveText('Cats');
|
||||
const edit = main(page).locator('.heading-actions > button').first();
|
||||
|
||||
@@ -37,6 +37,15 @@ test.afterEach(async ({ page, request, browserName }) => {
|
||||
await expect(await request.post('/__test__/tagging/reset', { data: {} })).toBeOK();
|
||||
});
|
||||
|
||||
async function openSettled(page, url) {
|
||||
// The page reads its selected file's details as soon as it lists. A source change before that read is answered
|
||||
// makes the page find the file again on its own, so the test's old-numbering locators are gone before they are
|
||||
// used (#484): every test changes sources only after the read was answered.
|
||||
const inspected = page.waitForResponse(item => new URL(item.url()).pathname.startsWith('/prompt_manager/images/inspect/') &&
|
||||
item.request().method() === 'GET');
|
||||
await page.goto(url);
|
||||
expect((await inspected).status()).toBe(200);
|
||||
}
|
||||
async function openTag(page, path = FIRST, root = 0) {
|
||||
const observed = answered(page, '/image-annotations/observe');
|
||||
await tagButton(page, path, root).click();
|
||||
@@ -53,7 +62,7 @@ async function savedTags(request, path, root) {
|
||||
}
|
||||
|
||||
test('renumbered sources under an open Tag dialog: it finds the file again, keeps the draft and Generate works', async ({ page, request }) => {
|
||||
await page.goto(images);
|
||||
await openSettled(page, images);
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
expect((await openTag(page)).status()).toBe(200);
|
||||
await manual(page).fill('kept_draft');
|
||||
@@ -91,7 +100,7 @@ test('renumbered sources under an open Tag dialog: it finds the file again, keep
|
||||
});
|
||||
|
||||
test('the inspector finds a renumbered file again; checked files from the old numbering are cleared', async ({ page, request }) => {
|
||||
await page.goto(images);
|
||||
await openSettled(page, images);
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
await card(page, FIRST).getByRole('checkbox', { name: /^Select / }).check();
|
||||
await expect(page.locator('.bulkbar')).toBeVisible();
|
||||
@@ -113,7 +122,7 @@ test('the inspector finds a renumbered file again; checked files from the old nu
|
||||
});
|
||||
|
||||
test('a scope that cannot be mapped: Generate and Save are visibly disabled, the reason and Refresh sit beside them', async ({ page }) => {
|
||||
await page.goto(images);
|
||||
await openSettled(page, images);
|
||||
await expect(card(page, FIRST)).toBeVisible();
|
||||
await expect(await page.request.post('/__test__/scope-key/rotate')).toBeOK();
|
||||
const rescoped = answered(page, '/images/rescope');
|
||||
@@ -157,7 +166,7 @@ test('a scope that cannot be mapped: Generate and Save are visibly disabled, the
|
||||
});
|
||||
|
||||
test('an unmappable inspector read keeps its visible Refresh, and Refresh recovers the panel', async ({ page }) => {
|
||||
await page.goto(images);
|
||||
await openSettled(page, images);
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
await expect(await page.request.post('/__test__/scope-key/rotate')).toBeOK();
|
||||
const rescoped = answered(page, '/images/rescope');
|
||||
@@ -173,7 +182,7 @@ test('an unmappable inspector read keeps its visible Refresh, and Refresh recove
|
||||
|
||||
test('a Load more cursor the server cannot verify offers Refresh instead of a dead Retry', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/tagging/reset', { data: { extra_images: 60 } })).toBeOK();
|
||||
await page.goto(images);
|
||||
await openSettled(page, images);
|
||||
await expect(cards(page).first()).toBeVisible();
|
||||
await expect(page.locator('#gallery-continuation')).toContainText('more load as you scroll');
|
||||
await expect(await request.post('/__test__/scope-key/rotate')).toBeOK();
|
||||
@@ -191,7 +200,7 @@ test('a Load more cursor the server cannot verify offers Refresh instead of a de
|
||||
});
|
||||
|
||||
test('a viewer whose media source was renumbered closes onto the file, kept selected', async ({ page, request }) => {
|
||||
await page.goto(images);
|
||||
await openSettled(page, images);
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
await expect(await request.post('/__test__/sources/reverse')).toBeOK();
|
||||
const rescoped = answered(page, '/images/rescope');
|
||||
@@ -217,7 +226,7 @@ test('a collection page finds a renumbered member again and stays on its members
|
||||
await expect(added).toBeOK();
|
||||
const collections = (await (await request.get('/prompt_manager/collections')).json()).collections;
|
||||
const { id } = collections.find(item => item.name === name);
|
||||
await page.goto(`/prompt_manager/workspace#/collection/${id}`);
|
||||
await openSettled(page, `/prompt_manager/workspace#/collection/${id}`);
|
||||
await expect(cards(page)).toHaveCount(1);
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
await expect(await request.post('/__test__/sources/reverse')).toBeOK();
|
||||
|
||||
@@ -11,8 +11,8 @@ const STATUS_URL = '/prompt_manager/backup/status';
|
||||
const PREVIEW_URL = '/prompt_manager/restore/preview';
|
||||
const APPLY_URL = '/prompt_manager/restore/apply';
|
||||
const WHEN = { dateStyle: 'medium', timeStyle: 'short' };
|
||||
const SAFETY_DESCRIPTION = 'Keep a verified copy of the database on disk before maintenance, cleanups and LoRA imports change ' +
|
||||
'records. A restore always keeps one.';
|
||||
const SAFETY_DESCRIPTION = 'Keep a verified copy of the database on disk before maintenance, cleanups, LoRA imports and ' +
|
||||
'prompt deletes change records. A restore always keeps one.';
|
||||
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
|
||||
@@ -274,11 +274,18 @@ test('level filter is minimum-level over all six choices', async ({ page, reques
|
||||
for (const line of (await lines(page).textContent()).split('\n')) expect(line).toMatch(/^\d\d:\d\d:\d\d [A-Z]+ +\S/);
|
||||
expect(urls.at(-1)).toBe('?limit=100');
|
||||
for (const minimum of LEVELS) {
|
||||
// Wait for this level's answer: the previous level's lines also hold the CRITICAL line, so seeing it proved
|
||||
// nothing, and under load the filtered answer painted after the check (#501 merge queue, firefox).
|
||||
const answered = page.waitForResponse(item => new URL(item.url()).pathname === LOGS_URL
|
||||
&& new URL(item.url()).search === `?limit=100&level=${minimum}`);
|
||||
await level.selectOption(minimum);
|
||||
await answered;
|
||||
await expect.poll(() => urls.at(-1)).toBe(`?limit=100&level=${minimum}`);
|
||||
await expect(lines(page)).toContainText('CRITICAL The database could not be opened');
|
||||
const shown = await shownLevels(page);
|
||||
expect(shown.every(name => LEVELS.indexOf(name) >= LEVELS.indexOf(minimum)), `${minimum}: ${shown}`).toBe(true);
|
||||
await expect.poll(async () => {
|
||||
const shown = await shownLevels(page);
|
||||
return shown.every(name => LEVELS.indexOf(name) >= LEVELS.indexOf(minimum)) || `${minimum}: ${shown}`;
|
||||
}).toBe(true);
|
||||
if (minimum === 'DEBUG') await expect(lines(page)).toContainText('DEBUG Thumbnail cache lookup');
|
||||
if (minimum === 'WARNING') {
|
||||
await expect(lines(page)).not.toContainText('Database backup verified');
|
||||
|
||||
@@ -73,7 +73,7 @@ const jobStart = page => page.waitForRequest(item => item.method() === 'POST' &&
|
||||
|
||||
test('selection bar shows Add tags then Auto-tag selected before Set category', async ({ page }) => {
|
||||
await select(page, FIRST);
|
||||
await expect(bar(page).locator(':scope > *')).toHaveText(['1 selected', 'Add tags', 'Auto-tag selected', 'Set category', 'Add to collection', 'Export metadata', 'Clear']);
|
||||
await expect(bar(page).locator(':scope > *')).toHaveText(['1 selected', 'Add tags', 'Auto-tag selected', 'Set category', 'Add to collection', 'Export metadata', 'Delete…', 'Clear']);
|
||||
});
|
||||
|
||||
test('Add tags runs a job and the cards show the tags after it finishes', async ({ page, request }) => {
|
||||
|
||||
@@ -314,6 +314,16 @@ test('fresh registered PM variants preserve exact raw whitespace and known empty
|
||||
const runId = (await readExecutionRuns(execution.databasePath, id)).rows[0].run_id;
|
||||
const response = await execution.request.get(`/prompt_manager/library/${value.positive_prompt_id}/history`);expect(response.ok()).toBe(true);
|
||||
expect((await response.json()).items.map(item => item.run_id)).toContain(runId);
|
||||
if (kind !== 'PromptManager') continue;
|
||||
// #460: an upper-case re-queue counts toward the same prompt and joins its history
|
||||
const upper = await submit(execution.request, recipeGraph({ kind, text: `${kind.toUpperCase()} E\u0301` }));
|
||||
await history(execution.request, upper);
|
||||
const variant = (await capture(execution, upper)).contexts[0];
|
||||
expect(variant.positive.raw).toBe(`${kind.toUpperCase()} E\u0301`);
|
||||
expect(variant.positive_prompt_id, 'the case variant names the same prompt').toBe(value.positive_prompt_id);
|
||||
const variantRun = (await readExecutionRuns(execution.databasePath, upper)).rows[0].run_id;
|
||||
const scoped = await execution.request.get(`/prompt_manager/library/${value.positive_prompt_id}/history`);
|
||||
expect((await scoped.json()).items.map(item => item.run_id)).toContain(variantRun);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""A fake system trash for tests (#480): renames into a folder outside every root.
|
||||
|
||||
``install(folder)`` replaces ``utils.trash.BACKEND`` and returns the fake;
|
||||
``uninstall`` puts the real one back. ``refuse[path] = code`` refuses one
|
||||
file with that code. The fake never copies or unlinks; it records each call.
|
||||
The disposable browser server installs it when ``PM_TEST_FAKE_TRASH=1``, so
|
||||
no test touches a real trash.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from utils import trash
|
||||
|
||||
|
||||
class FakeTrash:
|
||||
def __init__(self, folder):
|
||||
self.folder = str(folder)
|
||||
os.makedirs(self.folder, exist_ok=True)
|
||||
self.refuse = {}
|
||||
self.moved = [] # (source, destination)
|
||||
self.calls = []
|
||||
|
||||
def available(self, path, roots):
|
||||
self.calls.append(("available", str(path)))
|
||||
return self.refuse.get(str(path))
|
||||
|
||||
def move(self, path, roots):
|
||||
self.calls.append(("move", str(path)))
|
||||
code = self.refuse.get(str(path))
|
||||
if code:
|
||||
raise trash.TrashRefused(code)
|
||||
destination = os.path.join(
|
||||
self.folder, f"{len(self.moved)}-{os.path.basename(path)}"
|
||||
)
|
||||
if os.path.lexists(destination):
|
||||
raise trash.TrashRefused(trash.FAILED)
|
||||
os.rename(path, destination)
|
||||
self.moved.append((str(path), destination))
|
||||
return "fake trash"
|
||||
|
||||
|
||||
_ORIGINAL = trash.BACKEND
|
||||
|
||||
|
||||
def install(folder):
|
||||
fake = FakeTrash(folder)
|
||||
trash.BACKEND = fake
|
||||
return fake
|
||||
|
||||
|
||||
def uninstall():
|
||||
trash.BACKEND = _ORIGINAL
|
||||
@@ -118,6 +118,7 @@
|
||||
"POST /prompt_manager/image-annotations/observe",
|
||||
"POST /prompt_manager/image-annotations/suggest",
|
||||
"POST /prompt_manager/images/clear-thumbnails",
|
||||
"POST /prompt_manager/images/delete/preview",
|
||||
"POST /prompt_manager/images/generate-thumbnails",
|
||||
"POST /prompt_manager/images/generate-thumbnails/progress",
|
||||
"POST /prompt_manager/images/link",
|
||||
@@ -131,6 +132,9 @@
|
||||
"POST /prompt_manager/jobs/thumbnails",
|
||||
"POST /prompt_manager/jobs/{job_id}/cancel",
|
||||
"POST /prompt_manager/library",
|
||||
"POST /prompt_manager/library/delete",
|
||||
"POST /prompt_manager/library/delete/preview",
|
||||
"POST /prompt_manager/library/restore",
|
||||
"POST /prompt_manager/library/{positive_id}/variants",
|
||||
"POST /prompt_manager/logs/config",
|
||||
"POST /prompt_manager/logs/truncate",
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
// #480 (D159): Images › Delete…: the review's rows, sentences, labels and outcome toasts.
|
||||
const loaded = await import('../../web/js/workspace/images-delete.mjs').catch(() => null);
|
||||
const module = () => { assert.ok(loaded, 'images-delete.mjs is not implemented'); return loaded; };
|
||||
const pageScope = () => {
|
||||
const controller = new AbortController();
|
||||
return { signal: controller.signal, cleanup: () => {}, abort: () => controller.abort() };
|
||||
};
|
||||
const plan = (over = {}) => ({
|
||||
token: 't', library: { count: 3 }, trash: { count: 2 },
|
||||
refused: { count: 1, counts: { 'not-listed': 1 }, rows: [{ path: 'sub/gone.png', code: 'not-listed', detail: 'No longer listed in Images.' }] },
|
||||
trash_refused: { count: 1, counts: { 'no-trash': 1 }, rows: [{ path: 'b.png', code: 'no-trash', detail: 'This drive has no trash.' }] },
|
||||
records: { prompt_links: 2, pair_links: 0, image_tags: 1, auto_tagged: 0, categories: 1, collections: 2, favorites: 0, review_items: 0 },
|
||||
orphaned_prompts: { count: 1, titles: ['Red fox'] },
|
||||
backup: { policy: 'on' }, disk: null, ...over,
|
||||
});
|
||||
|
||||
test('the words of the review and its routes', () => {
|
||||
const { PREVIEW_URL, MAX_DELETE, TOO_MANY, QUESTION, MODES } = module();
|
||||
assert.equal(PREVIEW_URL, '/prompt_manager/images/delete/preview');
|
||||
assert.equal(MAX_DELETE, 500);
|
||||
assert.equal(TOO_MANY, 'Select at most 500 images.');
|
||||
assert.equal(QUESTION, 'What should happen to the files?');
|
||||
assert.deepEqual(MODES.map(mode => mode.label), ['Remove from the library only', 'Also move the files to the trash']);
|
||||
});
|
||||
|
||||
test('library rows: scope, refusals, links, details, backup and media', () => {
|
||||
const { deleteRows } = module();
|
||||
const rows = Object.fromEntries(deleteRows(plan(), 'library'));
|
||||
assert.equal(rows.Scope, '3 images');
|
||||
assert.equal(rows['Not changed'], '1: gone.png. No longer listed in Images.');
|
||||
assert.equal(rows['Prompt links'], '2 links. 1 prompt loses its last image: Red fox. The prompt stays on the Library page.');
|
||||
assert.equal(rows['Image details'], '1 with image tags, 1 category, 2 collection memberships');
|
||||
assert.equal(rows.Backup, 'Create and verify before changes (the database, not the files)');
|
||||
assert.equal(rows['Media on disk'], 'Files stay on disk, hidden from Images.');
|
||||
});
|
||||
|
||||
test('trash rows add the trash refusals and the trash media line', () => {
|
||||
const { deleteRows } = module();
|
||||
const rows = Object.fromEntries(deleteRows(plan({ backup: { policy: 'off' } }), 'trash'));
|
||||
assert.equal(rows.Scope, '2 images');
|
||||
assert.match(rows['Not changed'], /^2: gone\.png, b\.png\. No longer listed in Images\. This drive has no trash\.$/);
|
||||
assert.equal(rows.Backup, 'Off: no safety backup (System › Backup & export › Recovery policy)');
|
||||
assert.match(rows['Media on disk'], /Nothing is deleted permanently/);
|
||||
});
|
||||
|
||||
test('rows leave out what is zero', () => {
|
||||
const { deleteRows } = module();
|
||||
const bare = plan({ refused: { count: 0, rows: [] }, trash_refused: { count: 0, rows: [] },
|
||||
records: { prompt_links: 0 }, orphaned_prompts: { count: 0, titles: [] } });
|
||||
assert.deepEqual(deleteRows(bare, 'library').map(([term]) => term), ['Scope', 'Backup', 'Media on disk']);
|
||||
const many = plan({ orphaned_prompts: { count: 5, titles: ['A', 'B', 'C'] } });
|
||||
assert.match(Object.fromEntries(deleteRows(many, 'library'))['Prompt links'],
|
||||
/5 prompts lose their last image: A, B, C and 2 more\. The prompts stay on the Library page\./);
|
||||
assert.equal(module().refusedValue({ count: 5, rows: [{ path: 'a.png' }] }), '5: a.png and 4 more');
|
||||
assert.equal(module().refusedValue({ count: 0 }), null);
|
||||
});
|
||||
|
||||
test('sentences follow the mode and the backup policy', () => {
|
||||
const { deleteSentences, SENTENCES, USE_LIBRARY_ONLY, NO_TRASH_HERE } = module();
|
||||
assert.deepEqual(deleteSentences(plan(), 'library'), [SENTENCES.libraryBackup]);
|
||||
assert.deepEqual(deleteSentences(plan({ backup: { policy: 'off' } }), 'library'), [SENTENCES.libraryNoBackup]);
|
||||
assert.deepEqual(deleteSentences(plan(), 'trash'), [SENTENCES.trash, USE_LIBRARY_ONLY]);
|
||||
const nowhere = plan({ trash: { count: 0 }, trash_refused: { count: 3, rows: [] } });
|
||||
assert.deepEqual(deleteSentences(nowhere, 'trash'), [SENTENCES.trash, NO_TRASH_HERE, USE_LIBRARY_ONLY]);
|
||||
// #500: no "yet" (Windows is unavailable by design); each file's reason says why.
|
||||
assert.equal(NO_TRASH_HERE, 'Moving to the trash is not available for these files; each one lists why.');
|
||||
assert.deepEqual(deleteSentences(plan({ trash_refused: { count: 0 } }), 'trash'), [SENTENCES.trash]);
|
||||
});
|
||||
|
||||
test('primary labels and the close-only reasons', () => {
|
||||
const { primaryLabel, closeOnlyReason, NONE_DELETABLE } = module();
|
||||
assert.equal(primaryLabel('library', 1), 'Remove 1 image from the library');
|
||||
assert.equal(primaryLabel('trash', 4), 'Move 4 images to the trash');
|
||||
assert.equal(closeOnlyReason(plan()), null);
|
||||
assert.equal(closeOnlyReason(plan({ library: { count: 0 }, trash: { count: 0 } })), NONE_DELETABLE);
|
||||
assert.equal(closeOnlyReason(plan({ disk: { sufficient: false, required_bytes: 2048, free_bytes: 1024 } })),
|
||||
'There is not enough free space for the safety backup (needs 2,048 bytes, has 1,024).');
|
||||
});
|
||||
|
||||
test('outcome toasts per mode and failure', () => {
|
||||
const { outcomeToast } = module();
|
||||
assert.equal(outcomeToast({ scope: { mode: 'library' }, counters: { removed: 3, not_changed: 0 }, total: 3 }),
|
||||
'Removed 3 images from the library.');
|
||||
assert.equal(outcomeToast({ scope: { mode: 'trash' }, counters: { trashed: 2, not_changed: 1 }, total: 3 }),
|
||||
'Moved 2 images to the trash. 1 was not changed (see Activity).');
|
||||
assert.equal(outcomeToast({ scope: { mode: 'trash' }, counters: { trashed: 0, not_changed: 2 }, total: 2 }),
|
||||
'Moved 0 images to the trash. 2 were not changed (see Activity).');
|
||||
assert.equal(outcomeToast({ scope: { mode: 'library', failure: 'records-failed' }, counters: { removed: 1 }, total: 4 }),
|
||||
'Stopped after 1 of 4 images; the rest were not changed. See Activity.');
|
||||
assert.equal(outcomeToast({ scope: { failure: 'backup-failed' } }), 'The safety backup failed; nothing was changed.');
|
||||
assert.equal(outcomeToast({ scope: { failure: 'sources-changed' } }), 'The sources changed; nothing was changed.');
|
||||
});
|
||||
|
||||
test('review failures name the cause', () => {
|
||||
const { reviewFailure, SOURCE_CHANGED, STORAGE_CHANGED, TOO_MANY, REVIEW_FAILED } = module();
|
||||
assert.equal(reviewFailure({ code: 'source-changed' }), SOURCE_CHANGED);
|
||||
assert.equal(reviewFailure({ code: 'storage-changed' }), STORAGE_CHANGED);
|
||||
assert.equal(reviewFailure({ code: 'too-many-images' }), TOO_MANY);
|
||||
assert.equal(reviewFailure({ code: 'busy', message: 'Scan running' }), 'Scan running');
|
||||
assert.equal(reviewFailure(new Error('boom')), REVIEW_FAILED);
|
||||
});
|
||||
|
||||
test('over 500 refuses with the toast before any request', async () => {
|
||||
const { openDeleteImages, TOO_MANY } = module();
|
||||
const toasts = [];
|
||||
const api = { post: () => assert.fail('no request') };
|
||||
await openDeleteImages({ api, scope: pageScope(), images: [], filter: {}, count: 501, notify: text => toasts.push(text) });
|
||||
assert.deepEqual(toasts, [TOO_MANY]);
|
||||
await openDeleteImages({ api, scope: pageScope(), images: [], notify: text => toasts.push(text) });
|
||||
assert.deepEqual(toasts, [TOO_MANY]);
|
||||
});
|
||||
|
||||
test('a failed review toasts and releases its trigger', async () => {
|
||||
const { openDeleteImages, SOURCE_CHANGED } = module();
|
||||
const toasts = [], sent = [];
|
||||
const attributes = new Map();
|
||||
const trigger = { textContent: 'Delete…', getAttribute: name => attributes.get(name) ?? null,
|
||||
setAttribute: (name, value) => attributes.set(name, value), removeAttribute: name => attributes.delete(name) };
|
||||
const api = { post: async (url, body) => { sent.push([url, body]); throw Object.assign(new Error('x'), { code: 'source-changed' }); } };
|
||||
await openDeleteImages({ api, scope: pageScope(), trigger, notify: text => toasts.push(text),
|
||||
images: [{ root_index: 0, root_scope: 'a'.repeat(64), path: 'a.png', filename: 'a.png' }] });
|
||||
assert.deepEqual(sent, [['/prompt_manager/images/delete/preview', { items: [{ root_index: 0, root_scope: 'a'.repeat(64), path: 'a.png' }] }]]);
|
||||
assert.deepEqual(toasts, [SOURCE_CHANGED]);
|
||||
assert.equal(trigger.textContent, 'Delete…');
|
||||
assert.equal(trigger.getAttribute('aria-disabled'), null);
|
||||
});
|
||||
|
||||
test('apply sends the token and the mode, and toasts that the job started', async () => {
|
||||
const { applyDelete } = module();
|
||||
const sent = [], toasts = [];
|
||||
const api = { post: async (url, body) => { sent.push([url, body]); return { job: { id: 'j', title: 'Delete images (move to the trash)' } }; } };
|
||||
const store = { track: job => job, load: () => {} };
|
||||
const job = await applyDelete({ api, store, plan: plan(), mode: 'trash', notify: text => toasts.push(text) });
|
||||
assert.equal(job.id, 'j');
|
||||
assert.deepEqual(sent, [['/prompt_manager/maintenance/apply', { token: 't', mode: 'trash' }]]);
|
||||
assert.deepEqual(toasts, ['Delete images (move to the trash) started.']);
|
||||
const refused = await applyDelete({ api: { post: async () => { throw Object.assign(new Error('Busy'), { code: 'busy', running: null }); } },
|
||||
store, plan: plan(), mode: 'library', notify: text => toasts.push(text) });
|
||||
assert.equal(refused, null);
|
||||
});
|
||||
|
||||
test('apply refuses to send a delete without an explicit mode', async () => {
|
||||
const { applyDelete } = module();
|
||||
const api = { post: () => assert.fail('no request without a mode') };
|
||||
for (const mode of [undefined, null, '', 'TRASH', 'delete']) {
|
||||
await assert.rejects(applyDelete({ api, store: null, plan: plan(), mode, notify: () => {} }), TypeError);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,150 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
// #443 D152: the Prompt library's Delete… review (delete unpaired, archive paired, keep protected) and Restore.
|
||||
const source = import('../../web/js/workspace/library-delete.mjs');
|
||||
|
||||
const preview = (patch = {}) => ({success: true, library_epoch: 'e', delete: [], archive: [], protected: [],
|
||||
already_archived: 0, not_found: 0, image_links: 0, backup: 'on', disk: null, ...patch});
|
||||
|
||||
test('the routes are the workspace ones, never the legacy bulk delete', async () => {
|
||||
const {PREVIEW_URL, DELETE_URL, RESTORE_URL} = await source;
|
||||
assert.equal(PREVIEW_URL, '/prompt_manager/library/delete/preview');
|
||||
assert.equal(DELETE_URL, '/prompt_manager/library/delete');
|
||||
assert.equal(RESTORE_URL, '/prompt_manager/library/restore');
|
||||
});
|
||||
|
||||
test('the title, primary and body follow what the review will do', async () => {
|
||||
const {reviewOf, reviewTitle, primaryLabel, deleteBody} = await source;
|
||||
const both = reviewOf(preview({delete: [1, 2], archive: [3]}));
|
||||
assert.equal(reviewTitle(both), 'Delete or archive selected prompts');
|
||||
assert.equal(primaryLabel(both), 'Delete 2 and archive 1 prompt');
|
||||
assert.deepEqual(deleteBody(both), {archive_ids: [3], delete_ids: [1, 2], expected_epoch: 'e', backup: 'on'});
|
||||
const onlyDelete = reviewOf(preview({delete: [1]}));
|
||||
assert.equal(reviewTitle(onlyDelete), 'Delete selected prompts');
|
||||
assert.equal(primaryLabel(onlyDelete), 'Delete 1 prompt');
|
||||
const onlyArchive = reviewOf(preview({archive: [4, 5]}));
|
||||
assert.equal(reviewTitle(onlyArchive), 'Archive selected prompts');
|
||||
assert.equal(primaryLabel(onlyArchive), 'Archive 2 prompts');
|
||||
});
|
||||
|
||||
test('the grid shows each row only when it applies, with protected titles in selection order', async () => {
|
||||
const {reviewOf, reviewGrid} = await source;
|
||||
const names = {7: 'Seven', 8: 'Eight', 9: 'Nine', 10: 'Ten'};
|
||||
const review = reviewOf(preview({delete: [1], archive: [2], protected: [7, 8, 9, 10], already_archived: 1,
|
||||
not_found: 2, image_links: 3}));
|
||||
assert.deepEqual(reviewGrid(review, id => names[id]), [
|
||||
['Delete permanently', '1 prompt not in a saved pair'],
|
||||
['Archive', '1 prompt in a saved pair: hidden, restorable'],
|
||||
['Protected (kept)', '4: Seven, Eight, Nine and 1 more'],
|
||||
['Already archived', '1'],
|
||||
['No longer exist', '2'],
|
||||
['Image links removed', '3 images will no longer show these prompts (files stay on disk)'],
|
||||
['Backup', 'Create and verify before deleting'],
|
||||
]);
|
||||
const archiveOnly = reviewOf(preview({archive: [2]}));
|
||||
assert.deepEqual(reviewGrid(archiveOnly).map(([term]) => term), ['Delete permanently', 'Archive']);
|
||||
const off = reviewOf(preview({delete: [1], backup: 'off'}));
|
||||
assert.equal(reviewGrid(off).at(-1)[1], 'Off: no safety backup (System › Backup & export › Recovery policy)');
|
||||
});
|
||||
|
||||
test('the Archive row names the saved pair, never negative variants, for one or many prompts', async () => {
|
||||
const {reviewOf, reviewGrid} = await source;
|
||||
// A saved-pair prompt with 0 negative variants (live prompt 7901) is archive-only for its pair, not for variants.
|
||||
const one = reviewGrid(reviewOf(preview({archive: [2]}))).find(([term]) => term === 'Archive');
|
||||
assert.deepEqual(one, ['Archive', '1 prompt in a saved pair: hidden, restorable']);
|
||||
const many = reviewGrid(reviewOf(preview({archive: [2, 3, 4]}))).find(([term]) => term === 'Archive');
|
||||
assert.deepEqual(many, ['Archive', '3 prompts in a saved pair: hidden, restorable']);
|
||||
for (const [, value] of [one, many]) assert.doesNotMatch(value, /variant/);
|
||||
});
|
||||
|
||||
test('the sentences explain archive, irreversibility per backup policy, re-creation and protection', async () => {
|
||||
const {reviewOf, reviewSentences} = await source;
|
||||
const [archive, remove, guarded] = reviewSentences(reviewOf(preview({delete: [1], archive: [2], protected: [3]})));
|
||||
// Live test (#470): the archive reason is the saved pair, which a prompt with 0 negative variants has too.
|
||||
assert.match(archive, /^Prompts in a saved pair, as its positive or its negative, are archived instead of deleted, so the pair and its runs stay intact\./);
|
||||
assert.doesNotMatch(archive, /variant/);
|
||||
assert.match(archive, /Archived prompts cannot be deleted\. To bring them back, press Show archived prompts/);
|
||||
assert.match(remove, /the only way back is to restore the safety backup taken first \(System › Backup & export\), which returns the whole library to that moment\. It is kept among the last 10 safety backups\./);
|
||||
assert.match(remove, /A later scan, a new image or a node run with the same text adds it again as a new prompt\.$/);
|
||||
assert.equal(guarded, 'Protected prompts are tagged __protected__; remove the tag to delete them.');
|
||||
const [off] = reviewSentences(reviewOf(preview({delete: [1], backup: 'off'})));
|
||||
assert.match(off, /Deleted prompts cannot be brought back\. Image files/);
|
||||
assert.doesNotMatch(off, /safety backup/);
|
||||
});
|
||||
|
||||
test('a review that cannot start says why: nothing to change, or no room for the backup', async () => {
|
||||
const {reviewOf, blockedReason, COPY} = await source;
|
||||
assert.equal(blockedReason(reviewOf(preview({protected: [1]}))), COPY.none);
|
||||
assert.equal(blockedReason(reviewOf(preview({delete: [1], disk: {sufficient: false, message: 'Not enough. Nothing was changed.'}}))),
|
||||
'Not enough. Nothing was changed.');
|
||||
assert.match(blockedReason(reviewOf(preview({delete: [1], disk: {sufficient: false}}))), /not enough free space/);
|
||||
assert.equal(blockedReason(reviewOf(preview({delete: [1], disk: {sufficient: true}}))), null);
|
||||
assert.equal(blockedReason(reviewOf(preview({archive: [1]}))), null);
|
||||
});
|
||||
|
||||
test('a delete outcome counts each part, singular and plural, and names the backup', async () => {
|
||||
const {reviewOf, deleteOutcome} = await source;
|
||||
const review = reviewOf(preview({delete: [1, 2], archive: [3], protected: [9]}));
|
||||
assert.deepEqual(deleteOutcome({success: true, deleted: 2, archived: 3, already_archived: 0, not_found: 0, changed: 0,
|
||||
backup: 'prompts.db.backup_x'}, null, review), {committed: true, reload: true,
|
||||
toast: 'Deleted 2 prompts. Archived 3 prompts. 1 protected prompt was kept. Safety backup saved: prompts.db.backup_x (System › Backup & export).'});
|
||||
assert.equal(deleteOutcome({success: true, deleted: 1, archived: 0, already_archived: 1, not_found: 1, changed: 1,
|
||||
backup: null}, null, reviewOf(preview())).toast,
|
||||
'Deleted 1 prompt. 1 was already archived. 1 no longer exists. 1 changed since the review and was kept.');
|
||||
assert.equal(deleteOutcome({success: true, deleted: 0, archived: 0, already_archived: 2, not_found: 3, changed: 2,
|
||||
backup: null}, null, reviewOf(preview({protected: [1, 2]}))).toast,
|
||||
'Nothing was deleted or archived. 2 were already archived. 3 no longer exist. 2 changed since the review and were kept. 2 protected prompts were kept. The Library was reloaded.');
|
||||
assert.equal(deleteOutcome({success: true, deleted: 0, archived: 0}, null, reviewOf(preview())).committed, false);
|
||||
});
|
||||
|
||||
test('a failed delete says nothing changed, or that it may have, and keeps or reloads the selection', async () => {
|
||||
const {deleteOutcome, COPY} = await source;
|
||||
assert.deepEqual(deleteOutcome(null, {status: 409, code: 'storage-changed'}), {toast: COPY.storage, committed: false, reload: true});
|
||||
assert.equal(COPY.storage, 'The library was restored or moved since this page loaded. Nothing was deleted or archived.');
|
||||
assert.deepEqual(deleteOutcome(null, {status: 409, code: 'busy', running: {title: 'Scan'}}),
|
||||
{toast: 'Scan is running. Nothing was changed; try again when it finishes.', committed: false, reload: false});
|
||||
assert.deepEqual(deleteOutcome(null, {status: 500, code: 'backup-failed', message: 'The safety backup could not be taken and verified; nothing was changed.'}),
|
||||
{toast: 'The safety backup failed. The safety backup could not be taken and verified; nothing was changed.', committed: false, reload: false});
|
||||
assert.deepEqual(deleteOutcome(null, {status: 507, code: 'insufficient-space', message: 'Not enough free disk space. Nothing was changed.'}),
|
||||
{toast: 'Not enough free disk space. Nothing was changed.', committed: false, reload: false});
|
||||
for (const failure of [new TypeError('Failed to fetch'), {status: 502}, {status: 503}, {status: 504}]) {
|
||||
assert.deepEqual(deleteOutcome(null, failure), {toast: COPY.unconfirmed, committed: true, reload: true});
|
||||
}
|
||||
assert.deepEqual(deleteOutcome(null, {status: 400}), {toast: COPY.failed, committed: false, reload: false});
|
||||
});
|
||||
|
||||
test('a failed preview reloads only after a restore; Restore outcomes have their own words', async () => {
|
||||
const {previewOutcome, restoreOutcome, COPY} = await source;
|
||||
assert.deepEqual(previewOutcome({status: 409, code: 'storage-changed'}), {toast: COPY.storage, committed: false, reload: true});
|
||||
assert.deepEqual(previewOutcome({status: 500}), {toast: COPY.reviewFailed, committed: false, reload: false});
|
||||
assert.equal(previewOutcome({status: 409, code: 'busy', running: {title: 'Optimize'}}).toast,
|
||||
'Optimize is running. Nothing was changed; try again when it finishes.');
|
||||
assert.deepEqual(restoreOutcome({success: true, restored: 2, unchanged: 1, not_found: 1}, null),
|
||||
{toast: 'Restored 2 prompts to the Prompt library. 1 was not archived. 1 no longer exists.', committed: true, reload: true});
|
||||
assert.deepEqual(restoreOutcome({success: true, restored: 0, unchanged: 2, not_found: 0}, null),
|
||||
{toast: 'No prompt was restored. 2 were not archived.', committed: false, reload: true});
|
||||
assert.deepEqual(restoreOutcome(null, {status: 409, code: 'storage-changed'}), {toast: COPY.restoreStorage, committed: false, reload: true});
|
||||
assert.deepEqual(restoreOutcome(null, {status: 503}), {toast: COPY.restoreUnconfirmed, committed: true, reload: true});
|
||||
assert.deepEqual(restoreOutcome(null, {status: 400}), {toast: COPY.restoreFailed, committed: false, reload: false});
|
||||
});
|
||||
|
||||
test('reviewOf keeps only well-formed ids and counts from an answer', async () => {
|
||||
const {reviewOf} = await source;
|
||||
const review = reviewOf({library_epoch: 3, delete: [1, 'x', -2, 2], archive: null, protected: [1.5], already_archived: -1,
|
||||
not_found: 'x', image_links: 2, backup: 'maybe'});
|
||||
assert.deepEqual(review, {epoch: null, delete: [1, 2], archive: [], protected: [], alreadyArchived: 0, notFound: 0,
|
||||
imageLinks: 2, backup: 'on', disk: null});
|
||||
});
|
||||
|
||||
test('the archived notice says why (the saved pair) and names negative variants only when there are any', async () => {
|
||||
const {archivedNoticeText} = await source;
|
||||
assert.equal(archivedNoticeText(0), 'This prompt is archived because it is in a saved pair. It is hidden from the Prompt ' +
|
||||
'library; nothing was removed.');
|
||||
assert.equal(archivedNoticeText(1), 'This prompt is archived because it is in a saved pair. It and its 1 negative variant ' +
|
||||
'are hidden from the Prompt library; nothing was removed.');
|
||||
assert.equal(archivedNoticeText(3), 'This prompt is archived because it is in a saved pair. It and its 3 negative variants ' +
|
||||
'are hidden from the Prompt library; nothing was removed.');
|
||||
// An unknown count (an older server) never claims variants.
|
||||
assert.equal(archivedNoticeText(undefined), archivedNoticeText(0));
|
||||
});
|
||||
@@ -104,6 +104,20 @@ test('archived variant reads use their own scope and cursor without changing the
|
||||
const active=[];const plain=createLibraryDetail({api:{get:async url=>{active.push(url);return {positive:item(1),variants:[]};}},onChange:()=>{}});await plain.load(1);assert.equal(new URL(active[0],'https://x').searchParams.has('archive'),false);
|
||||
assert.throws(()=>createLibraryDetail({api:{},onChange:()=>{},archive:'parents'}));
|
||||
});
|
||||
test('tag mode any and a folder are sent, echoed and validated (#444)',async()=>{
|
||||
const {createLibraryStream,LIBRARY_TAG_MODES}=await source;assert.deepEqual(LIBRARY_TAG_MODES,['all','any','untagged']);
|
||||
const calls=[];const pending=[{...page([item(1)]),tag_mode:'any',folder:'2026/Aug'},{...page([item(2)]),tag_mode:'all',folder:''},page([item(3)]),{...page([item(4)]),tag_mode:'all'}];
|
||||
const s=createLibraryStream({api:{get:async(url)=>{calls.push(new URL(url,'https://x').searchParams);return pending.shift();}}});
|
||||
assert.equal(await s.reset({tags:['a','b'],tagMode:'any',folder:'2026/Aug'}),1);
|
||||
assert.equal(calls[0].get('tag_mode'),'any');assert.equal(calls[0].get('folder'),'2026/Aug');assert.deepEqual(calls[0].getAll('tag'),['a','b']);
|
||||
// An older server that ignored them answers without the echo: an error, never the unfiltered shelf.
|
||||
assert.equal(await s.reset({tags:['a','b'],tagMode:'any',folder:'2026/Aug'}),0);assert.ok(s.state.error instanceof TypeError);
|
||||
// The defaults send neither, so they need no echo.
|
||||
assert.equal(await s.reset({tagMode:'all',folder:''}),1);assert.equal(calls[2].has('tag_mode'),false);assert.equal(calls[2].has('folder'),false);
|
||||
assert.equal(await s.reset({folder:'x'}),0);assert.ok(s.state.error instanceof TypeError,'a folder without its echo is refused');
|
||||
await assert.rejects(s.reset({tagMode:'or'}),TypeError);await assert.rejects(s.reset({folder:3}),TypeError);
|
||||
assert.equal(calls.length,4);s.dispose();
|
||||
});
|
||||
test('tag mode untagged is sent, echoed and never combined with tags (#444)',async()=>{
|
||||
const {createLibraryStream}=await source;const calls=[];const pending=[{...page([item(1)]),tag_mode:'untagged'},page([item(2)]),page([item(3)])];
|
||||
const s=createLibraryStream({api:{get:async(url)=>{calls.push(url);return pending.shift();}}});
|
||||
@@ -112,7 +126,7 @@ test('tag mode untagged is sent, echoed and never combined with tags (#444)',asy
|
||||
assert.equal(await s.reset({tagMode:'untagged'}),0);assert.ok(s.state.error instanceof TypeError);assert.deepEqual(s.state.items,[]);
|
||||
// The default sends no tag_mode, so it needs no echo.
|
||||
assert.equal(await s.reset({tagMode:'all'}),1);assert.equal(new URL(calls[2],'https://x').searchParams.has('tag_mode'),false);
|
||||
await assert.rejects(s.reset({tagMode:'untagged',tags:['a']}),TypeError);await assert.rejects(s.reset({tagMode:'none'}),TypeError);
|
||||
await assert.rejects(s.reset({tagMode:'untagged',tags:['a']}),TypeError);await assert.rejects(s.reset({tagMode:'or'}),TypeError);
|
||||
assert.equal(calls.length,3);s.dispose();
|
||||
});
|
||||
test('the stream keeps the first page\'s storage epoch and refuses a later page from another one (#437)',async()=>{
|
||||
@@ -131,3 +145,28 @@ test('the shelf no longer sorts by rating (D153)',async()=>{
|
||||
assert.throws(()=>createLibraryStream({api:{get:async()=>page([])},restore:{filters:{sort:'rating'},state:{}}}),/Invalid library sort/);
|
||||
for(const sort of ['newest','oldest','name']) assert.equal(await s.reset({sort}),0);s.dispose();
|
||||
});
|
||||
test('Refine › Folder nests each folder under its exact parent, by path segment (#444 live B1)',async()=>{
|
||||
const {libraryFolderOptions}=await source;assert.equal(typeof libraryFolderOptions,'function');
|
||||
const I=' ';
|
||||
// The server's flat list sorts full paths as text, so "-" and "_" land before "/" (live report shapes).
|
||||
const folders=['CCC','CCC/mabes9sans-dataset','CCC/mabes9sans-dataset-v1','CCC/mabes9sans-dataset/mabes9sans-dataset_IMG_GEN',
|
||||
'FLUX','FLUX/2025-12-11','flux','flux/images','flux-klein','illustrious','illustrious-base','illustrious/2026-10-26','illustrious/training',
|
||||
'illustrious/training/a','kiko','kiko-v6','kiko/char','kiko/XXX','QWEN','QWEN/x','Qwen','Qwen/y','Qwen-Training','Video','Video/v','video'].sort();
|
||||
const options=libraryFolderOptions(folders);
|
||||
assert.deepEqual(options[0],['','All output folders']);
|
||||
assert.deepEqual(options.slice(1),[
|
||||
['CCC','CCC'],['CCC/mabes9sans-dataset',`${I}mabes9sans-dataset`],['CCC/mabes9sans-dataset/mabes9sans-dataset_IMG_GEN',`${I}${I}mabes9sans-dataset_IMG_GEN`],
|
||||
['CCC/mabes9sans-dataset-v1',`${I}mabes9sans-dataset-v1`],
|
||||
['FLUX','FLUX'],['FLUX/2025-12-11',`${I}2025-12-11`],['flux','flux'],['flux/images',`${I}images`],['flux-klein','flux-klein'],
|
||||
['illustrious','illustrious'],['illustrious/2026-10-26',`${I}2026-10-26`],['illustrious/training',`${I}training`],['illustrious/training/a',`${I}${I}a`],
|
||||
['illustrious-base','illustrious-base'],
|
||||
['kiko','kiko'],['kiko/char',`${I}char`],['kiko/XXX',`${I}XXX`],['kiko-v6','kiko-v6'],
|
||||
['QWEN','QWEN'],['QWEN/x',`${I}x`],['Qwen','Qwen'],['Qwen/y',`${I}y`],['Qwen-Training','Qwen-Training'],
|
||||
['Video','Video'],['Video/v',`${I}v`],['video','video'],
|
||||
]);
|
||||
// Every parent precedes its children directly; a chosen folder missing from the list is kept, with its parents.
|
||||
assert.deepEqual(libraryFolderOptions(['a'],'a/b/c').slice(1),[['a','a'],['a/b',`${I}b`],['a/b/c',`${I}${I}c`]]);
|
||||
assert.deepEqual(libraryFolderOptions([],''),[['','All output folders']]);
|
||||
// A POSIX folder whose name holds a backslash is one segment.
|
||||
assert.deepEqual(libraryFolderOptions(['Metadata\\Klein_4b Wildcard Run']).slice(1),[['Metadata\\Klein_4b Wildcard Run','Metadata\\Klein_4b Wildcard Run']]);
|
||||
});
|
||||
|
||||
@@ -5,7 +5,7 @@ const assert = require('node:assert/strict');
|
||||
const source = import('../../web/js/workspace/saved-views.mjs');
|
||||
|
||||
const IMAGE_DEFAULTS = { q: '', category: '', sort: 'newest', source: 'all', folder: '', media: 'all', tags: [], favorites: false, collection: '' };
|
||||
const LIBRARY_DEFAULTS = { q: '', category: '', sort: 'newest', tags: [], tag_mode: 'all' };
|
||||
const LIBRARY_DEFAULTS = { q: '', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' };
|
||||
|
||||
test('a saved state always holds every page key and keeps only values its page accepts', async () => {
|
||||
const { normalizeViewState, SAVED_VIEW_PAGES } = await source;
|
||||
@@ -31,8 +31,8 @@ test('a saved state always holds every page key and keeps only values its page a
|
||||
assert.deepEqual(normalizeViewState('images', { ...full, q: 'x'.repeat(501), folder: 7, tags: Array(21).fill('t') }),
|
||||
{ ...full, q: '', folder: '', tags: [] });
|
||||
assert.deepEqual(normalizeViewState('images', { ...full, category: 'bad\u0007', tags: [' '] }), { ...full, category: '', tags: [] });
|
||||
// Library has no source, folder or media, and no Name Z–A.
|
||||
assert.deepEqual(normalizeViewState('library', { ...full, sort: 'name-desc' }), { q: 'castle', category: 'Portraits', sort: 'newest', tags: ['moody', 'fog'], tag_mode: 'all' });
|
||||
// Library has no source or media, and no Name Z–A; its folder (#444) is the Library's own.
|
||||
assert.deepEqual(normalizeViewState('library', { ...full, sort: 'name-desc' }), { q: 'castle', category: 'Portraits', sort: 'newest', tags: ['moody', 'fog'], tag_mode: 'all', folder: 'a/b' });
|
||||
assert.deepEqual(normalizeViewState('library', { sort: 'oldest' }), { ...LIBRARY_DEFAULTS, sort: 'oldest' });
|
||||
// The returned tags are a copy.
|
||||
const tags = ['one'];
|
||||
@@ -62,7 +62,7 @@ test('the listing keeps usable views in order and normalizes their state', async
|
||||
{ id: 1, page: 'images', name: 'Videos', state: { media: 'video', source: '9' } },
|
||||
] });
|
||||
assert.deepEqual(views, [
|
||||
{ id: 3, page: 'library', name: 'Neon', state: { q: 'neon', category: '', sort: 'name', tags: [], tag_mode: 'all' } },
|
||||
{ id: 3, page: 'library', name: 'Neon', state: { q: 'neon', category: '', sort: 'name', tags: [], tag_mode: 'all', folder: '' } },
|
||||
{ id: 1, page: 'images', name: 'Videos', state: { ...IMAGE_DEFAULTS, media: 'video', source: '9' } },
|
||||
]);
|
||||
assert.equal(parseSavedView(null), null);
|
||||
@@ -84,8 +84,8 @@ test('page filters and saved states convert both ways', async () => {
|
||||
assert.equal(imagesViewState({ ...filters, collection: undefined }).collection, '');
|
||||
assert.equal(imagesViewState({ ...filters, collection: 7 }).collection, '7');
|
||||
assert.equal(imagesViewState({ ...filters, favorites: undefined }).favorites, false);
|
||||
const shelf = { text: 'neon', category: 'Cities', sort: 'oldest', tags: ['night'], tagMode: 'all' };
|
||||
assert.deepEqual(libraryViewState(shelf), { q: 'neon', category: 'Cities', sort: 'oldest', tags: ['night'], tag_mode: 'all' });
|
||||
const shelf = { text: 'neon', category: 'Cities', sort: 'oldest', tags: ['night'], tagMode: 'any', folder: '2026' };
|
||||
assert.deepEqual(libraryViewState(shelf), { q: 'neon', category: 'Cities', sort: 'oldest', tags: ['night'], tag_mode: 'any', folder: '2026' });
|
||||
assert.deepEqual(libraryFilters(libraryViewState(shelf)), shelf);
|
||||
});
|
||||
|
||||
@@ -216,16 +216,27 @@ test('a state over the API 4 KB cap is refused before anything is sent (#249 M5)
|
||||
assert.deepEqual(api.calls, []);
|
||||
});
|
||||
|
||||
test('a Library view keeps its tag match, and untagged never travels with tags (#444)', async () => {
|
||||
test('a Library view keeps its tag match and folder; an old view reads the defaults (#444)', async () => {
|
||||
const { normalizeViewState, libraryFilters } = await source;
|
||||
assert.deepEqual(normalizeViewState('library', { tag_mode: 'any', folder: '2026/Aug' }), { ...LIBRARY_DEFAULTS, tag_mode: 'any', folder: '2026/Aug' });
|
||||
assert.deepEqual(normalizeViewState('library', { tag_mode: 'or', folder: 'x'.repeat(1025) }), LIBRARY_DEFAULTS);
|
||||
assert.deepEqual(libraryFilters({ q: 'a' }), { text: 'a', category: '', sort: 'newest', tags: [], tagMode: 'all', folder: '' });
|
||||
});
|
||||
|
||||
test('a Library view keeps untagged, and untagged never travels with tags (#444)', async () => {
|
||||
const { normalizeViewState, libraryFilters, libraryViewState } = await source;
|
||||
assert.equal(normalizeViewState('library', { tag_mode: 'untagged' }).tag_mode, 'untagged');
|
||||
assert.equal(normalizeViewState('library', { tag_mode: 'any' }).tag_mode, 'all');
|
||||
assert.deepEqual(normalizeViewState('library', { tags: ['a'], tag_mode: 'untagged' }), { ...LIBRARY_DEFAULTS, tags: ['a'] });
|
||||
const shelf = { text: '', category: '', sort: 'newest', tags: [], tagMode: 'untagged' };
|
||||
const shelf = { text: '', category: '', sort: 'newest', tags: [], tagMode: 'untagged', folder: '' };
|
||||
assert.equal(libraryViewState(shelf).tag_mode, 'untagged');
|
||||
assert.deepEqual(libraryFilters(libraryViewState(shelf)), shelf);
|
||||
});
|
||||
|
||||
test('a shelf without the #444 filters saves their defaults', async () => {
|
||||
const { libraryViewState } = await source;
|
||||
assert.deepEqual(libraryViewState({ text: '', category: '', sort: 'newest', tags: [] }), LIBRARY_DEFAULTS);
|
||||
});
|
||||
|
||||
test('a view saved under the retired Highest rated sort opens on Recently added (D153)', async () => {
|
||||
const { normalizeViewState, imagesFilters, libraryFilters } = await source;
|
||||
assert.equal(normalizeViewState('images', { sort: 'rating' }).sort, 'newest');
|
||||
|
||||
+44
-2
@@ -28,6 +28,10 @@ stack shows which). A dump after the summary is a hang at interpreter exit
|
||||
|
||||
A limit is a measurement, not a dial: change it only with new measurements
|
||||
written next to it (TESTING.md "Time limits and hangs").
|
||||
|
||||
PM_TEST_SLOWEST=N (opt-in, #487) lists the N slowest tests after the summary.
|
||||
A test's time runs from its ``start:`` line to the next one, so it includes
|
||||
the class and module fixtures in between, like the hang limit.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -36,6 +40,7 @@ import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
@@ -63,23 +68,57 @@ def read_limit(name, default):
|
||||
return int(text)
|
||||
|
||||
|
||||
def make_result_class(stream, per_test_limit, shutdown_limit):
|
||||
class TestClock:
|
||||
"""Seconds from each test's start to the next start (or the run's end)."""
|
||||
|
||||
def __init__(self, clock=time.perf_counter):
|
||||
self._clock = clock
|
||||
self._current = None
|
||||
self._started = 0.0
|
||||
self.durations = []
|
||||
|
||||
def start(self, test_id):
|
||||
self.stop()
|
||||
self._current = test_id
|
||||
self._started = self._clock()
|
||||
|
||||
def stop(self):
|
||||
if self._current is not None:
|
||||
self.durations.append((self._clock() - self._started, self._current))
|
||||
self._current = None
|
||||
|
||||
def report(self, count):
|
||||
"""Return the lines that list the ``count`` slowest tests."""
|
||||
slowest = sorted(self.durations, key=lambda item: item[0], reverse=True)
|
||||
lines = [f"slowest {min(count, len(slowest))} of {len(slowest)} tests:"]
|
||||
lines += [f"{seconds:8.2f}s {test_id}" for seconds, test_id in slowest[:count]]
|
||||
return lines
|
||||
|
||||
|
||||
def make_result_class(stream, per_test_limit, shutdown_limit, slowest=0):
|
||||
"""Build a TextTestResult that names each test and re-arms the watchdog."""
|
||||
|
||||
def arm(seconds):
|
||||
faulthandler.dump_traceback_later(seconds, exit=True, file=stream)
|
||||
|
||||
clock = TestClock()
|
||||
|
||||
class WatchdogResult(unittest.TextTestResult):
|
||||
def startTest(self, test):
|
||||
# At default verbosity the progress dots share a line; start ours.
|
||||
prefix = "\n" if self.dots else ""
|
||||
stream.write(f"{prefix}start: {test.id()}\n")
|
||||
stream.flush()
|
||||
clock.start(test.id())
|
||||
super().startTest(test)
|
||||
arm(per_test_limit)
|
||||
|
||||
def stopTestRun(self):
|
||||
clock.stop()
|
||||
super().stopTestRun()
|
||||
if slowest:
|
||||
stream.write("\n".join(["", *clock.report(slowest), ""]))
|
||||
stream.flush()
|
||||
arm(shutdown_limit)
|
||||
|
||||
return WatchdogResult
|
||||
@@ -138,6 +177,7 @@ def main(argv=None):
|
||||
try:
|
||||
per_test_limit = read_limit("PM_TEST_HANG_LIMIT", DEFAULT_PER_TEST_LIMIT)
|
||||
shutdown_limit = read_limit("PM_TEST_SHUTDOWN_LIMIT", DEFAULT_SHUTDOWN_LIMIT)
|
||||
slowest = read_limit("PM_TEST_SLOWEST", 0)
|
||||
except LimitError as error:
|
||||
stream.write(f"{error}\n")
|
||||
return USAGE_EXIT
|
||||
@@ -152,7 +192,9 @@ def main(argv=None):
|
||||
verbosity=args.verbosity,
|
||||
# As unittest.main does: show warnings unless -W/PYTHONWARNINGS set them.
|
||||
warnings=None if sys.warnoptions else "default",
|
||||
resultclass=make_result_class(stream, per_test_limit, shutdown_limit),
|
||||
resultclass=make_result_class(
|
||||
stream, per_test_limit, shutdown_limit, slowest
|
||||
),
|
||||
)
|
||||
result = runner.run(suite)
|
||||
finally:
|
||||
|
||||
@@ -684,6 +684,132 @@ class TestStaticRoutes(CoreRoutesTestCase):
|
||||
self.assertEqual(resp.status, 403)
|
||||
|
||||
|
||||
class TestRevalidatedCaching(CoreRoutesTestCase):
|
||||
"""Every served page and static file is revalidated on reload (#478).
|
||||
|
||||
``Cache-Control: no-cache`` makes the browser ask before it reuses a copy,
|
||||
and the ETag lets an unchanged file answer 304 without a body, so an update
|
||||
reaches the browser on a normal reload and an unchanged reload stays cheap.
|
||||
"""
|
||||
|
||||
STATIC_PATHS = (
|
||||
"/prompt_manager/js/workspace/index.mjs",
|
||||
"/prompt_manager/js/admin.js",
|
||||
"/prompt_manager/js/inspection-semantics.json",
|
||||
"/prompt_manager/lib/viewerjs/viewer.min.css",
|
||||
)
|
||||
|
||||
async def _assert_revalidated(self, path):
|
||||
resp = await self.client.request("GET", path)
|
||||
self.assertEqual(resp.status, 200, path)
|
||||
self.assertEqual(resp.headers.get("Cache-Control"), "no-cache", path)
|
||||
etag = resp.headers.get("ETag")
|
||||
self.assertTrue(etag, path)
|
||||
body = await resp.read()
|
||||
self.assertTrue(body, path)
|
||||
|
||||
again = await self.client.request("GET", path, headers={"If-None-Match": etag})
|
||||
self.assertEqual(again.status, 304, path)
|
||||
self.assertEqual(again.headers.get("ETag"), etag, path)
|
||||
self.assertEqual(again.headers.get("Cache-Control"), "no-cache", path)
|
||||
self.assertEqual(await again.read(), b"", path)
|
||||
|
||||
stale = await self.client.request(
|
||||
"GET", path, headers={"If-None-Match": 'W/"stale"'}
|
||||
)
|
||||
self.assertEqual(stale.status, 200, path)
|
||||
self.assertEqual(await stale.read(), body, path)
|
||||
return etag
|
||||
|
||||
async def test_static_files_are_revalidated(self):
|
||||
for path in self.STATIC_PATHS:
|
||||
await self._assert_revalidated(path)
|
||||
|
||||
async def test_pages_are_revalidated(self):
|
||||
for path in UI_PATHS:
|
||||
await self._assert_revalidated(path)
|
||||
|
||||
async def test_etag_matches_in_a_list_and_as_wildcard(self):
|
||||
path = "/prompt_manager/js/workspace/index.mjs"
|
||||
etag = (await self.client.request("GET", path)).headers["ETag"]
|
||||
for header in (f'W/"other", {etag}', etag.removeprefix("W/"), "*"):
|
||||
resp = await self.client.request(
|
||||
"GET", path, headers={"If-None-Match": header}
|
||||
)
|
||||
self.assertEqual(resp.status, 304, header)
|
||||
|
||||
async def test_changed_file_gets_a_new_etag_and_its_new_body(self):
|
||||
web_dir = self._fake_root()
|
||||
module = web_dir / "js" / "m.mjs"
|
||||
module.write_text("export const v = 1;")
|
||||
path = "/prompt_manager/js/m.mjs"
|
||||
first = await self.client.request("GET", path)
|
||||
etag = first.headers["ETag"]
|
||||
# Same size and, on coarse clocks, the same mtime: only the bytes differ.
|
||||
module.write_text("export const v = 2;")
|
||||
resp = await self.client.request("GET", path, headers={"If-None-Match": etag})
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertNotEqual(resp.headers["ETag"], etag)
|
||||
self.assertEqual(await resp.text(), "export const v = 2;")
|
||||
|
||||
async def test_gzipped_and_plain_bodies_share_a_weak_etag(self):
|
||||
"""The gzip middleware changes the bytes, so the tag must be weak."""
|
||||
resp = await self.client.request("GET", "/prompt_manager/js/admin.js")
|
||||
self.assertTrue(resp.headers["ETag"].startswith('W/"'))
|
||||
|
||||
async def test_errors_carry_no_validator(self):
|
||||
resp = await self.client.request("GET", "/prompt_manager/js/nope.js")
|
||||
self.assertEqual(resp.status, 404)
|
||||
self.assertNotIn("ETag", resp.headers)
|
||||
|
||||
|
||||
@web.middleware
|
||||
async def _comfyui_cache_control(request, handler):
|
||||
"""ComfyUI's own middleware (middleware/cache_middleware.py), JS/CSS branch.
|
||||
|
||||
It only fills a missing Cache-Control, and ``.mjs`` is not ``.js``: that is
|
||||
why the workspace modules went out with no header at all before #478.
|
||||
"""
|
||||
response = await handler(request)
|
||||
if request.path.endswith(".js") or request.path.endswith(".css"):
|
||||
response.headers.setdefault("Cache-Control", "no-store")
|
||||
return response
|
||||
|
||||
|
||||
class TestRevalidatedCachingBehindMiddlewares(CoreRoutesTestCase):
|
||||
"""The host's middlewares keep the header and the 304 intact."""
|
||||
|
||||
async def get_application(self):
|
||||
app = await super().get_application()
|
||||
app.middlewares.append(_comfyui_cache_control)
|
||||
app.middlewares.append(api_module._gzip_middleware)
|
||||
return app
|
||||
|
||||
async def test_gzipped_files_keep_no_cache_and_answer_304(self):
|
||||
for path in (
|
||||
"/prompt_manager/js/workspace/index.mjs",
|
||||
"/prompt_manager/js/admin.js",
|
||||
"/prompt_manager/lib/viewerjs/viewer.min.css",
|
||||
"/prompt_manager/workspace",
|
||||
):
|
||||
resp = await self.client.request(
|
||||
"GET", path, headers={"Accept-Encoding": "gzip"}
|
||||
)
|
||||
self.assertEqual(resp.status, 200, path)
|
||||
self.assertEqual(resp.headers.get("Content-Encoding"), "gzip", path)
|
||||
self.assertEqual(resp.headers["Cache-Control"], "no-cache", path)
|
||||
again = await self.client.request(
|
||||
"GET",
|
||||
path,
|
||||
headers={
|
||||
"Accept-Encoding": "gzip",
|
||||
"If-None-Match": resp.headers["ETag"],
|
||||
},
|
||||
)
|
||||
self.assertEqual(again.status, 304, path)
|
||||
self.assertEqual(again.headers["Cache-Control"], "no-cache", path)
|
||||
|
||||
|
||||
# ── instance helpers ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -45,7 +45,9 @@ SEEDED_PATHS = (
|
||||
"/srv/someone/renders",
|
||||
"C:\\Users\\someone\\Documents\\renders",
|
||||
"D:/Users/someone/prompts.db",
|
||||
"\\\\nas\\share\\someone\\renders",
|
||||
# A share on this machine: on Windows the report resolves every source,
|
||||
# and an unknown server name made that wait about 2.9 s (#487).
|
||||
"\\\\localhost\\pm-no-such-share\\someone\\renders",
|
||||
)
|
||||
_WINDOWS_ABSOLUTE = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
TOP_KEYS = {
|
||||
|
||||
@@ -18,11 +18,14 @@ from unittest.mock import patch
|
||||
|
||||
from aiohttp import FormData, web
|
||||
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
try:
|
||||
from test_api_images import ImageAPITestCase
|
||||
except ImportError:
|
||||
from tests.test_api_images import ImageAPITestCase
|
||||
|
||||
from database import restore_stage
|
||||
from database.operations import PromptDatabase
|
||||
from py.api.library import _encode, _scope
|
||||
from tests.test_execution_context_database import bundle, snapshot
|
||||
@@ -41,6 +44,21 @@ OTHER_SESSION = "aaaaaaaa-3333-4333-8333-333333333333"
|
||||
MISSING_RUN = "bbbbbbbb-4444-4444-8444-444444444444"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def no_wal_truncate_wait():
|
||||
"""Restore with the restore notification held back, as these tests need.
|
||||
|
||||
The read under test keeps its snapshot open across the restore, so the
|
||||
publish's WAL truncate can never succeed; without the short wait it would
|
||||
sit out the full CHECKPOINT_BUSY_MS (30 s) each time (#487).
|
||||
"""
|
||||
with (
|
||||
patch("database.execution_history.notify_history_restore"),
|
||||
patch.object(restore_stage, "CHECKPOINT_BUSY_MS", 50),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
class HistoryAPITests(ImageAPITestCase):
|
||||
async def tearDownAsync(self):
|
||||
if hasattr(self.api, "close_history_reads"):
|
||||
@@ -191,7 +209,11 @@ class HistoryAPITests(ImageAPITestCase):
|
||||
context["evidence"]["positive"]["raw"]["provenance"], "recorded"
|
||||
)
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
conn.execute("UPDATE prompts SET text='edited' WHERE id=?", (positive,))
|
||||
# An edit changes the text and its dedup hash together (#460)
|
||||
conn.execute(
|
||||
"UPDATE prompts SET text='edited',hash=? WHERE id=?",
|
||||
(generate_prompt_hash("edited"), positive),
|
||||
)
|
||||
_, changed = await self.detail(start.run_id)
|
||||
self.assertEqual(changed["contexts"][0]["positive"], context["positive"])
|
||||
self.assertFalse(changed["contexts"][0]["navigation"]["positive_available"])
|
||||
@@ -340,7 +362,7 @@ class HistoryAPITests(ImageAPITestCase):
|
||||
|
||||
def during_snapshot(*args, **kwargs):
|
||||
result = original(*args, **kwargs)
|
||||
with patch("database.execution_history.notify_history_restore"):
|
||||
with no_wal_truncate_wait():
|
||||
writer.model.restore_from_file(source)
|
||||
return result
|
||||
|
||||
@@ -884,7 +906,7 @@ class HistoryAPITests(ImageAPITestCase):
|
||||
|
||||
def during_snapshot(*args, **kwargs):
|
||||
result = original(*args, **kwargs)
|
||||
with patch("database.execution_history.notify_history_restore"):
|
||||
with no_wal_truncate_wait():
|
||||
writer.model.restore_from_file(source)
|
||||
return result
|
||||
|
||||
|
||||
@@ -1015,6 +1015,122 @@ class HistoryOutputAPITests(ImageAPITestCase):
|
||||
response = await self.current(descriptor["inspection_url"])
|
||||
self.assert_error(response, await response.json(), 422, "response-limit")
|
||||
|
||||
async def tag_image(self, relative, tags):
|
||||
"""Save *tags* on Images root 0's *relative* file through the Tag routes.
|
||||
|
||||
The real observe and save routes store the locator the Tag dialog
|
||||
stores (``media_capture._resolve``), so History must find it (#457).
|
||||
"""
|
||||
self.addCleanup(self.api.close_annotation_captures)
|
||||
roots = self.api._get_all_output_dirs()
|
||||
target = {
|
||||
"root_index": 0,
|
||||
"root_scope": self.api._gallery_root_scope(roots[0], 0),
|
||||
"path": relative,
|
||||
}
|
||||
response = await self.client.post(
|
||||
"/prompt_manager/image-annotations/observe", json={"target": target}
|
||||
)
|
||||
observed = await response.json()
|
||||
self.assertEqual(response.status, 200, observed)
|
||||
token = observed["capture"]["token"]
|
||||
try:
|
||||
response = await self.client.put(
|
||||
"/prompt_manager/image-annotations/tags",
|
||||
json={
|
||||
"target": target,
|
||||
"capture_token": token,
|
||||
"sha256": observed["capture"]["sha256"],
|
||||
"expected_epoch": observed["library_epoch"],
|
||||
"expected_tag_revision": observed["tag_revision"],
|
||||
"tags": list(tags),
|
||||
},
|
||||
)
|
||||
saved = await response.json()
|
||||
self.assertEqual(response.status, 200, saved)
|
||||
finally:
|
||||
await self.client.delete(
|
||||
f"/prompt_manager/image-annotations/capture/{token}"
|
||||
)
|
||||
return saved["tags"]
|
||||
|
||||
async def test_current_inspection_carries_the_files_image_tags(self):
|
||||
"""#457: History reads the file's own image tags, as Images does (S1)."""
|
||||
sub = self.output_dir / "batch"
|
||||
sub.mkdir()
|
||||
tagged = png(sub / "tagged.png", prompt=graph("Tagged file"))
|
||||
png(self.output_dir / "plain.png", prompt=graph("Plain file"))
|
||||
self.assertEqual(
|
||||
await self.tag_image("batch/tagged.png", ["dunes", "warm_light"]),
|
||||
["dunes", "warm_light"],
|
||||
)
|
||||
start = self.capture(
|
||||
[
|
||||
self.ref("1", subfolder="batch", filename="tagged.png"),
|
||||
self.ref("2", filename="plain.png"),
|
||||
]
|
||||
)
|
||||
_, page = await self.outputs(start)
|
||||
urls = [
|
||||
item["current_candidate"]["media"]["inspection_url"]
|
||||
for item in page["items"]
|
||||
]
|
||||
bodies = []
|
||||
for url in urls:
|
||||
response = await self.current(url)
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
bodies.append(body)
|
||||
self.assertEqual(
|
||||
bodies[0].get("image_tags"),
|
||||
{"state": "current", "tags": ["dunes", "warm_light"]},
|
||||
)
|
||||
# Image tags are the file's own search metadata, never prompt tags.
|
||||
self.assertEqual(bodies[0]["prompts"]["positive"]["text"], "Tagged file")
|
||||
self.assertEqual(bodies[1].get("image_tags"), {"state": "none", "tags": []})
|
||||
# New bytes at the same path: the tags belong to the old byte record.
|
||||
stat = tagged.stat()
|
||||
os.utime(tagged, ns=(stat.st_atime_ns, stat.st_mtime_ns + 10**9))
|
||||
response = await self.current(urls[0])
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
self.assertEqual(body["image_tags"], {"state": "none", "tags": []})
|
||||
|
||||
async def test_current_inspection_finds_image_tags_under_a_symlinked_output(self):
|
||||
"""An output folder reached through a link (another drive) still matches."""
|
||||
link = self.comfy_dir / "linked_output"
|
||||
self._make_symlink(self.output_dir, link, target_is_directory=True)
|
||||
self.api._get_all_output_dirs = lambda: [link]
|
||||
self.host_output = str(link)
|
||||
png(self.output_dir / "image.png", prompt=graph("Linked folder"))
|
||||
self.assertEqual(await self.tag_image("image.png", ["dunes"]), ["dunes"])
|
||||
start = self.capture([self.ref()])
|
||||
descriptor = await self.media(start)
|
||||
response = await self.current(descriptor["inspection_url"])
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
self.assertEqual(body["image_tags"], {"state": "current", "tags": ["dunes"]})
|
||||
|
||||
async def test_current_inspection_image_tag_failure_is_unavailable(self):
|
||||
png(self.output_dir / "image.png", prompt=graph("File evidence"))
|
||||
await self.tag_image("image.png", ["dunes"])
|
||||
start = self.capture([self.ref()])
|
||||
descriptor = await self.media(start)
|
||||
with (
|
||||
self.assertLogs(self.api.logger, "WARNING") as logs,
|
||||
patch.object(
|
||||
type(self.api.db),
|
||||
"current_image_tags",
|
||||
side_effect=sqlite3.OperationalError("locked"),
|
||||
),
|
||||
):
|
||||
response = await self.current(descriptor["inspection_url"])
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
self.assertEqual(body["image_tags"], {"state": "unavailable", "tags": []})
|
||||
self.assertEqual(body["prompts"]["positive"]["text"], "File evidence")
|
||||
self.assertNotIn(str(self.output_dir), "\n".join(logs.output))
|
||||
|
||||
async def test_optional_candidate_failure_preserves_every_occurrence_without_urls(
|
||||
self,
|
||||
):
|
||||
|
||||
@@ -35,6 +35,7 @@ from aiohttp import web # noqa: E402
|
||||
from aiohttp.test_utils import AioHTTPTestCase # noqa: E402
|
||||
from PIL import Image # noqa: E402
|
||||
|
||||
from database.models import PromptModel # noqa: E402
|
||||
from database.operations import PromptDatabase # noqa: E402
|
||||
import py.api.images as images_module
|
||||
from py.api import gallery_listing # noqa: E402
|
||||
@@ -92,14 +93,41 @@ class ImageAPITestCase(AioHTTPTestCase):
|
||||
self._anchor_patch.stop()
|
||||
# Windows refuses to unlink a database that still has open handles.
|
||||
self.api.db.close_all()
|
||||
for path in (
|
||||
self._temp_db.name,
|
||||
self._temp_db.name + "-wal",
|
||||
self._temp_db.name + "-shm",
|
||||
):
|
||||
if os.path.exists(path):
|
||||
os.unlink(path)
|
||||
self._output_tmp.cleanup()
|
||||
try:
|
||||
leaked = self._close_leaked_connections()
|
||||
finally:
|
||||
for path in (
|
||||
self._temp_db.name,
|
||||
self._temp_db.name + "-wal",
|
||||
self._temp_db.name + "-shm",
|
||||
):
|
||||
if os.path.exists(path):
|
||||
os.unlink(path)
|
||||
self._output_tmp.cleanup()
|
||||
if leaked:
|
||||
raise AssertionError(
|
||||
f"{leaked} connection(s) to the test database were still open at "
|
||||
"teardown; Windows cannot unlink it (WinError 32). Close every "
|
||||
"extra PromptDatabase before tearDownAsync, not in a cleanup."
|
||||
)
|
||||
|
||||
def _close_leaked_connections(self):
|
||||
"""Close and count connections other models still hold on the temp DB.
|
||||
|
||||
Linux unlinks an open SQLite file, Windows does not; counting here makes
|
||||
a handle left open past teardown fail on every OS (#475).
|
||||
"""
|
||||
key = os.path.normcase(os.path.realpath(self._temp_db.name))
|
||||
with PromptModel._instances_lock:
|
||||
models = list(PromptModel._instances)
|
||||
leaked = 0
|
||||
for model in models:
|
||||
if model._schema_key() != key:
|
||||
continue
|
||||
with model._conn_lock:
|
||||
leaked += len(model._connections)
|
||||
model.close_all()
|
||||
return leaked
|
||||
|
||||
# -- helpers ---------------------------------------------------------
|
||||
|
||||
|
||||
@@ -123,6 +123,72 @@ class TestLibraryAPI(ImageAPITestCase):
|
||||
},
|
||||
)
|
||||
|
||||
async def test_folder_filter_lists_and_matches_prompts_by_output_folder(self):
|
||||
deep = self._save_prompt("deep")
|
||||
top = self._save_prompt("top")
|
||||
other = self._save_prompt("other")
|
||||
self._save_prompt("no images")
|
||||
for prompt, parts in ((deep, ("a", "b")), (top, ("a",)), (other, ("ab",))):
|
||||
path = make_png(self.output_dir.joinpath(*parts, f"{prompt}.png"))
|
||||
self.api.db.link_image_to_prompt(prompt, str(path))
|
||||
response, options = await self.get("/options", include="folders")
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(options["folders"], ["a", "a/b", "ab"])
|
||||
_, plain = await self.get("/options")
|
||||
self.assertNotIn("folders", plain)
|
||||
_, first = await self.get(folder="a", limit=1)
|
||||
self.assertEqual((first["total"], first["folder"]), (2, "a"))
|
||||
# Outer slashes are dropped (a backslash separates only on Windows).
|
||||
outer = "\\a\\" if os.name == "nt" else "/a/"
|
||||
_, second = await self.get(folder=outer, limit=1, cursor=first["next_cursor"])
|
||||
self.assertEqual(
|
||||
sorted([first["items"][0]["text"], second["items"][0]["text"]]),
|
||||
["deep", "top"],
|
||||
)
|
||||
_, all_rows = await self.get()
|
||||
self.assertEqual((all_rows["total"], all_rows["folder"]), (4, ""))
|
||||
for params in (
|
||||
{"folder": "a/../ab"},
|
||||
*([{"folder": "C:/x"}] if os.name == "nt" else []),
|
||||
{"folder": "b", "cursor": first["next_cursor"]},
|
||||
{"cursor": first["next_cursor"]},
|
||||
):
|
||||
response, result = await self.get(limit=1, **params)
|
||||
self.assertEqual(response.status, 400, params)
|
||||
self.assertEqual(result["success"], False)
|
||||
response, _ = await self.get("/options", include="tags")
|
||||
self.assertEqual(response.status, 400)
|
||||
|
||||
@unittest.skipIf(os.name == "nt", "a backslash is a separator on Windows")
|
||||
async def test_a_posix_folder_name_with_a_backslash_filters_its_prompts(self):
|
||||
prompt = self._save_prompt("klein")
|
||||
name = "Metadata\\Klein_4b Wildcard Run"
|
||||
path = make_png(self.output_dir / name / "x.png")
|
||||
self.api.db.link_image_to_prompt(prompt, str(path))
|
||||
_, options = await self.get("/options", include="folders")
|
||||
self.assertIn(name, options["folders"])
|
||||
_, result = await self.get(folder=name)
|
||||
self.assertEqual((result["total"], result["folder"]), (1, name))
|
||||
|
||||
async def test_any_tag_mode_is_sent_through_and_echoed(self):
|
||||
self._save_prompt("a only", tags=["a"])
|
||||
self._save_prompt("b only", tags=["b"])
|
||||
self._save_prompt("neither")
|
||||
_, result = await self.get(tag=["a", "b"], tag_mode="any")
|
||||
self.assertEqual(result["tag_mode"], "any")
|
||||
self.assertEqual(
|
||||
sorted(i["text"] for i in result["items"]), ["a only", "b only"]
|
||||
)
|
||||
_, both = await self.get(tag=["a", "b"])
|
||||
self.assertEqual((both["total"], both["tag_mode"]), (0, "all"))
|
||||
_, first = await self.get(tag=["a", "b"], tag_mode="any", limit=1)
|
||||
response, _ = await self.get(
|
||||
tag=["a", "b"], limit=1, cursor=first["next_cursor"]
|
||||
)
|
||||
self.assertEqual(response.status, 400)
|
||||
response, _ = await self.get(tag_mode="or")
|
||||
self.assertEqual(response.status, 400)
|
||||
|
||||
async def test_variant_cursor_is_positive_scoped_and_required_limits_are_bounded(
|
||||
self,
|
||||
):
|
||||
|
||||
@@ -38,11 +38,25 @@ def _payload(token):
|
||||
|
||||
class TestRestartKeepsTheOpenPage(ImageAPITestCase):
|
||||
async def get_application(self):
|
||||
self._restarts = []
|
||||
app = await super().get_application()
|
||||
for index in range(3):
|
||||
make_png(self.output_dir / f"image_{index}.png")
|
||||
return app
|
||||
|
||||
async def tearDownAsync(self):
|
||||
# Before the base teardown unlinks the database: cleanups run after
|
||||
# tearDownAsync, and Windows cannot unlink a file still open (#475).
|
||||
errors = []
|
||||
while self._restarts:
|
||||
try:
|
||||
await self._restarts.pop()()
|
||||
except Exception as error: # every closer runs; the first error wins
|
||||
errors.append(error)
|
||||
await super().tearDownAsync()
|
||||
if errors:
|
||||
raise errors[0]
|
||||
|
||||
async def first_page(self, client=None, **params):
|
||||
response = await (client or self.client).get(
|
||||
LISTING, params={"mode": "workspace", "limit": "1", **params}
|
||||
@@ -67,7 +81,7 @@ class TestRestartKeepsTheOpenPage(ImageAPITestCase):
|
||||
api.close_auto_thumbnails()
|
||||
api.db.close_all()
|
||||
|
||||
self.addAsyncCleanup(close)
|
||||
self._restarts.append(close)
|
||||
return api, client
|
||||
|
||||
async def test_a_restart_keeps_every_held_scope_and_cursor_valid(self):
|
||||
@@ -152,8 +166,13 @@ class TestRestartKeepsTheOpenPage(ImageAPITestCase):
|
||||
other_db = self.comfy_dir / "elsewhere" / "prompts.db"
|
||||
other_db.parent.mkdir()
|
||||
api = PromptManagerAPI(db=PromptDatabase(str(other_db)))
|
||||
self.addCleanup(api.db.close_all)
|
||||
self.addCleanup(api.close_auto_thumbnails)
|
||||
|
||||
async def close():
|
||||
api.close_auto_thumbnails()
|
||||
api.db.close_all()
|
||||
|
||||
# Closed before teardown removes the folder that holds its database.
|
||||
self._restarts.append(close)
|
||||
self.assertNotEqual(
|
||||
api._gallery_root_scope(self.output_dir, 0), page["images"][0]["root_scope"]
|
||||
)
|
||||
|
||||
@@ -8,6 +8,7 @@ import contextlib
|
||||
import csv
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
@@ -15,7 +16,18 @@ import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, _REPO_ROOT)
|
||||
|
||||
# Run in a child interpreter: import py.autotag with a stub ComfyUI ``server``
|
||||
# and print every ML backend that the import pulled in.
|
||||
_IMPORT_AUTOTAG_REPORT_BACKENDS = """
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
sys.modules["server"] = MagicMock()
|
||||
import py.autotag
|
||||
print(*sorted({"onnxruntime", "llama_cpp"} & set(sys.modules)))
|
||||
"""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
@@ -142,17 +154,26 @@ class TestModelInjection(EngineTestCase):
|
||||
self.assertEqual(service._model_loader, service._load_model)
|
||||
|
||||
def test_importing_autotag_does_not_import_ml_backends(self):
|
||||
for heavy in ("onnxruntime", "llama_cpp"):
|
||||
self.assertNotIn(heavy, sys.modules)
|
||||
# A fresh interpreter: earlier tests in this process may have imported
|
||||
# an installed backend, so only a clean import can prove this (#472).
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", _IMPORT_AUTOTAG_REPORT_BACKENDS],
|
||||
cwd=_REPO_ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.split(), [])
|
||||
|
||||
def test_default_wd14_loader_reports_missing_onnxruntime(self):
|
||||
_touch_model_files(self.models_dir, "wd14-vit")
|
||||
service = AutoTagService(models_dir=self.models_dir)
|
||||
if "onnxruntime" in sys.modules:
|
||||
self.skipTest("onnxruntime is installed in this environment")
|
||||
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
service.load_model("wd14-vit")
|
||||
# A None entry makes onnxruntime unimportable even where it is installed.
|
||||
with _patch_modules({"onnxruntime": None}):
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
service.load_model("wd14-vit")
|
||||
|
||||
self.assertIn("onnxruntime", str(ctx.exception))
|
||||
self.assertFalse(service.is_model_loaded())
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""#424: each tagging backend reports the Python packages it cannot import.
|
||||
|
||||
``find_spec`` is patched throughout: no ML backend is installed or imported.
|
||||
``find_spec`` is patched throughout, and a backend an earlier test imported is
|
||||
hidden where it matters, so these pass with or without the ML packages (#472).
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -75,8 +76,13 @@ class TestMissingPackages(unittest.TestCase):
|
||||
def test_find_spec_errors_count_as_missing(self):
|
||||
for error in (ValueError("no spec"), ImportError("broken"), OSError("io")):
|
||||
with self.subTest(error=type(error).__name__):
|
||||
with patch.object(
|
||||
autotag.importlib.util, "find_spec", side_effect=error
|
||||
# Hide an onnxruntime an earlier test imported, so the check
|
||||
# reaches find_spec (#472).
|
||||
with (
|
||||
_patch_modules({"onnxruntime": None}),
|
||||
patch.object(
|
||||
autotag.importlib.util, "find_spec", side_effect=error
|
||||
),
|
||||
):
|
||||
self.assertEqual(missing_packages("wd14-vit"), ["onnxruntime"])
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""#516: the browser test server pins glibc's mmap threshold.
|
||||
|
||||
Large frees then go back to the OS instead of fragmenting per-thread arenas.
|
||||
"""
|
||||
|
||||
import ctypes
|
||||
import sys
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from tests.browser import malloc_tuning
|
||||
|
||||
|
||||
class FakeLibc:
|
||||
def __init__(self, result=1):
|
||||
self.calls = []
|
||||
self.result = result
|
||||
|
||||
def mallopt(self, option, value):
|
||||
self.calls.append((option, value))
|
||||
return self.result
|
||||
|
||||
|
||||
class PinMmapThresholdTests(unittest.TestCase):
|
||||
def test_sets_a_fixed_threshold_through_mallopt(self):
|
||||
libc = FakeLibc()
|
||||
with mock.patch.object(sys, "platform", "linux"):
|
||||
self.assertTrue(malloc_tuning.pin_mmap_threshold(libc))
|
||||
# M_MMAP_THRESHOLD is -3; setting it also turns off glibc's dynamic
|
||||
# raising of the threshold.
|
||||
self.assertEqual(libc.calls, [(-3, 128 * 1024)])
|
||||
|
||||
def test_reports_a_refused_setting(self):
|
||||
with mock.patch.object(sys, "platform", "linux"):
|
||||
self.assertFalse(malloc_tuning.pin_mmap_threshold(FakeLibc(result=0)))
|
||||
|
||||
def test_skips_a_libc_without_mallopt(self):
|
||||
with mock.patch.object(sys, "platform", "linux"):
|
||||
self.assertFalse(malloc_tuning.pin_mmap_threshold(object()))
|
||||
|
||||
def test_skips_other_platforms_without_loading_a_libc(self):
|
||||
for name in ("win32", "darwin"):
|
||||
with (
|
||||
self.subTest(name),
|
||||
mock.patch.object(sys, "platform", name),
|
||||
mock.patch.object(
|
||||
ctypes, "CDLL", side_effect=AssertionError("no libc load")
|
||||
),
|
||||
):
|
||||
self.assertFalse(malloc_tuning.pin_mmap_threshold())
|
||||
|
||||
def test_skips_when_the_process_libc_cannot_be_loaded(self):
|
||||
with (
|
||||
mock.patch.object(sys, "platform", "linux"),
|
||||
mock.patch.object(ctypes, "CDLL", side_effect=OSError("no libc")),
|
||||
):
|
||||
self.assertFalse(malloc_tuning.pin_mmap_threshold())
|
||||
|
||||
@unittest.skipUnless(
|
||||
sys.platform.startswith("linux")
|
||||
and hasattr(ctypes.CDLL(None), "gnu_get_libc_version"),
|
||||
"glibc only",
|
||||
)
|
||||
def test_glibc_accepts_the_setting(self):
|
||||
# This pins the threshold for the rest of this test process too; the
|
||||
# test server does the same at start, so nothing here depends on it.
|
||||
self.assertTrue(malloc_tuning.pin_mmap_threshold())
|
||||
|
||||
|
||||
class ServerMainTests(unittest.TestCase):
|
||||
def test_main_pins_the_threshold_before_it_starts_serving(self):
|
||||
from tests.browser import server
|
||||
|
||||
order = []
|
||||
|
||||
def bind(port):
|
||||
order.append("bind")
|
||||
raise RuntimeError("stop before serving")
|
||||
|
||||
with (
|
||||
mock.patch.object(
|
||||
server, "pin_mmap_threshold", side_effect=lambda: order.append("pin")
|
||||
),
|
||||
mock.patch.object(server, "bind_listener", side_effect=bind),
|
||||
):
|
||||
with self.assertRaises(RuntimeError):
|
||||
server.main(["--port", "0"])
|
||||
self.assertEqual(order, ["pin", "bind"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,526 @@
|
||||
"""#443 (D152): delete selected Library prompts; paired ones are archived.
|
||||
|
||||
The DB commands, and what every reader does with an archived parent.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from database.execution_history_schema import rotate_history_epoch
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.library_folders import folder_prefixes
|
||||
from database.operations import PromptDatabase
|
||||
from database.prompt_archive import TABLE as ARCHIVE
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
|
||||
def save(db, text, **metadata):
|
||||
return db.save_prompt(text, prompt_hash=generate_prompt_hash(text), **metadata)
|
||||
|
||||
|
||||
def rows(db, sql, params=()):
|
||||
with db.model.get_connection() as conn:
|
||||
return [tuple(row) for row in conn.execute(sql, params).fetchall()]
|
||||
|
||||
|
||||
def archived_ids(db):
|
||||
return sorted(row[0] for row in rows(db, f"SELECT prompt_id FROM {ARCHIVE}"))
|
||||
|
||||
|
||||
def prompt_ids(db):
|
||||
return sorted(row[0] for row in rows(db, "SELECT id FROM prompts"))
|
||||
|
||||
|
||||
_STAMP = re.compile(r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}")
|
||||
|
||||
|
||||
def _stable(value):
|
||||
return "<time>" if isinstance(value, str) and _STAMP.match(value) else value
|
||||
|
||||
|
||||
def table_dump(db):
|
||||
"""Every row of every table (times masked), to compare two databases."""
|
||||
tables = [
|
||||
row[0]
|
||||
for row in rows(
|
||||
db,
|
||||
"SELECT name FROM sqlite_master WHERE type='table' "
|
||||
"AND name NOT LIKE 'sqlite_%' AND name!='execution_history_metadata' "
|
||||
"ORDER BY name",
|
||||
)
|
||||
]
|
||||
dump = {}
|
||||
for table in tables:
|
||||
dump[table] = sorted(
|
||||
(tuple(map(_stable, row)) for row in rows(db, f"SELECT * FROM {table}")),
|
||||
key=repr,
|
||||
)
|
||||
return dump
|
||||
|
||||
|
||||
class LibraryDeleteCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
# Cleanups run last-in first-out and after every cleanup a test adds,
|
||||
# so a database a test opens (and closes via addCleanup) is closed
|
||||
# before the folder goes; Windows cannot delete an open file (#510).
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
self.db = PromptDatabase(str(self.root / "delete.db"))
|
||||
self.addCleanup(self.db.close_all)
|
||||
self.epoch = self.db.get_library_options()["library_epoch"]
|
||||
|
||||
def paired(self, positive, negative="blur"):
|
||||
pair = self.db.save_prompt_pair(positive, negative)
|
||||
found = self.db.get_prompt_pair(pair)
|
||||
return found["positive_prompt_id"], found["negative_prompt_id"], pair
|
||||
|
||||
def image(self, prompt_id, name):
|
||||
path = self.root / name
|
||||
path.write_bytes(b"png")
|
||||
return self.db.link_image_to_prompt(prompt_id, str(path))
|
||||
|
||||
def apply(self, ids, epoch=None, review=None):
|
||||
"""Review ``ids`` (or use ``review``), then write what was reviewed."""
|
||||
review = review or self.classify(ids)
|
||||
return self.db.archive_or_delete_library_prompts(
|
||||
review["archive"],
|
||||
review["delete"],
|
||||
expected_epoch=epoch or self.epoch,
|
||||
candidates=review["fingerprints"],
|
||||
)
|
||||
|
||||
def classify(self, ids, epoch=None):
|
||||
return self.db.classify_library_delete(ids, expected_epoch=epoch or self.epoch)
|
||||
|
||||
def restore(self, ids, epoch=None):
|
||||
return self.db.restore_library_prompts(ids, expected_epoch=epoch or self.epoch)
|
||||
|
||||
|
||||
class TestClassifyAndApply(LibraryDeleteCase):
|
||||
def test_each_class_in_order(self):
|
||||
plain = save(self.db, "Plain")
|
||||
protected = save(self.db, "Guarded", tags=["__protected__"])
|
||||
protected_paired, _neg, _pair = self.paired("Guarded pair")
|
||||
self.db.bulk_add_tags([protected_paired], ["__protected__"])
|
||||
positive, negative_only, _pair = self.paired("Cat", "only negative")
|
||||
both = save(self.db, "Both roles")
|
||||
self.image(both, "both.png") # a parent through its image
|
||||
self.db.save_prompt_pair("Other", "Both roles")
|
||||
archived_pair_positive, _n, old_pair = self.paired("Old")
|
||||
self.db.archive_prompt_pair(old_pair, True)
|
||||
already, _n2, _p2 = self.paired("Already")
|
||||
self.apply([already])
|
||||
self.image(plain, "plain.png")
|
||||
ids = [
|
||||
plain,
|
||||
protected,
|
||||
protected_paired,
|
||||
positive,
|
||||
negative_only,
|
||||
both,
|
||||
archived_pair_positive,
|
||||
already,
|
||||
99999,
|
||||
]
|
||||
result = self.classify(ids)
|
||||
self.assertEqual(
|
||||
result,
|
||||
{
|
||||
"library_epoch": self.epoch,
|
||||
"delete": [plain],
|
||||
"archive": [positive, both, archived_pair_positive],
|
||||
"protected": [protected, protected_paired],
|
||||
"already_archived": [already],
|
||||
"not_found": [negative_only, 99999],
|
||||
"image_links": 1,
|
||||
"fingerprints": result["fingerprints"],
|
||||
},
|
||||
)
|
||||
self.assertEqual(list(result["fingerprints"]), [plain])
|
||||
|
||||
def test_apply_archives_paired_deletes_unpaired_and_skips_protected(self):
|
||||
plain = save(self.db, "Plain", tags=["red"])
|
||||
protected = save(self.db, "Guarded", tags=["__protected__"])
|
||||
positive, negative, pair = self.paired("Cat")
|
||||
self.image(plain, "a.png")
|
||||
self.image(positive, "b.png")
|
||||
before = rows(self.db, "SELECT updated_at FROM prompts WHERE id=?", (positive,))
|
||||
result = self.apply([plain, protected, positive, 99999])
|
||||
self.assertEqual(
|
||||
result,
|
||||
{
|
||||
"library_epoch": self.epoch,
|
||||
"deleted": 1,
|
||||
"archived": 1,
|
||||
"already_archived": 0,
|
||||
"not_found": 0,
|
||||
"changed": 0,
|
||||
"image_links": 1,
|
||||
},
|
||||
)
|
||||
self.assertEqual(prompt_ids(self.db), sorted([protected, positive, negative]))
|
||||
self.assertEqual(archived_ids(self.db), [positive])
|
||||
# Archive keeps everything: the pair, its image link and the prompt row.
|
||||
self.assertEqual(
|
||||
len(rows(self.db, "SELECT 1 FROM prompt_pairs WHERE id=?", (pair,))), 1
|
||||
)
|
||||
self.assertEqual(
|
||||
len(
|
||||
rows(
|
||||
self.db,
|
||||
"SELECT 1 FROM generated_images WHERE prompt_id=?",
|
||||
(positive,),
|
||||
)
|
||||
),
|
||||
1,
|
||||
)
|
||||
after = rows(self.db, "SELECT updated_at FROM prompts WHERE id=?", (positive,))
|
||||
self.assertNotEqual(before, after)
|
||||
# The image file stays on disk.
|
||||
self.assertTrue((self.root / "a.png").exists())
|
||||
|
||||
def test_delete_has_exactly_the_legacy_row_effects(self):
|
||||
twin = PromptDatabase(str(self.root / "twin.db"))
|
||||
self.addCleanup(twin.close_all)
|
||||
twin_epoch = twin.get_library_options()["library_epoch"]
|
||||
for db in (self.db, twin):
|
||||
a = save(db, "Alpha", tags=["x", "y"], category="c")
|
||||
save(db, "Beta")
|
||||
path = self.root / "shared.png"
|
||||
path.write_bytes(b"png")
|
||||
db.link_image_to_prompt(a, str(path), {"workflow": {"nodes": []}})
|
||||
self.db.bulk_delete_prompts([a])
|
||||
twin.archive_or_delete_library_prompts(
|
||||
[], [a], expected_epoch=twin_epoch, candidates=None
|
||||
)
|
||||
self.assertEqual(table_dump(self.db), table_dump(twin))
|
||||
|
||||
def test_already_archived_and_restore(self):
|
||||
positive, _neg, _pair = self.paired("Cat")
|
||||
self.apply([positive])
|
||||
again = self.classify([positive])
|
||||
self.assertEqual(
|
||||
(again["archive"], again["already_archived"]), ([], [positive])
|
||||
)
|
||||
# Archived between the review and the write: counted, not archived twice.
|
||||
other, _n, _p = self.paired("Dog")
|
||||
review = self.classify([other])
|
||||
self.apply([other])
|
||||
late = self.apply([], review=review)
|
||||
self.assertEqual((late["archived"], late["already_archived"]), (0, 1))
|
||||
plain = save(self.db, "Plain")
|
||||
result = self.restore([positive, plain, 99999])
|
||||
self.assertEqual(
|
||||
result,
|
||||
{
|
||||
"library_epoch": self.epoch,
|
||||
"restored": 1,
|
||||
"unchanged": 1,
|
||||
"not_found": 1,
|
||||
},
|
||||
)
|
||||
self.assertEqual(archived_ids(self.db), [other])
|
||||
|
||||
def test_the_write_executes_only_what_was_reviewed(self):
|
||||
plain = save(self.db, "Plain")
|
||||
guarded = save(self.db, "Guarded later")
|
||||
touched = save(self.db, "Touched later")
|
||||
review = self.classify([plain, guarded, touched])
|
||||
# Between the review and the write: one becomes protected, one is used.
|
||||
self.db.bulk_add_tags([guarded], ["__protected__"])
|
||||
self.db.record_prompt_use(touched)
|
||||
result = self.apply([], review=review)
|
||||
self.assertEqual((result["deleted"], result["changed"]), (1, 2))
|
||||
self.assertEqual(prompt_ids(self.db), sorted([guarded, touched]))
|
||||
self.assertEqual(archived_ids(self.db), [])
|
||||
|
||||
def test_a_new_image_link_after_the_review_keeps_the_prompt(self):
|
||||
plain = save(self.db, "Plain")
|
||||
review = self.classify([plain])
|
||||
self.image(plain, "late.png")
|
||||
self.assertEqual(self.apply([], review=review)["changed"], 1)
|
||||
self.assertIn(plain, prompt_ids(self.db))
|
||||
|
||||
def test_reviewed_deletes_without_a_backup_are_kept(self):
|
||||
plain = save(self.db, "Plain")
|
||||
review = self.classify([plain])
|
||||
result = self.db.archive_or_delete_library_prompts(
|
||||
[], [plain], expected_epoch=self.epoch, candidates={}
|
||||
)
|
||||
self.assertEqual((result["deleted"], result["changed"]), (0, 1))
|
||||
self.assertEqual(review["delete"], [plain])
|
||||
|
||||
def test_a_reviewed_delete_that_became_paired_is_archived(self):
|
||||
plain = save(self.db, "Plain")
|
||||
review = self.classify([plain])
|
||||
self.db.save_prompt_pair("Plain", "late negative")
|
||||
result = self.apply([], review=review)
|
||||
self.assertEqual((result["deleted"], result["archived"]), (0, 1))
|
||||
|
||||
def test_a_reviewed_archive_that_lost_its_pair_is_archived_not_deleted(self):
|
||||
positive, _negative, pair = self.paired("Cat")
|
||||
review = self.classify([positive])
|
||||
self.db.delete_prompt_pair(pair)
|
||||
result = self.apply([], review=review)
|
||||
self.assertEqual((result["deleted"], result["archived"]), (0, 1))
|
||||
self.assertEqual(archived_ids(self.db), [positive])
|
||||
|
||||
def test_one_id_in_both_lists_or_no_ids_is_refused(self):
|
||||
plain = save(self.db, "Plain")
|
||||
for archive, delete in (([plain], [plain]), ([], [])):
|
||||
with self.subTest(archive=archive, delete=delete):
|
||||
with self.assertRaises(ValueError):
|
||||
self.db.archive_or_delete_library_prompts(
|
||||
archive, delete, expected_epoch=self.epoch, candidates=None
|
||||
)
|
||||
|
||||
def test_archiving_repairs_a_missing_or_stale_hash(self):
|
||||
missing, _n1, _p1 = self.paired("Hashless")
|
||||
stale, _n2, _p2 = self.paired("Stale hash")
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute("UPDATE prompts SET hash=NULL WHERE id=?", (missing,))
|
||||
conn.execute("UPDATE prompts SET hash='stale' WHERE id=?", (stale,))
|
||||
self.apply([missing, stale])
|
||||
for pid, text in ((missing, "Hashless"), (stale, "Stale hash")):
|
||||
self.assertEqual(
|
||||
rows(self.db, "SELECT hash FROM prompts WHERE id=?", (pid,)),
|
||||
[(generate_prompt_hash(text),)],
|
||||
)
|
||||
# So every ingestion lookup finds the archived row.
|
||||
self.assertEqual(save(self.db, text), pid)
|
||||
|
||||
def test_a_canonical_hash_another_row_holds_is_left_alone(self):
|
||||
stale, _n, _p = self.paired("Twin text")
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute("UPDATE prompts SET hash='stale' WHERE id=?", (stale,))
|
||||
twin = save(self.db, "twin text")
|
||||
self.apply([stale])
|
||||
self.assertEqual(
|
||||
rows(self.db, "SELECT hash FROM prompts WHERE id=?", (stale,)), [("stale",)]
|
||||
)
|
||||
self.assertEqual(save(self.db, "Twin text"), twin)
|
||||
|
||||
def test_restore_unhides_protected_and_non_parent_archived_prompts(self):
|
||||
guarded, _n, _p = self.paired("Guarded")
|
||||
self.apply([guarded])
|
||||
self.db.bulk_add_tags([guarded], ["__protected__"])
|
||||
negative_only = save(self.db, "Only a negative")
|
||||
self.db.save_prompt_pair("Owner", "Only a negative")
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
f"INSERT INTO {ARCHIVE} VALUES (?, '2026-10-28T00:00:00+00:00')",
|
||||
(negative_only,),
|
||||
)
|
||||
result = self.restore([guarded, negative_only])
|
||||
self.assertEqual(result["restored"], 2)
|
||||
self.assertEqual(archived_ids(self.db), [])
|
||||
|
||||
def test_restored_storage_changes_nothing(self):
|
||||
plain = save(self.db, "Plain")
|
||||
positive, _neg, _pair = self.paired("Cat")
|
||||
rotate_history_epoch(self.db.model.get_connection())
|
||||
for call in (
|
||||
lambda: self.apply([plain, positive]),
|
||||
lambda: self.classify([plain]),
|
||||
lambda: self.restore([positive]),
|
||||
):
|
||||
with self.assertRaises(LibraryCommandError) as caught:
|
||||
call()
|
||||
self.assertEqual(caught.exception.code, "storage-changed")
|
||||
self.assertIn(plain, prompt_ids(self.db))
|
||||
self.assertEqual(archived_ids(self.db), [])
|
||||
|
||||
def test_a_failure_mid_write_rolls_everything_back(self):
|
||||
plain = save(self.db, "Plain")
|
||||
positive, _neg, _pair = self.paired("Cat")
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"CREATE TRIGGER fail_delete BEFORE DELETE ON prompts "
|
||||
"BEGIN SELECT RAISE(ABORT, 'boom'); END"
|
||||
)
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
self.apply([positive, plain])
|
||||
self.assertIn(plain, prompt_ids(self.db))
|
||||
self.assertEqual(archived_ids(self.db), [])
|
||||
|
||||
def test_many_ids_are_handled_in_batches(self):
|
||||
ids = [save(self.db, f"Many {i}") for i in range(1000)]
|
||||
result = self.apply(ids)
|
||||
self.assertEqual(result["deleted"], 1000)
|
||||
|
||||
def test_rejects_invalid_arguments(self):
|
||||
plain = save(self.db, "Plain")
|
||||
for ids, epoch in (([], self.epoch), ([True], self.epoch), ([plain], "x")):
|
||||
for call in (self.apply, self.classify, self.restore):
|
||||
with self.subTest(ids=ids, epoch=epoch, call=call.__name__):
|
||||
with self.assertRaises(ValueError):
|
||||
call(ids, epoch)
|
||||
|
||||
|
||||
class TestArchivedReaders(LibraryDeleteCase):
|
||||
"""What each reader does with an archived parent (design §4.4)."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.kept, _kn, _kp = self.paired("Kept cat", "kept blur")
|
||||
self.gone, _gn, _gp = self.paired("Gone cat", "gone blur")
|
||||
for pid, category, tag in ((self.kept, "keep", "k"), (self.gone, "gone", "g")):
|
||||
self.db.update_prompt_metadata(pid, category=category, tags=[tag])
|
||||
self.apply([self.gone])
|
||||
|
||||
def ids_of(self, page):
|
||||
return [item["id"] for item in page["items"]]
|
||||
|
||||
def test_listing_search_and_totals(self):
|
||||
active = self.db.list_library()
|
||||
self.assertEqual(self.ids_of(active), [self.kept])
|
||||
self.assertEqual(
|
||||
(active["total"], active["variant_total"], active["archived_total"]),
|
||||
(1, 1, 1),
|
||||
)
|
||||
archived = self.db.list_library(shelf="archived")
|
||||
self.assertEqual(self.ids_of(archived), [self.gone])
|
||||
self.assertEqual((archived["total"], archived["variant_total"]), (1, 1))
|
||||
self.assertEqual(self.ids_of(self.db.list_library(text="Gone")), [])
|
||||
self.assertEqual(
|
||||
self.ids_of(self.db.list_library(text="Gone", shelf="archived")),
|
||||
[self.gone],
|
||||
)
|
||||
self.assertEqual(self.ids_of(self.db.list_library(tags=["g"])), [])
|
||||
self.assertEqual(self.db.list_library(category="gone")["archived_total"], 1)
|
||||
with self.assertRaises(ValueError):
|
||||
self.db.list_library(shelf="all")
|
||||
|
||||
def test_the_444_filters_and_tag_counts_hide_it(self):
|
||||
# #444's folder filter, tag match and tag ranking keep #443's shelves.
|
||||
bare, _bn, _bp = self.paired("Bare cat", "bare blur")
|
||||
self.apply([bare])
|
||||
folder = self.root / "out" / "cats"
|
||||
folder.mkdir(parents=True)
|
||||
for pid, name in ((self.kept, "kept.png"), (self.gone, "gone.png")):
|
||||
self.image(pid, str(Path("out") / "cats" / name))
|
||||
prefixes = folder_prefixes([str(self.root / "out")], "cats")
|
||||
self.assertEqual(
|
||||
self.ids_of(self.db.list_library(folder_prefixes=prefixes)), [self.kept]
|
||||
)
|
||||
self.assertEqual(
|
||||
self.ids_of(
|
||||
self.db.list_library(folder_prefixes=prefixes, shelf="archived")
|
||||
),
|
||||
[self.gone],
|
||||
)
|
||||
self.assertEqual(
|
||||
self.ids_of(self.db.list_library(tags=["k", "g"], tag_mode="any")),
|
||||
[self.kept],
|
||||
)
|
||||
self.assertEqual(self.ids_of(self.db.list_library(tag_mode="untagged")), [])
|
||||
self.assertEqual(
|
||||
self.ids_of(self.db.list_library(tag_mode="untagged", shelf="archived")),
|
||||
[bare],
|
||||
)
|
||||
tags = self.db.list_vocabulary_tags(sort="prompts")
|
||||
self.assertEqual(
|
||||
{row["name"]: row["prompt_count"] for row in tags["tags"]},
|
||||
{"k": 1, "g": 0},
|
||||
)
|
||||
self.assertEqual(tags["untagged_count"], 0)
|
||||
self.assertEqual(
|
||||
self.db.list_vocabulary_tags(search="g", sort="prompts")["tags"][0][
|
||||
"prompt_count"
|
||||
],
|
||||
0,
|
||||
)
|
||||
|
||||
def test_detail_and_options(self):
|
||||
self.assertTrue(self.db.get_library_positive(self.gone)["positive"]["archived"])
|
||||
self.assertFalse(
|
||||
self.db.get_library_positive(self.kept)["positive"]["archived"]
|
||||
)
|
||||
# Suggestions keep archived parents' categories: the archived view
|
||||
# filters by them.
|
||||
self.assertEqual(self.db.get_library_options()["categories"], ["gone", "keep"])
|
||||
|
||||
def test_counts(self):
|
||||
self.assertEqual(self.db.library_counts()["positive_prompts"], 1)
|
||||
self.assertEqual(self.db.library_counts()["negative_variants"], 1)
|
||||
tags = {
|
||||
row["name"]: row["prompt_count"]
|
||||
for row in self.db.list_vocabulary_tags()["tags"]
|
||||
}
|
||||
self.assertEqual((tags["k"], tags["g"]), (1, 0))
|
||||
categories = {
|
||||
row["value"]: row["prompt_count"]
|
||||
for row in self.db.list_vocabulary_categories()["categories"]
|
||||
}
|
||||
# A value only archived parents hold stays listed with 0, as a tag does.
|
||||
self.assertEqual(categories, {"keep": 1, "gone": 0})
|
||||
|
||||
def test_restore_summary_and_export(self):
|
||||
from database.restore_stage import library_summary
|
||||
|
||||
summary = library_summary(self.db.model.get_connection())
|
||||
self.assertEqual(summary["positives"], 1)
|
||||
exported = {
|
||||
item["positive_text"]: item["archived"]
|
||||
for item in self.db.export_library()["positives"]
|
||||
}
|
||||
self.assertEqual(exported, {"Kept cat": False, "Gone cat": True})
|
||||
|
||||
def test_usage_statistics_keep_archived_runs(self):
|
||||
self.db.record_prompt_use(self.gone, 3)
|
||||
runs = {row["category"]: row["runs"] for row in self.db.usage_by_category()}
|
||||
self.assertEqual(runs["gone"], 3)
|
||||
|
||||
def test_runs_node_saves_and_new_prompt_find_the_archived_row(self):
|
||||
count = len(prompt_ids(self.db))
|
||||
self.assertTrue(self.db.record_prompt_use(self.gone, 2))
|
||||
existing = self.db.get_prompt_by_hash(generate_prompt_hash("Gone cat"))
|
||||
self.assertEqual(existing["id"], self.gone)
|
||||
self.assertEqual(save(self.db, "Gone cat"), self.gone)
|
||||
self.db.save_prompt_pair("Gone cat", "new blur")
|
||||
self.assertEqual(len(prompt_ids(self.db)), count + 1) # only the new negative
|
||||
self.assertEqual(archived_ids(self.db), [self.gone])
|
||||
|
||||
def test_legacy_reads_still_list_it_and_legacy_delete_still_refuses(self):
|
||||
from database.pairs import PairDependencyError
|
||||
|
||||
texts = [p["text"] for p in self.db.get_recent_prompts(limit=50)["prompts"]]
|
||||
self.assertIn("Gone cat", texts)
|
||||
with self.assertRaises(PairDependencyError):
|
||||
self.db.bulk_delete_prompts([self.gone])
|
||||
|
||||
def test_as_another_parents_negative_it_stays_a_visible_variant(self):
|
||||
owner, _neg, pair = self.paired("Owner cat", "Gone cat")
|
||||
detail = self.db.get_library_positive(owner)
|
||||
self.assertEqual([v["id"] for v in detail["variants"]], [pair])
|
||||
self.assertEqual(self.ids_of(self.db.list_library(text="Gone")), [owner])
|
||||
|
||||
def test_maintenance_never_prunes_an_archived_prompt(self):
|
||||
lonely = save(self.db, "Lonely")
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
f"INSERT INTO {ARCHIVE} VALUES (?, '2026-10-28T00:00:00+00:00')",
|
||||
(lonely,),
|
||||
)
|
||||
orphaned = [row[0] for row in conn.execute(self.orphaned_sql()).fetchall()]
|
||||
self.assertNotIn(lonely, orphaned)
|
||||
|
||||
@staticmethod
|
||||
def orphaned_sql():
|
||||
from database.maintenance import ORPHANED_PROMPTS_SQL
|
||||
|
||||
return ORPHANED_PROMPTS_SQL
|
||||
|
||||
def test_bulk_metadata_writes_still_reach_an_archived_parent(self):
|
||||
result = self.db.bulk_set_library_category(
|
||||
[self.gone], "moved", expected_epoch=self.epoch
|
||||
)
|
||||
self.assertEqual(result["updated"], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -15,6 +15,7 @@ from database import execution_contexts
|
||||
from database.execution_history import HistoryEpochChanged
|
||||
from database.execution_context_schema import context_schema_error
|
||||
from database.execution_contexts import LIVE_HINT_PROMPT_IDS_SQL
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from database.execution_history_schema import rotate_history_epoch
|
||||
from utils.execution_contexts import (
|
||||
ContextBundle,
|
||||
@@ -466,7 +467,11 @@ class ContextDatabaseTests(unittest.TestCase):
|
||||
with conn:
|
||||
conn.execute("DELETE FROM prompts WHERE id=3")
|
||||
self.assertEqual(assert_same(), {1, 2, 4})
|
||||
# A case-only edit keeps the prompt's hash, so the run still counts
|
||||
# toward it (#460); another text drops the hint.
|
||||
db.update_prompt_text(4, "Other")
|
||||
self.assertEqual(assert_same(), {1, 2, 4})
|
||||
db.update_prompt_text(4, "another")
|
||||
self.assertEqual(assert_same(), {1, 2})
|
||||
with conn:
|
||||
conn.execute("INSERT INTO prompts(id,text) VALUES(5,'none')")
|
||||
@@ -522,6 +527,83 @@ class ContextDatabaseTests(unittest.TestCase):
|
||||
live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)}
|
||||
self.assertEqual(live, {1, 2})
|
||||
|
||||
def test_hints_follow_the_hash_for_case_variant_raw_text(self):
|
||||
# #460: the run names the prompt the Library counted it toward.
|
||||
db = self.database()
|
||||
first = db.save_prompt(
|
||||
"\u00e9t\u00e9 positive",
|
||||
prompt_hash=generate_prompt_hash("\u00e9t\u00e9 positive"),
|
||||
)
|
||||
second = db.save_prompt(
|
||||
"negative \u03c3", prompt_hash=generate_prompt_hash("negative \u03c3")
|
||||
)
|
||||
with db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) "
|
||||
"VALUES(7,?,?)",
|
||||
(first, second),
|
||||
)
|
||||
start = self.capture(
|
||||
db, 0, bundle("NEGATIVE \u03a3", positive="\u00c9T\u00c9 Positive")
|
||||
)
|
||||
context = db.get_execution_contexts(start.run_id)["contexts"][0]
|
||||
self.assertEqual(
|
||||
(
|
||||
context["positive_prompt_id"],
|
||||
context["negative_prompt_id"],
|
||||
context["pair_id"],
|
||||
),
|
||||
(first, second, 7),
|
||||
)
|
||||
conn = db.model.get_connection()
|
||||
live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)}
|
||||
self.assertEqual(live, {first, second})
|
||||
# A prompt that only shares SQLite's ASCII lower() is not a match
|
||||
other = db.save_prompt("other", prompt_hash=generate_prompt_hash("other"))
|
||||
start = self.capture(db, 1, bundle("x", positive="\u00d6THER"))
|
||||
context = db.get_execution_contexts(start.run_id)["contexts"][0]
|
||||
self.assertIsNone(context["positive_prompt_id"])
|
||||
self.assertNotIn(
|
||||
other, {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)}
|
||||
)
|
||||
|
||||
def test_hash_is_only_a_fallback_so_a_text_hint_stays_unambiguous(self):
|
||||
# #460 review: a hash match must never make a #438 text hint ambiguous,
|
||||
# or its prompt would lose the recorded-run prune protection.
|
||||
db = self.database()
|
||||
conn = db.model.get_connection()
|
||||
with conn:
|
||||
# A legacy row without a hash, and a case variant with the hash
|
||||
conn.execute("INSERT INTO prompts(id,text) VALUES(1,'a lighthouse')")
|
||||
conn.execute(
|
||||
"INSERT INTO prompts(id,text,hash) VALUES(2,'A Lighthouse',?)",
|
||||
(generate_prompt_hash("a lighthouse"),),
|
||||
)
|
||||
exact = self.capture(db, 0, bundle("x", positive="a lighthouse"))
|
||||
variant = self.capture(db, 1, bundle("x", positive="A LIGHTHOUSE"))
|
||||
hints = [
|
||||
db.get_execution_contexts(start.run_id)["contexts"][0]["positive_prompt_id"]
|
||||
for start in (exact, variant)
|
||||
]
|
||||
self.assertEqual(hints, [1, 2])
|
||||
live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)}
|
||||
self.assertEqual(live, {1, 2})
|
||||
self.assertEqual(db.prune_orphaned_prompts(), 0)
|
||||
|
||||
def test_live_hint_prompt_lookups_use_indexes(self):
|
||||
db = self.database()
|
||||
conn = db.model.get_connection()
|
||||
plan = [
|
||||
row[3]
|
||||
for row in conn.execute("EXPLAIN QUERY PLAN " + LIVE_HINT_PROMPT_IDS_SQL)
|
||||
]
|
||||
prompt_steps = [
|
||||
step for step in plan if " p " in f" {step} " or "TABLE p" in step
|
||||
]
|
||||
self.assertTrue(prompt_steps, plan)
|
||||
self.assertEqual([step for step in plan if step.startswith("SCAN p")], [])
|
||||
self.assertEqual([step for step in plan if step.startswith("SCAN q")], [])
|
||||
|
||||
def test_hints_require_exact_unique_raw_text_and_never_refresh(self):
|
||||
db = self.database()
|
||||
conn = db.model.get_connection()
|
||||
|
||||
@@ -19,6 +19,7 @@ from unittest.mock import patch
|
||||
from tests.history_waits import drain_writer
|
||||
from database import PromptDatabase
|
||||
from utils import usage_tracking
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.execution_history_writer import HistoryWriter
|
||||
|
||||
try:
|
||||
@@ -918,6 +919,30 @@ class CachedRuntimeHistoryTests(HooksCase):
|
||||
self.assertIsNone(named_prompt(self.db, self.rows()[1]["run_id"]))
|
||||
|
||||
|
||||
class CachedRuntimeCaseHistoryTests(CachedRuntimeHistoryTests):
|
||||
"""Raw text that differs only in case joins the saved prompt's history (#460).
|
||||
|
||||
The hash folds case, so the run counts toward the lower-case prompt saved
|
||||
first; the run keeps its exact raw spelling.
|
||||
"""
|
||||
|
||||
TEXT = "\u00c9 LIGHTHOUSE the LLM wrote"
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.db.delete_prompt(self.prompt_id)
|
||||
lower = self.TEXT.lower()
|
||||
self.prompt_id = self.db.save_prompt(
|
||||
text=lower, prompt_hash=generate_prompt_hash(lower)
|
||||
)
|
||||
|
||||
def test_the_library_counts_the_case_variant_as_this_prompt(self):
|
||||
self.assertEqual(
|
||||
self.db.get_prompt_by_hash(generate_prompt_hash(self.TEXT))["id"],
|
||||
self.prompt_id,
|
||||
)
|
||||
|
||||
|
||||
class CachedRuntimeWhitespaceHistoryTests(CachedRuntimeHistoryTests):
|
||||
"""Raw text with outer whitespace joins the history of the prompt it saves (#438).
|
||||
|
||||
|
||||
@@ -17,6 +17,8 @@ from utils.hashing import (
|
||||
generate_prompt_hash,
|
||||
is_duplicate_prompt,
|
||||
STRIPPED_CODE_POINTS,
|
||||
prompt_key,
|
||||
register_prompt_functions,
|
||||
stored_prompt_text,
|
||||
stored_prompt_text_sql,
|
||||
)
|
||||
@@ -158,5 +160,56 @@ class TestStoredPromptText(unittest.TestCase):
|
||||
self.assertEqual(self.sql_strip(value), stored_prompt_text(value))
|
||||
|
||||
|
||||
class TestPromptKey(unittest.TestCase):
|
||||
"""History compares the hash's own key; SQL calls Python for it (#460)."""
|
||||
|
||||
CASES = (
|
||||
"A Lighthouse",
|
||||
" \u00c9t\u00c9\n",
|
||||
"\u03a3\u03a3",
|
||||
"\u0130stanbul",
|
||||
"\u1e9e",
|
||||
"\u212a",
|
||||
"ok",
|
||||
)
|
||||
|
||||
def test_key_is_the_hash_normalization(self):
|
||||
import hashlib
|
||||
|
||||
for value in self.CASES:
|
||||
with self.subTest(value=repr(value)):
|
||||
self.assertEqual(
|
||||
generate_prompt_hash(value),
|
||||
hashlib.sha256(prompt_key(value).encode("utf-8")).hexdigest(),
|
||||
)
|
||||
self.assertEqual(prompt_key(" A Lighthouse\n"), "a lighthouse")
|
||||
|
||||
def test_sql_functions_give_pythons_result_where_sql_lower_does_not(self):
|
||||
import sqlite3
|
||||
|
||||
conn = sqlite3.connect(":memory:")
|
||||
try:
|
||||
register_prompt_functions(conn)
|
||||
for value in self.CASES:
|
||||
with self.subTest(value=repr(value)):
|
||||
key, digest = conn.execute(
|
||||
"SELECT pm_prompt_key(?), pm_prompt_hash(?)", (value, value)
|
||||
).fetchone()
|
||||
self.assertEqual(key, prompt_key(value))
|
||||
self.assertEqual(digest, generate_prompt_hash(value))
|
||||
self.assertEqual(
|
||||
conn.execute(
|
||||
"SELECT pm_prompt_key(1), pm_prompt_hash(NULL)"
|
||||
).fetchone(),
|
||||
(None, None),
|
||||
)
|
||||
# SQLite's own lower() folds ASCII only
|
||||
self.assertNotEqual(
|
||||
conn.execute("SELECT lower(?)", ("\u00c9",)).fetchone()[0], "\u00e9"
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -7,6 +7,7 @@ from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from database import restore_stage
|
||||
from database.execution_history import HistoryEpochChanged, notify_history_restore
|
||||
from tests import test_execution_history_database as history_tests
|
||||
|
||||
@@ -41,7 +42,12 @@ class HistoryReadFenceTests(unittest.TestCase):
|
||||
generation = fence.begin()
|
||||
with reader.recorded_history_snapshot() as epoch:
|
||||
original = reader.get_history_run(start.run_id)
|
||||
with patch("database.execution_history.notify_history_restore"):
|
||||
# The open snapshot keeps the WAL from being truncated; do not
|
||||
# sit out the full 30 s checkpoint wait for it (#487).
|
||||
with (
|
||||
patch("database.execution_history.notify_history_restore"),
|
||||
patch.object(restore_stage, "CHECKPOINT_BUSY_MS", 50),
|
||||
):
|
||||
writer.model.restore_from_file(source)
|
||||
self.assertEqual(reader.get_execution_epoch(), epoch)
|
||||
self.assertEqual(original["run"]["storage_epoch"], epoch)
|
||||
|
||||
+101
-3
@@ -17,6 +17,7 @@ from utils.execution_contexts import (
|
||||
)
|
||||
from utils.execution_outputs import OutputBundle
|
||||
from tests.test_execution_context_database import bundle, snapshot
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from tests.test_execution_output_database import bundle as outputs
|
||||
from tests import test_execution_history_database as history_tests
|
||||
|
||||
@@ -33,6 +34,10 @@ class HistoryReadTests(unittest.TestCase):
|
||||
conn.execute("INSERT INTO prompts(id,text) VALUES(?,?)", (identity, text))
|
||||
return identity
|
||||
|
||||
def hashed(self, db, text):
|
||||
"""A prompt saved as the nodes save it: with its dedup hash."""
|
||||
return db.save_prompt(text, prompt_hash=generate_prompt_hash(text))
|
||||
|
||||
def capture(self, db, item=0, context=None, output=None, time=100):
|
||||
start = self.start(db, item, time)
|
||||
db.write_execution_start(start)
|
||||
@@ -151,9 +156,9 @@ class HistoryReadTests(unittest.TestCase):
|
||||
)
|
||||
self.require_reads(db)
|
||||
with db.recorded_history_snapshot():
|
||||
# Outer whitespace joins the stored form's scope (#438); case and
|
||||
# Unicode normalization stay distinct.
|
||||
expected = (4, 1, 4, 4, 1, 1, 2, 2)
|
||||
# Outer whitespace (#438) and case (#460) join one scope;
|
||||
# Unicode normalization stays distinct.
|
||||
expected = (5, 5, 5, 5, 1, 1, 2, 2)
|
||||
for i, count in enumerate(expected, 1):
|
||||
self.assertEqual(db.list_history_window(i)["recorded_run_count"], count)
|
||||
self.assertIsNone(db.list_history_window(100))
|
||||
@@ -395,6 +400,32 @@ class HistoryReadTests(unittest.TestCase):
|
||||
pass
|
||||
self.assertFalse(conn.in_transaction)
|
||||
|
||||
def test_a_deleted_negative_is_unavailable_and_a_negative_only_one_is_not(self):
|
||||
# #443: a Library delete can remove an unpaired prompt a run used as
|
||||
# its negative. The run's prompt ids are live hints, so a deleted one
|
||||
# reads unavailable; a negative-only prompt stays available.
|
||||
db = self.database()
|
||||
self.saved(db)
|
||||
self.saved(db, "negative", 2)
|
||||
start = self.capture(db)
|
||||
with db.recorded_history_snapshot():
|
||||
context = db.get_history_run(start.run_id)["contexts"][0]
|
||||
self.assertTrue(context["navigation"]["negative_available"])
|
||||
with db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) "
|
||||
"VALUES(9,1,2)"
|
||||
)
|
||||
with db.recorded_history_snapshot():
|
||||
context = db.get_history_run(start.run_id)["contexts"][0]
|
||||
self.assertTrue(context["navigation"]["negative_available"])
|
||||
with db.model.get_connection() as conn:
|
||||
conn.execute("DELETE FROM prompt_pairs WHERE id=9")
|
||||
conn.execute("DELETE FROM prompts WHERE id=2")
|
||||
with db.recorded_history_snapshot():
|
||||
context = db.get_history_run(start.run_id)["contexts"][0]
|
||||
self.assertFalse(context["navigation"]["negative_available"])
|
||||
|
||||
def test_detail_preserves_full_text_snapshots_hints_and_archive(self):
|
||||
db = self.database()
|
||||
self.saved(db)
|
||||
@@ -1047,3 +1078,70 @@ class HistoryReadTests(unittest.TestCase):
|
||||
with db.recorded_history_snapshot():
|
||||
found = db.get_newest_pair_run(positive, pair)
|
||||
self.assertEqual(found["run"]["run_id"], newest.run_id)
|
||||
|
||||
def test_case_variant_raw_text_joins_the_saved_prompt_scope(self):
|
||||
# #460: the hash folds case, so these runs count toward this prompt.
|
||||
db = self.database()
|
||||
positive = self.hashed(db, "\u00e9t\u00e9 lighthouse \u03c3")
|
||||
raws = (
|
||||
"\u00c9T\u00c9 Lighthouse \u03a3\n",
|
||||
"\u00e9t\u00e9 lighthouse \u03c3",
|
||||
" \u00c9t\u00e9 LIGHTHOUSE \u03a3",
|
||||
)
|
||||
starts = [
|
||||
self.capture(db, i, bundle(positive=raw), time=100 + i)
|
||||
for i, raw in enumerate(raws)
|
||||
]
|
||||
# Unicode normalization and inner text stay distinct
|
||||
self.capture(db, 7, bundle(positive="e\u0301te\u0301 lighthouse \u03c3"))
|
||||
self.capture(db, 8, bundle(positive="\u00e9t\u00e9 lighthouse \u03c3"))
|
||||
dotted = self.hashed(db, "\u0130stanbul")
|
||||
dotted_run = self.capture(db, 9, bundle(positive="\u0130STANBUL"), time=300)
|
||||
with db.recorded_history_snapshot():
|
||||
page = db.list_history_window(positive)
|
||||
details = [db.get_history_run(start.run_id) for start in starts]
|
||||
turkish = db.list_history_window(dotted)
|
||||
self.assertEqual(page["recorded_run_count"], 3)
|
||||
self.assertEqual(
|
||||
[r["run_id"] for r in page["items"]], [s.run_id for s in starts]
|
||||
)
|
||||
for raw, detail in zip(raws, details):
|
||||
context = detail["contexts"][0]
|
||||
self.assertEqual(context["positive"]["raw"], raw)
|
||||
self.assertEqual(
|
||||
context["positive_raw_text_id"], page["group"]["raw_text_id"]
|
||||
)
|
||||
self.assertEqual(context["positive_prompt_id"], positive)
|
||||
self.assertEqual([r["run_id"] for r in turkish["items"]], [dotted_run.run_id])
|
||||
|
||||
def test_newest_pair_run_follows_case_variant_raw_text(self):
|
||||
db = self.database()
|
||||
pair = db.save_prompt_pair("a lighthouse", "fog")
|
||||
positive = db.get_prompt_pair(pair)["positive_prompt_id"]
|
||||
newest = self.capture(db, 0, bundle("FOG", positive="A Lighthouse"), time=100)
|
||||
with db.recorded_history_snapshot():
|
||||
found = db.get_newest_pair_run(positive, pair)
|
||||
self.assertEqual(found["run"]["run_id"], newest.run_id)
|
||||
|
||||
def test_a_position_from_before_460_resumes_the_same_scope(self):
|
||||
db = self.database()
|
||||
positive = self.hashed(db, "A Lighthouse")
|
||||
for i in range(3):
|
||||
self.capture(db, i, bundle(positive="a lighthouse"), time=100 + i)
|
||||
with db.recorded_history_snapshot():
|
||||
first = db.list_history_window(positive, limit=2)
|
||||
position = first["next_position"]
|
||||
legacy = replace(
|
||||
position, raw_text_id=content_digest("text", ("A Lighthouse",))
|
||||
)
|
||||
rest = db.list_history_window(positive, limit=2, after=legacy)
|
||||
with self.assertRaises(ValueError):
|
||||
db.list_history_window(
|
||||
positive,
|
||||
after=replace(position, raw_text_id=content_digest("text", ("x",))),
|
||||
)
|
||||
self.assertEqual(first["recorded_run_count"], 3)
|
||||
self.assertEqual(len(rest["items"]), 1)
|
||||
self.assertEqual(
|
||||
rest["group"]["raw_text_id"], content_digest("text", ("a lighthouse",))
|
||||
)
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
"""Delete selected images: the review and the apply job (#480).
|
||||
|
||||
``POST /prompt_manager/images/delete/preview`` reviews both modes;
|
||||
``POST /prompt_manager/maintenance/apply {token, mode}`` removes the files'
|
||||
records (library mode, with a removal mark) or moves the files to the trash
|
||||
first (trash mode, the fake trash here). No path unlinks a media file.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests import fake_trash
|
||||
from tests.disk_waits import DISK_BOUND_SECONDS
|
||||
from tests.test_api_duplicate_files import DuplicateFilesTestCase
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
import fake_trash
|
||||
from disk_waits import DISK_BOUND_SECONDS
|
||||
from test_api_duplicate_files import DuplicateFilesTestCase
|
||||
|
||||
from py.api import image_delete
|
||||
from py.api.image_delete import file_check, lexical_problem
|
||||
|
||||
PREVIEW = "/prompt_manager/images/delete/preview"
|
||||
APPLY = "/prompt_manager/maintenance/apply"
|
||||
LISTING = "/prompt_manager/images/output"
|
||||
|
||||
|
||||
class ImageDeleteTestCase(DuplicateFilesTestCase):
|
||||
async def get_application(self):
|
||||
app = await super().get_application()
|
||||
self.fake = fake_trash.install(Path(self.tmpdir) / "fake-trash")
|
||||
self.addCleanup(fake_trash.uninstall)
|
||||
return app
|
||||
|
||||
def target(self, path, index=0):
|
||||
roots = self.api._get_all_output_dirs()
|
||||
return {
|
||||
"root_index": index,
|
||||
"root_scope": self.api._gallery_root_scope(roots[index], index),
|
||||
"path": path,
|
||||
}
|
||||
|
||||
async def review(self, paths, status=200):
|
||||
items = [p if isinstance(p, dict) else self.target(p) for p in paths]
|
||||
return await self.post(PREVIEW, {"items": items}, status)
|
||||
|
||||
async def delete(self, paths, mode):
|
||||
plan = await self.review(paths)
|
||||
body = await self.post(APPLY, {"token": plan["token"], "mode": mode}, 202)
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
return plan, job
|
||||
|
||||
async def listed(self):
|
||||
response = await self.client.get(LISTING, params={"mode": "workspace"})
|
||||
page = await response.json()
|
||||
return [image["path"] for image in page["images"]]
|
||||
|
||||
def link(self, name, text="a prompt"):
|
||||
prompt_id = self.save(text)
|
||||
return self.api.db.link_image_to_prompt(
|
||||
str(prompt_id), str(self.output_dir / name)
|
||||
)
|
||||
|
||||
def count(self, sql):
|
||||
conn = sqlite3.connect(self.api.db.model.db_path)
|
||||
try:
|
||||
return conn.execute(sql).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@staticmethod
|
||||
def details(job, group):
|
||||
return [row["detail"] for row in job.detail()["outcomes"][group]]
|
||||
|
||||
|
||||
class TestReview(ImageDeleteTestCase):
|
||||
async def test_review_counts_records_and_refuses_what_it_cannot_delete(self):
|
||||
self.write("a.png", b"aaa", mtime=1_000_000)
|
||||
self.write("b.png", b"bbb", mtime=2_000_000)
|
||||
self.link("a.png", "only a")
|
||||
self.fake.refuse[str(self.output_dir / "b.png")] = "no-trash"
|
||||
plan = await self.review(["a.png", "b.png", "missing.png"])
|
||||
self.assertEqual(plan["library"], {"count": 2})
|
||||
self.assertEqual(plan["trash"], {"count": 1})
|
||||
self.assertEqual(plan["refused"]["counts"], {"not-listed": 1})
|
||||
self.assertEqual(plan["trash_refused"]["counts"], {"no-trash": 1})
|
||||
self.assertEqual(plan["records"]["prompt_links"], 1)
|
||||
self.assertEqual(plan["orphaned_prompts"]["count"], 1)
|
||||
self.assertIn(plan["backup"]["policy"], ("on", "off"))
|
||||
self.assertNotIn(self.tmpdir, str(plan))
|
||||
# The review wrote nothing.
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 1)
|
||||
self.assertEqual(self.fake.moved, [])
|
||||
|
||||
async def test_body_shapes_and_the_cap(self):
|
||||
await self.post(PREVIEW, {}, 400)
|
||||
await self.post(PREVIEW, {"items": [], "mode": "trash"}, 400)
|
||||
too_many = [self.target(f"{n}.png") for n in range(501)]
|
||||
await self.post(PREVIEW, {"items": too_many}, 400)
|
||||
|
||||
async def test_mode_is_required_and_only_for_this_plan(self):
|
||||
self.write("a.png", b"aaa")
|
||||
self.link("a.png")
|
||||
plan = await self.review(["a.png"])
|
||||
# No explicit, exact mode: refused, never a default (least of all trash).
|
||||
for body in (
|
||||
{"token": plan["token"]},
|
||||
{"token": plan["token"], "mode": None},
|
||||
{"token": plan["token"], "mode": ""},
|
||||
{"token": plan["token"], "mode": "TRASH"},
|
||||
{"token": plan["token"], "mode": "delete"},
|
||||
{"token": plan["token"], "mode": ["trash"]},
|
||||
{"token": plan["token"], "mode": True},
|
||||
):
|
||||
with self.subTest(body=body):
|
||||
await self.post(APPLY, body, 400)
|
||||
self.assertTrue((self.output_dir / "a.png").exists())
|
||||
self.assertEqual(self.fake.moved, [])
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 1)
|
||||
# The refusals left the token usable: an explicit mode still applies.
|
||||
body = await self.post(APPLY, {"token": plan["token"], "mode": "library"}, 202)
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
self.assertEqual(job.scope["mode"], "library")
|
||||
self.assertTrue((self.output_dir / "a.png").exists())
|
||||
self.assertEqual(self.fake.moved, [])
|
||||
|
||||
async def test_mode_on_another_plan_is_refused(self):
|
||||
plan = await self.post(
|
||||
"/prompt_manager/maintenance/preview",
|
||||
{"operation": "missing_links", "options": {}},
|
||||
200,
|
||||
)
|
||||
await self.post(APPLY, {"token": plan["token"], "mode": "trash"}, 400)
|
||||
|
||||
|
||||
class TestLibraryMode(ImageDeleteTestCase):
|
||||
async def test_records_go_and_the_file_stays_hidden(self):
|
||||
self.write("a.png", b"aaa", mtime=1_000_000)
|
||||
self.write("b.png", b"bbb", mtime=2_000_000)
|
||||
self.link("a.png")
|
||||
self.assertEqual(sorted(await self.listed()), ["a.png", "b.png"])
|
||||
before = os.stat(self.output_dir / "a.png")
|
||||
_plan, job = await self.delete(["a.png"], "library")
|
||||
self.assertEqual(job.state, "finished", job.summary())
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 0)
|
||||
after = os.stat(self.output_dir / "a.png")
|
||||
self.assertEqual(
|
||||
(before.st_size, before.st_mtime_ns), (after.st_size, after.st_mtime_ns)
|
||||
)
|
||||
self.assertEqual(await self.listed(), ["b.png"])
|
||||
# A later scan or link never brings it back.
|
||||
self.assertEqual(self.link("a.png", "rescanned"), 0)
|
||||
self.assertEqual(
|
||||
self.api._removed_paths([self.output_dir / "a.png"]),
|
||||
{str(self.output_dir / "a.png")},
|
||||
)
|
||||
self.assertEqual(self.fake.moved, [])
|
||||
|
||||
async def test_a_modified_file_shows_again(self):
|
||||
self.write("a.png", b"aaa", mtime=1_000_000)
|
||||
await self.delete(["a.png"], "library")
|
||||
self.assertEqual(await self.listed(), [])
|
||||
self.write("a.png", b"new and longer", mtime=3_000_000)
|
||||
self.api.invalidate_gallery_cache()
|
||||
self.assertEqual(await self.listed(), ["a.png"])
|
||||
|
||||
async def test_a_safety_backup_is_taken_first(self):
|
||||
self.write("a.png", b"aaa")
|
||||
with patch.object(self.api, "_safety_backup_policy", return_value="on"):
|
||||
await self.delete(["a.png"], "library")
|
||||
self.assertEqual(len(self.backups()), 1)
|
||||
|
||||
|
||||
class TestTrashMode(ImageDeleteTestCase):
|
||||
async def test_the_file_moves_first_then_its_records_go(self):
|
||||
self.write("a.png", b"aaa")
|
||||
self.link("a.png")
|
||||
seen = {}
|
||||
original = self.fake.move
|
||||
|
||||
def move(path, roots):
|
||||
# The records still exist while the file moves.
|
||||
seen["links"] = self.count("SELECT COUNT(*) FROM generated_images")
|
||||
return original(path, roots)
|
||||
|
||||
self.fake.move = move
|
||||
_plan, job = await self.delete(["a.png"], "trash")
|
||||
self.assertEqual(job.state, "finished", job.summary())
|
||||
self.assertEqual(seen["links"], 1)
|
||||
self.assertFalse((self.output_dir / "a.png").exists())
|
||||
[(source, destination)] = self.fake.moved
|
||||
self.assertTrue(os.path.exists(destination))
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 0)
|
||||
self.assertEqual(self.api.db.removal_marks(), {})
|
||||
|
||||
async def test_a_refused_move_changes_nothing(self):
|
||||
self.write("a.png", b"aaa")
|
||||
self.link("a.png")
|
||||
plan = await self.review(["a.png"])
|
||||
self.fake.refuse[str(self.output_dir / "a.png")] = "locked"
|
||||
body = await self.post(APPLY, {"token": plan["token"], "mode": "trash"}, 202)
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
self.assertTrue((self.output_dir / "a.png").exists())
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 1)
|
||||
self.assertEqual(self.api.db.removal_marks(), {})
|
||||
self.assertEqual(job.counters["not_changed"], 1)
|
||||
self.assertEqual(
|
||||
self.details(job, "not_changed"), [image_delete.REASONS["locked"]]
|
||||
)
|
||||
|
||||
async def test_a_file_changed_after_the_review_is_not_changed(self):
|
||||
self.write("a.png", b"aaa", mtime=1_000_000)
|
||||
plan = await self.review(["a.png"])
|
||||
self.write("a.png", b"changed bytes", mtime=2_000_000)
|
||||
body = await self.post(APPLY, {"token": plan["token"], "mode": "trash"}, 202)
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
self.assertTrue((self.output_dir / "a.png").exists())
|
||||
self.assertEqual(self.fake.moved, [])
|
||||
self.assertEqual(
|
||||
self.details(job, "not_changed"), [image_delete.REASONS["changed"]]
|
||||
)
|
||||
|
||||
async def test_a_records_failure_stops_the_job(self):
|
||||
self.write("a.png", b"aaa", mtime=1_000_000)
|
||||
self.write("b.png", b"bbb", mtime=2_000_000)
|
||||
with patch.object(
|
||||
type(self.api.db),
|
||||
"remove_image_records",
|
||||
side_effect=sqlite3.OperationalError("locked"),
|
||||
):
|
||||
_plan, job = await self.delete(["a.png", "b.png"], "library")
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertEqual(job.scope.get("failure"), "records-failed")
|
||||
self.assertEqual(job.counters["errors"], 1)
|
||||
self.assertEqual(
|
||||
self.details(job, "not_changed"), [image_delete.REASONS["not-attempted"]]
|
||||
)
|
||||
|
||||
async def test_nothing_is_ever_unlinked(self):
|
||||
self.write("a.png", b"aaa")
|
||||
self.write("b.png", b"bbb")
|
||||
media = str(self.output_dir)
|
||||
real_remove, real_unlink = os.remove, os.unlink
|
||||
|
||||
def guard(real):
|
||||
def call(path, *args, **kwargs):
|
||||
text = os.path.realpath(str(path))
|
||||
if (
|
||||
text.startswith(os.path.realpath(media))
|
||||
and "thumbnails" not in text
|
||||
):
|
||||
raise AssertionError(f"unlinked a media file: {path}")
|
||||
return real(path, *args, **kwargs)
|
||||
|
||||
return call
|
||||
|
||||
with (
|
||||
patch("os.remove", guard(real_remove)),
|
||||
patch("os.unlink", guard(real_unlink)),
|
||||
):
|
||||
await self.delete(["a.png"], "trash")
|
||||
await self.delete(["b.png"], "library")
|
||||
self.assertTrue((self.output_dir / "b.png").exists())
|
||||
|
||||
|
||||
class TestFileCheck(ImageDeleteTestCase):
|
||||
def test_lexical_codes(self):
|
||||
self.assertEqual(lexical_problem("thumbnails/a.png"), "thumbnail")
|
||||
self.assertEqual(lexical_problem(".Trash-1000/files/a.png"), "trash-folder")
|
||||
self.assertEqual(lexical_problem("x/$RECYCLE.BIN/a.png"), "trash-folder")
|
||||
self.assertEqual(lexical_problem("a.txt"), "not-media")
|
||||
self.assertIsNone(lexical_problem(".Trashy/a.png"))
|
||||
self.assertIsNone(lexical_problem("sub/a.png"))
|
||||
|
||||
def test_links_and_folders_leading_outside_are_refused(self):
|
||||
root = str(self.output_dir)
|
||||
outside = Path(self.tmpdir) / "elsewhere"
|
||||
outside.mkdir()
|
||||
(outside / "x.png").write_bytes(b"x")
|
||||
try:
|
||||
os.symlink(outside / "x.png", self.output_dir / "link.png")
|
||||
os.symlink(outside, self.output_dir / "linked", target_is_directory=True)
|
||||
except OSError:
|
||||
self.skipTest("symlinks are not permitted here")
|
||||
self.assertEqual(
|
||||
file_check(str(self.output_dir / "link.png"), root, "link.png")[0], "link"
|
||||
)
|
||||
self.assertEqual(
|
||||
file_check(str(self.output_dir / "linked/x.png"), root, "linked/x.png")[0],
|
||||
"outside",
|
||||
)
|
||||
self.assertEqual(
|
||||
file_check(str(self.output_dir / "gone.png"), root, "gone.png")[0], "gone"
|
||||
)
|
||||
self.write("ok.png", b"ok")
|
||||
self.assertIsNone(
|
||||
file_check(str(self.output_dir / "ok.png"), root, "ok.png")[0]
|
||||
)
|
||||
|
||||
def test_windows_reason_says_why_plainly(self):
|
||||
self.assertEqual(
|
||||
image_delete.REASONS["windows-recycle"],
|
||||
"Moving files to the Recycle Bin isn't available on Windows: files "
|
||||
"larger than the bin's capacity would be deleted permanently.",
|
||||
)
|
||||
|
||||
def test_reasons_cover_every_code(self):
|
||||
from utils import trash
|
||||
|
||||
for code in trash.CODES:
|
||||
self.assertIn(code, image_delete.REASONS)
|
||||
|
||||
|
||||
class TestNativeBackend(ImageDeleteTestCase):
|
||||
"""The real backend of this OS: Windows and macOS have no trash yet (#480)."""
|
||||
|
||||
async def test_library_only_works_and_the_trash_is_unavailable_here(self):
|
||||
fake_trash.uninstall()
|
||||
self.write("a.png", b"aaa")
|
||||
self.link("a.png")
|
||||
plan = await self.review(["a.png"])
|
||||
native_trash = sys.platform not in ("win32", "darwin")
|
||||
if not native_trash:
|
||||
self.assertEqual(plan["trash"], {"count": 0})
|
||||
code = "windows-recycle" if sys.platform == "win32" else "no-trash"
|
||||
self.assertEqual(plan["trash_refused"]["counts"], {code: 1})
|
||||
body = await self.post(APPLY, {"token": plan["token"], "mode": "library"}, 202)
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
self.assertEqual(job.counters["removed"], 1, job.summary())
|
||||
self.assertTrue((self.output_dir / "a.png").exists())
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 0)
|
||||
self.assertEqual(await self.listed(), [])
|
||||
@@ -0,0 +1,319 @@
|
||||
"""Removing one file's records from the library, and removal marks (#480)."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.image_removal_schema import (
|
||||
image_removals_schema_error,
|
||||
migrate_image_removals,
|
||||
)
|
||||
from database.models import PromptModel
|
||||
from database.operations import PromptDatabase
|
||||
|
||||
|
||||
def _locator(path):
|
||||
return os.path.normcase(os.path.realpath(str(path)))
|
||||
|
||||
|
||||
class _RemovalCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.addCleanup(assert_closed, self, self.temp.name)
|
||||
self.addCleanup(PromptModel.reset_schema_cache)
|
||||
base = Path(self.temp.name)
|
||||
self.root = base / "output"
|
||||
self.root.mkdir()
|
||||
# A second spelling of the folder (a symlink); Windows without the
|
||||
# privilege has none, and the records then carry one spelling.
|
||||
self.alias = base / "alias"
|
||||
try:
|
||||
self.alias.symlink_to(self.root, target_is_directory=True)
|
||||
except OSError:
|
||||
self.alias = None
|
||||
self.db = PromptDatabase(str(base / "prompts.db"))
|
||||
self.addCleanup(self.db.close_all)
|
||||
self.epoch = self.db.get_execution_epoch()
|
||||
|
||||
def file(self, name, data=b"image-bytes"):
|
||||
path = self.root / name
|
||||
path.write_bytes(data)
|
||||
return path
|
||||
|
||||
def removal(self, path):
|
||||
root_key = os.path.normcase(os.path.realpath(str(self.root)))
|
||||
return {
|
||||
"spellings": [str(path)]
|
||||
+ ([str(self.alias / path.name)] if self.alias else []),
|
||||
"locator": _locator(path),
|
||||
"favorites": [(root_key, path.name.lower())],
|
||||
"category": (root_key, path.name.lower()),
|
||||
}
|
||||
|
||||
def link(self, path, text="a prompt", spelled=None):
|
||||
prompt_id = self.db.save_prompt(text)
|
||||
image_id = self.db.link_image_to_prompt(str(prompt_id), str(spelled or path))
|
||||
self.assertTrue(image_id)
|
||||
return prompt_id, image_id
|
||||
|
||||
def tag(self, path, tags):
|
||||
data = path.read_bytes()
|
||||
info = os.stat(path)
|
||||
return self.db.replace_image_tags(
|
||||
locator=_locator(path),
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
size=info.st_size,
|
||||
mtime_ns=info.st_mtime_ns,
|
||||
expected_epoch=self.epoch,
|
||||
expected_tag_revision=0,
|
||||
tags=tags,
|
||||
)
|
||||
|
||||
def queue(self, path):
|
||||
data = path.read_bytes()
|
||||
info = os.stat(path)
|
||||
return self.db.record_review_item(
|
||||
locator=_locator(path),
|
||||
spelled_path=os.path.realpath(str(path)),
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
size=info.st_size,
|
||||
mtime_ns=info.st_mtime_ns,
|
||||
epoch=self.epoch,
|
||||
resolver_version=1,
|
||||
evidence={"candidates": []},
|
||||
candidate_count=0,
|
||||
limited=True,
|
||||
)
|
||||
|
||||
def count(self, sql, *params):
|
||||
conn = sqlite3.connect(self.db.model.db_path)
|
||||
try:
|
||||
return conn.execute(sql, params).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def decorate(self, path):
|
||||
"""Give *path* one record of every kind."""
|
||||
removal = self.removal(path)
|
||||
root_key, relative = removal["favorites"][0]
|
||||
self.tag(path, ["portrait"])
|
||||
self.db.set_image_favorite(root_key, relative)
|
||||
self.db.set_image_categories([removal["category"]], "keepers")
|
||||
self.db.add_to_collection(items=[(root_key, relative)], name="Set")
|
||||
return removal
|
||||
|
||||
|
||||
class ImageRemovalRecordsTests(_RemovalCase):
|
||||
def test_counts_and_removes_every_record_under_every_spelling(self):
|
||||
path = self.file("a.png")
|
||||
other = self.file("b.png", b"other")
|
||||
orphan, _ = self.link(path, "only this image")
|
||||
shared, _ = self.link(path, "two images")
|
||||
self.db.link_image_to_prompt(str(shared), str(other))
|
||||
# A record under the symlinked folder spelling (or a third prompt).
|
||||
self.link(path, "aliased", spelled=(self.alias or self.root) / "a.png")
|
||||
removal = self.decorate(path)
|
||||
self.assertTrue(self.queue(path))
|
||||
|
||||
counted = self.db.count_removal_records([removal])
|
||||
self.assertEqual(
|
||||
counted["records"],
|
||||
{
|
||||
"prompt_links": 3,
|
||||
"pair_links": 0,
|
||||
"image_tags": 1,
|
||||
"auto_tagged": 0,
|
||||
"categories": 1,
|
||||
"collections": 1,
|
||||
"favorites": 1,
|
||||
"review_items": 1,
|
||||
},
|
||||
)
|
||||
self.assertEqual(counted["orphaned_prompts"]["count"], 2)
|
||||
|
||||
removed = self.db.remove_image_records(removal)
|
||||
self.assertEqual(removed["prompt_links"], 3)
|
||||
self.assertEqual(removed["review_items"], 1)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 1)
|
||||
for table in (
|
||||
"media_image_tags",
|
||||
"media_annotation_files",
|
||||
"media_review_items",
|
||||
"image_favorites",
|
||||
"image_categories",
|
||||
"image_collection_items",
|
||||
):
|
||||
with self.subTest(table=table):
|
||||
self.assertEqual(self.count(f"SELECT COUNT(*) FROM {table}"), 0)
|
||||
# Prompts and the collection itself stay.
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM prompts"), 3)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM image_collections"), 1)
|
||||
self.assertTrue(path.exists())
|
||||
self.assertIsNotNone(self.db.get_prompt_by_id(orphan))
|
||||
|
||||
def test_a_kept_review_item_keeps_the_annotation_rows(self):
|
||||
path = self.file("a.png")
|
||||
removal = self.decorate(path)
|
||||
self.assertTrue(self.queue(path))
|
||||
conn = sqlite3.connect(self.db.model.db_path)
|
||||
try:
|
||||
# A revised item is not settled; it still cites the bytes, whose
|
||||
# foreign key is ON DELETE RESTRICT.
|
||||
conn.execute("UPDATE media_review_items SET revision=1")
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
removed = self.db.remove_image_records(removal)
|
||||
self.assertEqual(removed["review_items"], 0)
|
||||
self.assertEqual(removed["image_tags"], 1)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM media_image_tags"), 0)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM media_review_items"), 1)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM media_annotation_files"), 1)
|
||||
|
||||
def test_mark_is_written_with_the_records_and_bumps_the_revision(self):
|
||||
path = self.file("a.png")
|
||||
self.link(path)
|
||||
before = self.db.removal_revision()
|
||||
info = os.stat(path)
|
||||
self.db.remove_image_records(
|
||||
self.removal(path), mark=(info.st_size, info.st_mtime_ns)
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.removal_marks(),
|
||||
{_locator(path): (info.st_size, info.st_mtime_ns)},
|
||||
)
|
||||
self.assertEqual(self.db.removal_revision(), before + 1)
|
||||
self.assertEqual(self.db.clear_removal_marks([_locator(path)]), 1)
|
||||
self.assertEqual(self.db.removal_marks(), {})
|
||||
self.assertEqual(self.db.removal_revision(), before + 2)
|
||||
self.assertEqual(self.db.clear_removal_marks([_locator(path)]), 0)
|
||||
|
||||
def test_a_failure_rolls_back_everything(self):
|
||||
path = self.file("a.png")
|
||||
self.link(path)
|
||||
removal = self.decorate(path)
|
||||
broken = {**removal, "category": ("only-one-value",)}
|
||||
with self.assertRaises(sqlite3.Error):
|
||||
self.db.remove_image_records(broken, mark=(1, 1))
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 1)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM media_image_tags"), 1)
|
||||
self.assertEqual(self.db.removal_marks(), {})
|
||||
|
||||
def test_parity_with_the_legacy_single_delete(self):
|
||||
path = self.file("a.png")
|
||||
_prompt, image_id = self.link(path)
|
||||
twin = PromptDatabase(str(Path(self.temp.name) / "twin.db"))
|
||||
try:
|
||||
twin_prompt = twin.save_prompt("a prompt")
|
||||
twin_image = twin.link_image_to_prompt(str(twin_prompt), str(path))
|
||||
self.assertTrue(twin.delete_image(twin_image))
|
||||
self.db.remove_image_records(self.removal(path))
|
||||
for table in (
|
||||
"generated_images",
|
||||
"image_prompt_pairs",
|
||||
"workflow_documents",
|
||||
):
|
||||
with self.subTest(table=table):
|
||||
conn = sqlite3.connect(twin.model.db_path)
|
||||
try:
|
||||
legacy = conn.execute(
|
||||
f"SELECT COUNT(*) FROM {table}"
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(
|
||||
self.count(f"SELECT COUNT(*) FROM {table}"), legacy
|
||||
)
|
||||
finally:
|
||||
twin.close_all()
|
||||
self.assertIsNotNone(image_id)
|
||||
|
||||
|
||||
class RemovalMarkGateTests(_RemovalCase):
|
||||
"""A marked file is never linked again; any record write shows it again."""
|
||||
|
||||
def hide(self, path):
|
||||
info = os.stat(path)
|
||||
self.db.remove_image_records(
|
||||
self.removal(path), mark=(info.st_size, info.st_mtime_ns)
|
||||
)
|
||||
|
||||
def test_a_marked_file_is_not_linked_again(self):
|
||||
path = self.file("a.png")
|
||||
self.hide(path)
|
||||
prompt_id = self.db.save_prompt("scanned again")
|
||||
self.assertEqual(self.db.link_image_to_prompt(str(prompt_id), str(path)), 0)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 0)
|
||||
self.assertIn(_locator(path), self.db.removal_marks())
|
||||
|
||||
def test_a_changed_file_drops_its_stale_mark_and_links(self):
|
||||
path = self.file("a.png")
|
||||
self.hide(path)
|
||||
path.write_bytes(b"new bytes, new size")
|
||||
prompt_id = self.db.save_prompt("a new image at the same path")
|
||||
self.assertTrue(self.db.link_image_to_prompt(str(prompt_id), str(path)))
|
||||
self.assertEqual(self.db.removal_marks(), {})
|
||||
|
||||
def test_explicit_record_writes_clear_the_mark(self):
|
||||
writers = {
|
||||
"favorite": lambda path, key: self.db.set_image_favorite(*key),
|
||||
"category": lambda path, key: self.db.set_image_categories([key], "c"),
|
||||
"collection": lambda path, key: self.db.add_to_collection(
|
||||
items=[key], name="Set"
|
||||
),
|
||||
"tags": lambda path, key: self.tag(path, ["t"]),
|
||||
}
|
||||
for kind, write in writers.items():
|
||||
with self.subTest(kind=kind):
|
||||
path = self.file(f"{kind}.png", kind.encode())
|
||||
self.hide(path)
|
||||
self.assertIn(_locator(path), self.db.removal_marks())
|
||||
write(path, self.removal(path)["favorites"][0])
|
||||
self.assertNotIn(_locator(path), self.db.removal_marks())
|
||||
|
||||
|
||||
class ImageRemovalSchemaTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.conn = sqlite3.connect(":memory:")
|
||||
self.addCleanup(self.conn.close)
|
||||
|
||||
def test_missing_is_allowed_only_for_old_backups(self):
|
||||
self.assertIsNone(image_removals_schema_error(self.conn))
|
||||
self.assertIsNotNone(
|
||||
image_removals_schema_error(self.conn, allow_missing=False)
|
||||
)
|
||||
|
||||
def test_install_is_valid_and_idempotent(self):
|
||||
migrate_image_removals(self.conn)
|
||||
migrate_image_removals(self.conn)
|
||||
self.assertIsNone(image_removals_schema_error(self.conn, allow_missing=False))
|
||||
self.assertEqual(
|
||||
self.conn.execute("SELECT revision FROM image_removals_meta").fetchall(),
|
||||
[(0,)],
|
||||
)
|
||||
|
||||
def test_invalid_values_and_objects_are_refused(self):
|
||||
migrate_image_removals(self.conn)
|
||||
self.conn.execute("INSERT INTO image_removals VALUES ('', 1, 1, 'now')")
|
||||
self.assertIn("invalid values", image_removals_schema_error(self.conn))
|
||||
self.conn.execute("DELETE FROM image_removals")
|
||||
self.conn.execute("CREATE INDEX extra ON image_removals(size)")
|
||||
self.assertIn("unsupported", image_removals_schema_error(self.conn))
|
||||
|
||||
def test_a_foreign_table_definition_is_refused(self):
|
||||
self.conn.execute("CREATE TABLE image_removals (locator TEXT)")
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
migrate_image_removals(self.conn)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,251 @@
|
||||
"""Removal-mark gaps after #480, and trash folders inside output roots (#501).
|
||||
|
||||
- Thumbnail jobs and the legacy auto-tag listing skip a library-removed file.
|
||||
- Missing-file links also forgets the marks of files that are gone (only under
|
||||
a source that can be read now).
|
||||
- Every walker skips a trash folder that sits inside an output root.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.disk_waits import DISK_BOUND_SECONDS
|
||||
from tests.test_image_delete_routes import ImageDeleteTestCase
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from disk_waits import DISK_BOUND_SECONDS
|
||||
from test_image_delete_routes import ImageDeleteTestCase
|
||||
|
||||
from py.api import images as images_module
|
||||
from py.api.admin import _collect_media_files
|
||||
from py.api.autotag_routes import AutotagRoutesMixin
|
||||
from utils.image_monitor import ImageGenerationHandler
|
||||
|
||||
MAINTENANCE_PREVIEW = "/prompt_manager/maintenance/preview"
|
||||
MAINTENANCE_APPLY = "/prompt_manager/maintenance/apply"
|
||||
|
||||
|
||||
class _GapCase(ImageDeleteTestCase):
|
||||
def marks(self):
|
||||
conn = sqlite3.connect(self.api.db.model.db_path)
|
||||
try:
|
||||
return [
|
||||
row[0] for row in conn.execute("SELECT locator FROM image_removals")
|
||||
]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def revision(self):
|
||||
conn = sqlite3.connect(self.api.db.model.db_path)
|
||||
try:
|
||||
return conn.execute(
|
||||
"SELECT revision FROM image_removals_meta WHERE singleton_id=1"
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class TestThumbnailJobsSkipHiddenFiles(_GapCase):
|
||||
async def test_missing_and_rebuild_plans_leave_a_removed_file_out(self):
|
||||
self.write("hidden.png", b"hidden bytes", mtime=1_000_000)
|
||||
self.write("shown.png", b"shown bytes", mtime=2_000_000)
|
||||
await self.delete(["hidden.png"], "library")
|
||||
thumbs = self.output_dir / "thumbnails"
|
||||
for mode in ("missing", "rebuild"):
|
||||
with self.subTest(mode=mode):
|
||||
plan = self.api._plan_thumbnails(self.output_dir, thumbs, mode=mode)
|
||||
sources = [src.name for src, _dst, _video in plan["targets"]]
|
||||
self.assertEqual(sources, ["shown.png"])
|
||||
self.assertEqual(plan["missing"], 1)
|
||||
|
||||
async def test_a_changed_file_gets_its_preview_again(self):
|
||||
self.write("hidden.png", b"hidden bytes", mtime=1_000_000)
|
||||
await self.delete(["hidden.png"], "library")
|
||||
self.write("hidden.png", b"new and longer bytes", mtime=3_000_000)
|
||||
plan = self.api._plan_thumbnails(
|
||||
self.output_dir, self.output_dir / "thumbnails"
|
||||
)
|
||||
self.assertEqual([t[0].name for t in plan["targets"]], ["hidden.png"])
|
||||
|
||||
|
||||
class TestLegacyAutotagListing(_GapCase):
|
||||
async def test_scan_output_dir_leaves_a_removed_file_out(self):
|
||||
self.write("hidden.png", b"hidden bytes", mtime=1_000_000)
|
||||
self.write("shown.png", b"shown bytes", mtime=2_000_000)
|
||||
await self.delete(["hidden.png"], "library")
|
||||
with patch.object(
|
||||
self.api, "_find_comfyui_output_dir", return_value=str(self.output_dir)
|
||||
):
|
||||
response = await self.client.get("/prompt_manager/scan_output_dir")
|
||||
body = await response.json()
|
||||
self.assertEqual([image["filename"] for image in body["images"]], ["shown.png"])
|
||||
|
||||
|
||||
class TestMissingFileLinksPrunesMarks(_GapCase):
|
||||
async def links_review(self, status=200):
|
||||
return await self.post(
|
||||
MAINTENANCE_PREVIEW, {"operation": "missing_links", "options": {}}, status
|
||||
)
|
||||
|
||||
async def apply(self, token, status=202):
|
||||
body = await self.post(MAINTENANCE_APPLY, {"token": token}, status)
|
||||
if status != 202:
|
||||
return body
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
return job
|
||||
|
||||
async def removed(self, name):
|
||||
self.write("other.png", b"another output") # the source lists something
|
||||
self.write(name, f"bytes of {name}".encode(), mtime=1_000_000)
|
||||
await self.delete([name], "library")
|
||||
self.assertEqual(len(self.marks()), 1)
|
||||
|
||||
async def test_the_mark_of_a_gone_file_is_counted_and_forgotten(self):
|
||||
await self.removed("gone.png")
|
||||
# Not an unlink by PromptManager: the user deleted it elsewhere.
|
||||
os.remove(self.output_dir / "gone.png")
|
||||
plan = await self.links_review()
|
||||
self.assertEqual(plan["scope"]["count"], 1)
|
||||
self.assertEqual(
|
||||
plan["scope"]["label"], "1 removed image whose file is gone or changed"
|
||||
)
|
||||
before = self.revision()
|
||||
job = await self.apply(plan["token"])
|
||||
self.assertEqual(job.state, "finished", job.summary())
|
||||
self.assertEqual(self.marks(), [])
|
||||
self.assertGreater(self.revision(), before)
|
||||
|
||||
async def test_the_mark_of_a_present_file_stays(self):
|
||||
await self.removed("kept.png")
|
||||
plan = await self.links_review()
|
||||
self.assertEqual(plan["scope"]["count"], 0)
|
||||
self.assertEqual(plan["scope"]["label"], "No links to missing files")
|
||||
self.assertEqual(len(self.marks()), 1)
|
||||
|
||||
async def test_a_mark_under_an_unreadable_source_stays(self):
|
||||
await self.removed("away.png")
|
||||
os.remove(self.output_dir / "away.png")
|
||||
with patch(
|
||||
"database.maintenance.readable_roots",
|
||||
side_effect=lambda r: [False] * len(r),
|
||||
):
|
||||
plan = await self.links_review()
|
||||
self.assertEqual(plan["scope"]["count"], 0)
|
||||
self.assertEqual(len(self.marks()), 1)
|
||||
|
||||
async def test_the_mark_of_a_changed_file_is_forgotten(self):
|
||||
await self.removed("changed.png")
|
||||
self.write("changed.png", b"other and longer bytes", mtime=4_000_000)
|
||||
plan = await self.links_review()
|
||||
self.assertEqual(plan["scope"]["count"], 1)
|
||||
job = await self.apply(plan["token"])
|
||||
self.assertEqual(job.state, "finished", job.summary())
|
||||
self.assertEqual(self.marks(), [])
|
||||
|
||||
async def test_marks_under_an_empty_source_stay(self):
|
||||
# An unplugged drive leaves its mount point: an empty, readable folder.
|
||||
await self.removed("mounted.png")
|
||||
for child in list(self.output_dir.iterdir()):
|
||||
if child.is_dir():
|
||||
shutil.rmtree(child)
|
||||
else:
|
||||
os.remove(child)
|
||||
plan = await self.links_review()
|
||||
self.assertEqual(plan["scope"]["count"], 0)
|
||||
self.assertEqual(len(self.marks()), 1)
|
||||
|
||||
async def test_a_name_the_folder_still_lists_is_not_gone(self):
|
||||
# A case-sensitive NTFS folder: lstat of the normcased locator fails.
|
||||
await self.removed("Listed.png")
|
||||
real_lstat = os.lstat
|
||||
locator = self.marks()[0]
|
||||
|
||||
def lstat(path, *args, **kwargs):
|
||||
if os.fspath(path) == locator:
|
||||
raise FileNotFoundError(path)
|
||||
return real_lstat(path, *args, **kwargs)
|
||||
|
||||
with (
|
||||
patch("database.maintenance.os.lstat", side_effect=lstat),
|
||||
patch("database.maintenance.os.path.normcase", side_effect=str.lower),
|
||||
):
|
||||
from database.maintenance import _stale_marks
|
||||
|
||||
stale = _stale_marks([(locator, 1, 1)], [str(self.output_dir)])
|
||||
self.assertEqual(stale, [])
|
||||
|
||||
async def test_a_file_back_before_the_apply_changes_the_plan(self):
|
||||
await self.removed("back.png")
|
||||
path = self.output_dir / "back.png"
|
||||
saved = Path(self.tmpdir) / "saved.png"
|
||||
shutil.copy2(path, saved)
|
||||
os.remove(path)
|
||||
plan = await self.links_review()
|
||||
shutil.copy2(saved, path)
|
||||
job = await self.apply(plan["token"])
|
||||
self.assertEqual(job.state, "failed", job.summary())
|
||||
self.assertEqual(len(self.marks()), 1)
|
||||
|
||||
|
||||
class TestWalkersSkipTrashFolders(_GapCase):
|
||||
def seed(self):
|
||||
self.write("kept.png", b"kept")
|
||||
self.write(
|
||||
"x.png", b"trashed x", folder=self.output_dir / ".Trash-1000" / "files"
|
||||
)
|
||||
self.write(
|
||||
"y.png", b"trashed y", folder=self.output_dir / "sub" / "$RECYCLE.BIN"
|
||||
)
|
||||
self.write("z.png", b"trashed z", folder=self.output_dir / ".Trashes")
|
||||
|
||||
def test_walk_media_and_the_settled_entry(self):
|
||||
self.seed()
|
||||
walked = sorted(
|
||||
Path(entry.path).name
|
||||
for entry in images_module._walk_media(self.output_dir, 100)
|
||||
)
|
||||
self.assertEqual(walked, ["kept.png"])
|
||||
for relative in (
|
||||
os.path.join(".Trash-1000", "files", "x.png"),
|
||||
os.path.join("sub", "$RECYCLE.BIN", "y.png"),
|
||||
):
|
||||
with self.subTest(relative=relative):
|
||||
self.assertIsNone(
|
||||
images_module._settled_entry(self.output_dir, relative)
|
||||
)
|
||||
|
||||
def test_scan_duplicate_and_autotag_walkers(self):
|
||||
self.seed()
|
||||
collected = [
|
||||
Path(p).name for p in _collect_media_files([self.output_dir], [".png"])
|
||||
]
|
||||
self.assertEqual(collected, ["kept.png"])
|
||||
found, _truncated = AutotagRoutesMixin._collect_scan_files(self.output_dir)
|
||||
self.assertEqual([p.name for p in found], ["kept.png"])
|
||||
|
||||
def test_the_image_monitor(self):
|
||||
monitor = ImageGenerationHandler.__new__(ImageGenerationHandler)
|
||||
monitor.supported_extensions = (".png",)
|
||||
base = str(self.output_dir)
|
||||
self.assertTrue(monitor.is_image_file(os.path.join(base, "a.png")))
|
||||
for folder in (".Trash-1000", "$Recycle.Bin", ".Trashes", ".trash"):
|
||||
with self.subTest(folder=folder):
|
||||
self.assertFalse(
|
||||
monitor.is_image_file(os.path.join(base, folder, "files", "a.png"))
|
||||
)
|
||||
|
||||
async def test_the_workspace_listing(self):
|
||||
self.seed()
|
||||
self.assertEqual(await self.listed(), ["kept.png"])
|
||||
|
||||
def test_a_root_named_like_a_trash_folder_still_lists(self):
|
||||
root = Path(self.tmpdir) / ".Trash-1000"
|
||||
(root / "inner").mkdir(parents=True)
|
||||
(root / "inner" / "a.png").write_bytes(b"a")
|
||||
walked = [Path(e.path).name for e in images_module._walk_media(root, 10)]
|
||||
self.assertEqual(walked, ["a.png"])
|
||||
@@ -887,6 +887,36 @@ class TestImageTagsFacet(unittest.TestCase):
|
||||
self.assertNotIn("image_tags", json.loads(encode(inspection({}))))
|
||||
self.assertNotIn("image_tags", self.encoded(None))
|
||||
|
||||
def history_encoded(self, facet):
|
||||
from utils.inspection_response import encode_history_inspection_response
|
||||
|
||||
body = encode_history_inspection_response(
|
||||
current_output={"history_epoch": "epoch"},
|
||||
file={"observation": "current-file", "size": 100},
|
||||
prompts=PROMPTS,
|
||||
resolution={"state": "resolved", "sampler_id": "3", "reason": ""},
|
||||
inspection=inspection({"prompt": graph_text()}),
|
||||
library={"status": "none"},
|
||||
image_tags=facet,
|
||||
)
|
||||
return json.loads(body)
|
||||
|
||||
def test_history_envelope_validates_image_tags_like_the_rooted_one(self):
|
||||
"""#457: History's current-file read carries the same facet."""
|
||||
for facet in (
|
||||
{"state": "current", "tags": ["dunes", "warm_light"]},
|
||||
{"state": "none", "tags": []},
|
||||
{"state": "unavailable", "tags": []},
|
||||
{"state": "current", "tags": ["a,b"]},
|
||||
{"state": "current", "tags": [], "locator": "/abs/path.png"},
|
||||
):
|
||||
with self.subTest(facet=facet):
|
||||
self.assertEqual(
|
||||
self.history_encoded(facet)["image_tags"],
|
||||
self.encoded(facet)["image_tags"],
|
||||
)
|
||||
self.assertNotIn("image_tags", self.history_encoded(None))
|
||||
|
||||
|
||||
class TestVideoProbeBoundsMatchPngBudgets(unittest.TestCase):
|
||||
"""ffprobe escapes tag text; the stdout bounds must never gate a record
|
||||
|
||||
@@ -345,7 +345,8 @@ class TestLibrary(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
(parent["title"], parent["notes"]), ("Portrait", "keep\nnotes")
|
||||
)
|
||||
self.assertNotIn("archived", parent)
|
||||
# The parent's own archive state (#443), never a pair's.
|
||||
self.assertIs(parent["archived"], False)
|
||||
self.assertRegex(parent["edit_token"], r"^[0-9a-f]{64}$")
|
||||
self.assertEqual(page["items"][1]["title"], " \t\u00a0\u3000")
|
||||
self.assertEqual(self.db.list_library(text="Portrait")["total"], 1)
|
||||
@@ -1520,6 +1521,7 @@ class TestLibrary(unittest.TestCase):
|
||||
"tags",
|
||||
"rating",
|
||||
"notes",
|
||||
"archived",
|
||||
"variants",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
"""#443 (D152): the Library delete, archive and restore routes, and ingestion after."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
from tests.test_api_admin import AdminAPITestCase # first: it stubs ComfyUI
|
||||
from database.execution_history_schema import rotate_history_epoch # noqa: E402
|
||||
from database.prompt_archive import TABLE as ARCHIVE # noqa: E402
|
||||
from database.safety_backup import SafetyBackupFailed # noqa: E402
|
||||
from py.config import PromptManagerConfig # noqa: E402
|
||||
from utils.hashing import generate_prompt_hash # noqa: E402
|
||||
|
||||
PREVIEW = "/prompt_manager/library/delete/preview"
|
||||
DELETE = "/prompt_manager/library/delete"
|
||||
RESTORE = "/prompt_manager/library/restore"
|
||||
|
||||
|
||||
def graph(positive, negative="blur"):
|
||||
return {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": positive}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": negative}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["2", 0]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class LibraryDeleteRouteCase(AdminAPITestCase):
|
||||
def epoch(self):
|
||||
return self.api.db.get_library_options()["library_epoch"]
|
||||
|
||||
def save(self, text, **metadata):
|
||||
return self.api.db.save_prompt(
|
||||
text, prompt_hash=generate_prompt_hash(text), **metadata
|
||||
)
|
||||
|
||||
def paired(self, positive, negative="blur"):
|
||||
pair = self.api.db.save_prompt_pair(positive, negative)
|
||||
return self.api.db.get_prompt_pair(pair)["positive_prompt_id"]
|
||||
|
||||
def rows(self, sql, params=()):
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
return [tuple(row) for row in conn.execute(sql, params).fetchall()]
|
||||
|
||||
def ids(self):
|
||||
return sorted(row[0] for row in self.rows("SELECT id FROM prompts"))
|
||||
|
||||
def archived(self):
|
||||
return sorted(row[0] for row in self.rows(f"SELECT prompt_id FROM {ARCHIVE}"))
|
||||
|
||||
def backups(self):
|
||||
folder = os.path.dirname(self.api.db.model.db_path)
|
||||
return sorted(name for name in os.listdir(folder) if ".backup_" in name)
|
||||
|
||||
async def post(self, url, body):
|
||||
response = await self.client.request("POST", url, json=body)
|
||||
return response.status, await response.json()
|
||||
|
||||
async def review(self, ids):
|
||||
status, body = await self.post(
|
||||
PREVIEW, {"prompt_ids": ids, "expected_epoch": self.epoch()}
|
||||
)
|
||||
self.assertEqual(status, 200, body)
|
||||
return body
|
||||
|
||||
async def confirm(self, review, backup=None):
|
||||
return await self.post(
|
||||
DELETE,
|
||||
{
|
||||
"archive_ids": review["archive"],
|
||||
"delete_ids": review["delete"],
|
||||
"expected_epoch": review["library_epoch"],
|
||||
"backup": backup or review["backup"],
|
||||
},
|
||||
)
|
||||
|
||||
def policy(self, on):
|
||||
previous = PromptManagerConfig.SAFETY_BACKUP
|
||||
PromptManagerConfig.SAFETY_BACKUP = on
|
||||
self.addCleanup(setattr, PromptManagerConfig, "SAFETY_BACKUP", previous)
|
||||
|
||||
|
||||
class TestLibraryDeleteRoutes(LibraryDeleteRouteCase):
|
||||
async def test_preview_then_delete_archives_deletes_and_backs_up(self):
|
||||
self.policy(True)
|
||||
plain = self.save("Plain")
|
||||
guarded = self.save("Guarded", tags=["__protected__"])
|
||||
positive = self.paired("Cat")
|
||||
review = await self.review([plain, guarded, positive, 99999])
|
||||
self.assertEqual(
|
||||
{
|
||||
key: review[key]
|
||||
for key in ("delete", "archive", "protected", "not_found")
|
||||
},
|
||||
{
|
||||
"delete": [plain],
|
||||
"archive": [positive],
|
||||
"protected": [guarded],
|
||||
"not_found": 1,
|
||||
},
|
||||
)
|
||||
self.assertEqual(review["backup"], "on")
|
||||
self.assertTrue(review["disk"]["sufficient"])
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(
|
||||
(body["deleted"], body["archived"], body["changed"]), (1, 1, 0)
|
||||
)
|
||||
self.assertEqual(self.backups(), [body["backup"]])
|
||||
self.assertNotIn(os.sep, body["backup"])
|
||||
self.assertNotIn(plain, self.ids())
|
||||
self.assertEqual(self.archived(), [positive])
|
||||
# The backup still holds the deleted prompt.
|
||||
import sqlite3
|
||||
|
||||
path = os.path.join(os.path.dirname(self.api.db.model.db_path), body["backup"])
|
||||
with sqlite3.connect(path) as copy:
|
||||
self.assertEqual(
|
||||
copy.execute(
|
||||
"SELECT COUNT(*) FROM prompts WHERE id=?", (plain,)
|
||||
).fetchone()[0],
|
||||
1,
|
||||
)
|
||||
copy.close()
|
||||
# The hold ended: nothing is listed as running.
|
||||
self.assertEqual(self.api._jobs.running(), [])
|
||||
|
||||
async def test_archive_only_takes_no_backup(self):
|
||||
self.policy(True)
|
||||
positive = self.paired("Cat")
|
||||
review = await self.review([positive])
|
||||
self.assertIsNone(review["disk"])
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual((status, body["archived"], body["backup"]), (200, 1, None))
|
||||
self.assertEqual(self.backups(), [])
|
||||
|
||||
async def test_policy_off_takes_none_unless_the_review_promised_one(self):
|
||||
plain = self.save("Plain")
|
||||
other = self.save("Other")
|
||||
self.policy(False)
|
||||
review = await self.review([plain])
|
||||
self.assertEqual((review["backup"], review["disk"]), ("off", None))
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual((status, body["deleted"], body["backup"]), (200, 1, None))
|
||||
# Reviewed with the policy on, turned off before the confirm: kept.
|
||||
review = await self.review([other])
|
||||
status, body = await self.confirm(review, backup="on")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertIsNotNone(body["backup"])
|
||||
|
||||
async def test_a_failed_backup_changes_nothing(self):
|
||||
self.policy(True)
|
||||
plain = self.save("Plain")
|
||||
positive = self.paired("Cat")
|
||||
review = await self.review([plain, positive])
|
||||
with patch(
|
||||
"py.api.maintenance.take_safety_backup",
|
||||
side_effect=SafetyBackupFailed("x"),
|
||||
):
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual((status, body["code"]), (500, "backup-failed"))
|
||||
self.assertIn("nothing was changed", body["error"])
|
||||
self.assertIn(plain, self.ids())
|
||||
self.assertEqual(self.archived(), [])
|
||||
|
||||
async def test_a_short_disk_refuses_before_any_change(self):
|
||||
self.policy(True)
|
||||
plain = self.save("Plain")
|
||||
from py.api import maintenance
|
||||
|
||||
with patch.object(maintenance, "_free_bytes", return_value=0):
|
||||
review = await self.review([plain])
|
||||
self.assertFalse(review["disk"]["sufficient"])
|
||||
self.assertIn("Nothing was changed", review["disk"]["message"])
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual((status, body["code"]), (507, "insufficient-space"))
|
||||
self.assertIn(plain, self.ids())
|
||||
|
||||
async def test_busy_beside_a_record_changing_job(self):
|
||||
plain = self.save("Plain")
|
||||
review = await self.review([plain])
|
||||
async with self.api._jobs.hold("scan", "Scan", admission="records"):
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual((status, body["code"]), (409, "busy"))
|
||||
self.assertEqual(body["running"]["title"], "Scan")
|
||||
self.assertIn(plain, self.ids())
|
||||
|
||||
async def test_restored_storage_answers_409_on_every_route(self):
|
||||
plain = self.save("Plain")
|
||||
review = await self.review([plain])
|
||||
rotate_history_epoch(self.api.db.model.get_connection())
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual((status, body["code"]), (409, "storage-changed"))
|
||||
for url in (PREVIEW, RESTORE):
|
||||
status, body = await self.post(
|
||||
url, {"prompt_ids": [plain], "expected_epoch": review["library_epoch"]}
|
||||
)
|
||||
self.assertEqual((status, body["code"]), (409, "storage-changed"))
|
||||
self.assertIn(plain, self.ids())
|
||||
|
||||
async def test_restore_route(self):
|
||||
positive = self.paired("Cat")
|
||||
await self.confirm(await self.review([positive]))
|
||||
status, body = await self.post(
|
||||
RESTORE, {"prompt_ids": [positive, 99999], "expected_epoch": self.epoch()}
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual((body["restored"], body["not_found"]), (1, 1))
|
||||
self.assertEqual(self.archived(), [])
|
||||
|
||||
async def test_bodies_are_validated(self):
|
||||
plain = self.save("Plain")
|
||||
epoch = self.epoch()
|
||||
selection = {"prompt_ids": [plain], "expected_epoch": epoch}
|
||||
for url, body in (
|
||||
(PREVIEW, {**selection, "extra": 1}),
|
||||
(PREVIEW, {"prompt_ids": [], "expected_epoch": epoch}),
|
||||
(PREVIEW, {"prompt_ids": [True], "expected_epoch": epoch}),
|
||||
(PREVIEW, {"prompt_ids": list(range(1, 1002)), "expected_epoch": epoch}),
|
||||
(PREVIEW, {"prompt_ids": [plain], "expected_epoch": "nope"}),
|
||||
(RESTORE, {"prompt_ids": [plain]}),
|
||||
(DELETE, {**selection}),
|
||||
(
|
||||
DELETE,
|
||||
{
|
||||
"archive_ids": [],
|
||||
"delete_ids": [],
|
||||
"expected_epoch": epoch,
|
||||
"backup": "on",
|
||||
},
|
||||
),
|
||||
(
|
||||
DELETE,
|
||||
{
|
||||
"archive_ids": [plain],
|
||||
"delete_ids": [plain],
|
||||
"expected_epoch": epoch,
|
||||
"backup": "on",
|
||||
},
|
||||
),
|
||||
(
|
||||
DELETE,
|
||||
{
|
||||
"archive_ids": [],
|
||||
"delete_ids": [plain],
|
||||
"expected_epoch": epoch,
|
||||
"backup": "maybe",
|
||||
},
|
||||
),
|
||||
(
|
||||
DELETE,
|
||||
{
|
||||
"archive_ids": list(range(1, 600)),
|
||||
"delete_ids": list(range(600, 1100)),
|
||||
"expected_epoch": epoch,
|
||||
"backup": "off",
|
||||
},
|
||||
),
|
||||
):
|
||||
with self.subTest(url=url, body=body):
|
||||
status, answer = await self.post(url, body)
|
||||
self.assertEqual(status, 400, answer)
|
||||
self.assertFalse(answer["success"])
|
||||
for url in (PREVIEW, DELETE, RESTORE):
|
||||
response = await self.client.request(
|
||||
"POST", url, data="[1]", headers={"Content-Type": "application/json"}
|
||||
)
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertIn(plain, self.ids())
|
||||
|
||||
async def test_the_legacy_bulk_delete_is_unchanged(self):
|
||||
plain = self.save("Plain")
|
||||
positive = self.paired("Cat")
|
||||
response = await self.client.request(
|
||||
"POST", "/prompt_manager/bulk/delete", json={"prompt_ids": [positive]}
|
||||
)
|
||||
self.assertEqual(response.status, 409)
|
||||
response = await self.client.request(
|
||||
"POST", "/prompt_manager/bulk/delete", json={"prompt_ids": [plain]}
|
||||
)
|
||||
self.assertEqual((await response.json())["deleted_count"], 1)
|
||||
self.assertEqual(self.archived(), [])
|
||||
|
||||
async def test_shelf_parameter_and_cursor_scope(self):
|
||||
kept = [self.paired(f"Kept {i}") for i in range(3)]
|
||||
gone = [self.paired(f"Gone {i}") for i in range(3)]
|
||||
await self.confirm(await self.review(gone))
|
||||
response = await self.client.request(
|
||||
"GET", "/prompt_manager/library?limit=2&shelf=active"
|
||||
)
|
||||
page = await response.json()
|
||||
self.assertEqual(page["archived_total"], 3)
|
||||
self.assertTrue(set(item["id"] for item in page["items"]) <= set(kept))
|
||||
response = await self.client.request(
|
||||
"GET",
|
||||
"/prompt_manager/library?limit=2&shelf=archived"
|
||||
f"&cursor={page['next_cursor']}",
|
||||
)
|
||||
self.assertEqual(response.status, 400)
|
||||
response = await self.client.request(
|
||||
"GET", "/prompt_manager/library?shelf=archived"
|
||||
)
|
||||
archived = await response.json()
|
||||
self.assertEqual(sorted(item["id"] for item in archived["items"]), gone)
|
||||
self.assertTrue(all(item["archived"] for item in archived["items"]))
|
||||
response = await self.client.request("GET", "/prompt_manager/library?shelf=all")
|
||||
self.assertEqual(response.status, 400)
|
||||
|
||||
|
||||
class TestIngestionAfterArchive(LibraryDeleteRouteCase):
|
||||
def scan(self, name, positive, negative="blur"):
|
||||
meta = {"prompt": json.dumps(graph(positive, negative))}
|
||||
path = self.output_dir / name
|
||||
path.write_bytes(b"png")
|
||||
return self.api._ingest_scanned_file(path, meta)
|
||||
|
||||
async def test_a_scan_after_archive_creates_no_duplicate_and_keeps_it_archived(
|
||||
self,
|
||||
):
|
||||
self.assertEqual(self.scan("first.png", "Sunset"), "added")
|
||||
positive = self.api.db.get_prompt_by_hash(generate_prompt_hash("Sunset"))["id"]
|
||||
await self.confirm(await self.review([positive]))
|
||||
before = self.ids()
|
||||
self.scan("second.png", "Sunset")
|
||||
self.scan("third.png", " sunset ", "other negative")
|
||||
self.assertEqual(
|
||||
[i for i in self.ids() if i not in before],
|
||||
[
|
||||
self.api.db.get_prompt_by_hash(generate_prompt_hash("other negative"))[
|
||||
"id"
|
||||
]
|
||||
],
|
||||
)
|
||||
self.assertEqual(self.archived(), [positive])
|
||||
linked = self.rows(
|
||||
"SELECT COUNT(*) FROM generated_images WHERE prompt_id=?", (positive,)
|
||||
)
|
||||
self.assertEqual(linked, [(3,)])
|
||||
response = await self.client.request("GET", "/prompt_manager/library")
|
||||
self.assertNotIn(
|
||||
positive, [item["id"] for item in (await response.json())["items"]]
|
||||
)
|
||||
|
||||
async def test_a_scan_after_delete_creates_the_prompt_again(self):
|
||||
plain = self.save("Lonely sunset")
|
||||
await self.confirm(await self.review([plain]))
|
||||
self.assertEqual(self.scan("again.png", "Lonely sunset"), "added")
|
||||
again = self.api.db.get_prompt_by_hash(generate_prompt_hash("Lonely sunset"))
|
||||
self.assertIsNotNone(again)
|
||||
self.assertNotEqual(again["id"], plain)
|
||||
self.assertEqual(self.archived(), [])
|
||||
@@ -0,0 +1,243 @@
|
||||
"""#444: the Library's output-folder filter and multi-tag OR."""
|
||||
|
||||
import ntpath
|
||||
import os
|
||||
import posixpath
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from database.library_folders import (
|
||||
folder_prefixes,
|
||||
normalize_folder,
|
||||
relative_folder,
|
||||
)
|
||||
from database.operations import PromptDatabase
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
|
||||
class TestFolderNames(unittest.TestCase):
|
||||
def test_normalize_strips_outer_slashes_on_both_flavors(self):
|
||||
for flavor in (posixpath, ntpath):
|
||||
with self.subTest(flavor=flavor.__name__):
|
||||
self.assertEqual(normalize_folder("", flavor), "")
|
||||
self.assertEqual(normalize_folder("/", flavor), "")
|
||||
self.assertEqual(normalize_folder("/a/b/", flavor), "a/b")
|
||||
self.assertEqual(normalize_folder("..foo/b", flavor), "..foo/b")
|
||||
|
||||
def test_a_backslash_separates_only_on_windows(self):
|
||||
# Live B1: on POSIX a backslash is part of a folder name, as listed.
|
||||
name = "Metadata\\Klein_4b Wildcard Run"
|
||||
self.assertEqual(normalize_folder(name, posixpath), name)
|
||||
self.assertEqual(
|
||||
normalize_folder("2026\\08-Aug\\", posixpath), "2026\\08-Aug\\"
|
||||
)
|
||||
self.assertEqual(
|
||||
normalize_folder(name, ntpath), "Metadata/Klein_4b Wildcard Run"
|
||||
)
|
||||
self.assertEqual(normalize_folder("2026\\08-Aug\\", ntpath), "2026/08-Aug")
|
||||
# A colon names a drive only on Windows.
|
||||
self.assertEqual(normalize_folder("a:b/c", posixpath), "a:b/c")
|
||||
|
||||
def test_normalize_refuses_anything_but_a_folder_below_a_root(self):
|
||||
for flavor, value in (
|
||||
*(
|
||||
(flavor, value)
|
||||
for flavor in (posixpath, ntpath)
|
||||
for value in ("a/../b", "./a", "a//b", "a\x00b", 3, "x" * 1025)
|
||||
),
|
||||
(ntpath, "C:/x"),
|
||||
(ntpath, "c:"),
|
||||
(ntpath, "a\\..\\b"),
|
||||
):
|
||||
with self.subTest(flavor=flavor.__name__, value=value):
|
||||
with self.assertRaises(ValueError):
|
||||
normalize_folder(value, flavor)
|
||||
|
||||
def test_the_default_flavor_is_the_platform_path_module(self):
|
||||
self.assertEqual(normalize_folder("a\\b"), normalize_folder("a\\b", os.path))
|
||||
|
||||
def test_posix_prefixes_are_exact_and_end_in_a_separator(self):
|
||||
self.assertEqual(
|
||||
folder_prefixes(["/out", "/data/out/"], "a/b", posixpath),
|
||||
["/out/a/b/", "/data/out/a/b/"],
|
||||
)
|
||||
self.assertEqual(folder_prefixes(["/"], "a", posixpath), ["/a/"])
|
||||
self.assertEqual(
|
||||
folder_prefixes(["/out", "/out/"], "a", posixpath), ["/out/a/"]
|
||||
)
|
||||
with self.assertRaises(ValueError):
|
||||
folder_prefixes([], "a", posixpath)
|
||||
|
||||
def test_windows_prefixes_fold_case_including_non_ascii(self):
|
||||
self.assertEqual(
|
||||
folder_prefixes(["C:\\ComfyUI\\Output"], "Ä/Sub", ntpath),
|
||||
["c:\\comfyui\\output\\ä\\sub\\"],
|
||||
)
|
||||
self.assertEqual(folder_prefixes(["D:\\"], "x", ntpath), ["d:\\x\\"])
|
||||
self.assertEqual(
|
||||
folder_prefixes(["\\\\Server\\Share\\Out"], "a", ntpath),
|
||||
["\\\\server\\share\\out\\a\\"],
|
||||
)
|
||||
# The stored file_path is normcase(normpath(abspath)), the same fold.
|
||||
stored = ntpath.normcase(ntpath.normpath("C:/ComfyUI/output/Ä/sub/x.png"))
|
||||
self.assertTrue(
|
||||
stored.startswith(
|
||||
folder_prefixes(["C:/ComfyUI/Output"], "ä/SUB", ntpath)[0]
|
||||
)
|
||||
)
|
||||
|
||||
def test_relative_folder_keeps_the_stored_spelling(self):
|
||||
self.assertEqual(
|
||||
relative_folder("/out/2026/Aug/x.png", ["/out"], posixpath), "2026/Aug"
|
||||
)
|
||||
self.assertIsNone(relative_folder("/out/x.png", ["/out"], posixpath))
|
||||
self.assertIsNone(relative_folder("/outside/x.png", ["/out"], posixpath))
|
||||
self.assertIsNone(relative_folder("/output2/a/x.png", ["/out"], posixpath))
|
||||
self.assertEqual(
|
||||
relative_folder("/out/..foo/x.png", ["/out"], posixpath), "..foo"
|
||||
)
|
||||
self.assertEqual(
|
||||
relative_folder("C:\\Out\\Ä\\Sub\\x.png", ["c:\\out"], ntpath), "Ä/Sub"
|
||||
)
|
||||
self.assertEqual(relative_folder("D:\\a\\x.png", ["D:\\"], ntpath), "a")
|
||||
|
||||
|
||||
class TestLibraryFolderAndAnyTag(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
base = Path(os.path.realpath(self.temp.name))
|
||||
self.root = base / "output"
|
||||
self.root.mkdir()
|
||||
self.db = PromptDatabase(str(base / "folders.db"))
|
||||
|
||||
def tearDown(self):
|
||||
try:
|
||||
self.db.close_all()
|
||||
finally:
|
||||
self.temp.cleanup()
|
||||
|
||||
def save(self, text, **metadata):
|
||||
return self.db.save_prompt(
|
||||
text, prompt_hash=generate_prompt_hash(text), **metadata
|
||||
)
|
||||
|
||||
def link(self, prompt_id, *parts):
|
||||
path = self.root.joinpath(*parts)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(b"x")
|
||||
self.db.link_image_to_prompt(prompt_id, str(path))
|
||||
|
||||
def texts(self, **options):
|
||||
return sorted(item["text"] for item in self.db.list_library(**options)["items"])
|
||||
|
||||
def in_folder(self, folder, roots=None):
|
||||
return self.texts(
|
||||
folder_prefixes=folder_prefixes(roots or [str(self.root)], folder)
|
||||
)
|
||||
|
||||
def test_folder_matches_its_descendants_only(self):
|
||||
self.link(self.save("deep"), "a", "b", "x.png")
|
||||
self.link(self.save("top"), "a", "y.png")
|
||||
self.link(self.save("sibling"), "ab", "z.png")
|
||||
self.link(self.save("literal"), "100%_off", "w.png")
|
||||
self.save("no images")
|
||||
self.assertEqual(self.in_folder("a"), ["deep", "top"])
|
||||
self.assertEqual(self.in_folder("a/b"), ["deep"])
|
||||
self.assertEqual(self.in_folder("ab"), ["sibling"])
|
||||
self.assertEqual(self.in_folder("100%_off"), ["literal"])
|
||||
self.assertEqual(self.in_folder("1"), [])
|
||||
cut = self.db.list_library(
|
||||
folder_prefixes=folder_prefixes([str(self.root)], "a"), limit=1
|
||||
)
|
||||
self.assertEqual(cut["total"], 2)
|
||||
|
||||
def test_folder_combines_with_text_category_and_tags(self):
|
||||
self.link(self.save("red cat", category="Art", tags=["cat"]), "a", "1.png")
|
||||
self.link(self.save("red dog", category="Art", tags=["dog"]), "a", "2.png")
|
||||
self.link(self.save("red bird", category="Art", tags=["cat"]), "b", "3.png")
|
||||
prefixes = folder_prefixes([str(self.root)], "a")
|
||||
self.assertEqual(
|
||||
self.texts(folder_prefixes=prefixes, tags=["cat"]), ["red cat"]
|
||||
)
|
||||
self.assertEqual(
|
||||
self.texts(folder_prefixes=prefixes, text="dog", category="Art"),
|
||||
["red dog"],
|
||||
)
|
||||
|
||||
@unittest.skipIf(os.name == "nt", "symlinks need privileges on Windows")
|
||||
def test_folder_listing_and_match_through_a_symlinked_root(self):
|
||||
link = self.root.parent / "linked"
|
||||
os.symlink(self.root, link)
|
||||
prompt = self.save("via link")
|
||||
(self.root / "a").mkdir()
|
||||
(self.root / "a" / "x.png").write_bytes(b"x")
|
||||
self.db.link_image_to_prompt(prompt, str(link / "a" / "x.png"))
|
||||
spellings = [str(self.root), str(link)]
|
||||
self.assertEqual(self.in_folder("a", spellings), ["via link"])
|
||||
self.assertEqual(self.in_folder("a", [str(self.root)]), [])
|
||||
self.assertEqual(self.db.get_library_options(spellings)["folders"], ["a"])
|
||||
|
||||
@unittest.skipIf(os.name == "nt", "a backslash is a separator on Windows")
|
||||
def test_a_posix_folder_with_a_backslash_lists_and_filters_alike(self):
|
||||
name = "Metadata\\Klein_4b Wildcard Run"
|
||||
self.link(self.save("klein"), name, "x.png")
|
||||
self.link(self.save("other"), "Metadata", "y.png")
|
||||
listed = self.db.get_library_options([str(self.root)])["folders"]
|
||||
self.assertEqual(listed, ["Metadata", name])
|
||||
self.assertEqual(self.in_folder(normalize_folder(name)), ["klein"])
|
||||
self.assertEqual(self.in_folder(normalize_folder("Metadata")), ["other"])
|
||||
|
||||
def test_folder_listing_offers_ancestors_and_skips_outside_images(self):
|
||||
self.link(self.save("deep"), "2026", "08-Aug", "x.png")
|
||||
self.link(self.save("root image"), "y.png")
|
||||
outside = Path(self.temp.name) / "elsewhere" / "z.png"
|
||||
outside.parent.mkdir()
|
||||
outside.write_bytes(b"x")
|
||||
self.db.link_image_to_prompt(self.save("outside"), str(outside))
|
||||
options = self.db.get_library_options([str(self.root)])
|
||||
self.assertEqual(options["folders"], ["2026", "2026/08-Aug"])
|
||||
self.assertNotIn("folders", self.db.get_library_options())
|
||||
|
||||
def test_folder_combines_with_untagged(self):
|
||||
self.link(self.save("plain in a"), "a", "1.png")
|
||||
self.link(self.save("tagged in a", tags=["t"]), "a", "2.png")
|
||||
self.link(self.save("plain in b"), "b", "3.png")
|
||||
prefixes = folder_prefixes([str(self.root)], "a")
|
||||
self.assertEqual(
|
||||
self.texts(folder_prefixes=prefixes, tag_mode="untagged"), ["plain in a"]
|
||||
)
|
||||
|
||||
def test_any_tag_mode_is_the_union_across_parent_and_pair(self):
|
||||
self.save("parent a", tags=["a"])
|
||||
self.save("parent b", tags=["b"])
|
||||
self.save("both", tags=["a", "b"])
|
||||
self.save("neither", tags=["c"])
|
||||
self.db.save_prompt_pair("pair b", "noise", tags=["b"])
|
||||
split = self.save("split", tags=["a"])
|
||||
self.db.save_prompt_pair("split", "split noise", tags=["b"])
|
||||
archived = self.db.save_prompt_pair("archived b", "old", tags=["b"])
|
||||
self.db.archive_prompt_pair(archived)
|
||||
self.assertIsInstance(split, int)
|
||||
self.assertEqual(
|
||||
self.texts(tags=["a", "b"], tag_mode="any"),
|
||||
["both", "pair b", "parent a", "parent b", "split"],
|
||||
)
|
||||
self.assertEqual(self.texts(tags=["a", "b"], tag_mode="all"), ["both"])
|
||||
self.assertEqual(self.texts(tags=["a", "b"]), ["both"])
|
||||
for tags in (["a"], ["b"], ["c"]):
|
||||
self.assertEqual(
|
||||
self.texts(tags=tags, tag_mode="any"), self.texts(tags=tags)
|
||||
)
|
||||
self.assertEqual(
|
||||
self.texts(tags=["a", "b"], tag_mode="any", text="parent"),
|
||||
["parent a", "parent b"],
|
||||
)
|
||||
self.assertEqual(self.texts(tag_mode="any"), self.texts())
|
||||
for mode in ("none", None, "ANY"):
|
||||
with self.subTest(mode=mode), self.assertRaises(ValueError):
|
||||
self.db.list_library(tag_mode=mode)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -14,6 +14,7 @@ import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
@@ -753,6 +754,17 @@ class LocalMediaTests(LayoutTestCase):
|
||||
self.assertEqual(layout_module.VIDEO_SUFFIXES, frozenset(VIDEO_EXTENSIONS))
|
||||
|
||||
|
||||
# Windows path rules without the filesystem. On Windows ntpath.realpath asks
|
||||
# the network for \\server\share and waits seconds for a host that does not
|
||||
# exist (#487); elsewhere ntpath.realpath already is ntpath.abspath.
|
||||
LEXICAL_NT = SimpleNamespace(
|
||||
normcase=ntpath.normcase,
|
||||
realpath=ntpath.abspath,
|
||||
dirname=ntpath.dirname,
|
||||
sep=ntpath.sep,
|
||||
)
|
||||
|
||||
|
||||
class FolderGuardTests(unittest.TestCase):
|
||||
"""Portable path-flavor controls (TESTING.md Windows portability preflight)."""
|
||||
|
||||
@@ -762,7 +774,7 @@ class FolderGuardTests(unittest.TestCase):
|
||||
|
||||
def guard_nt(self, path, home="C:\\Users\\Vito"):
|
||||
return layout_module._guard_folder(
|
||||
path, home, pathmod=ntpath, isdir=lambda _p: True
|
||||
path, home, pathmod=LEXICAL_NT, isdir=lambda _p: True
|
||||
)
|
||||
|
||||
def test_filesystem_roots_and_home_are_rejected(self):
|
||||
@@ -832,8 +844,12 @@ class FolderGuardTests(unittest.TestCase):
|
||||
|
||||
def test_missing_folders_add_nothing(self):
|
||||
self.assertIsNone(layout_module._guard_folder(str(self.tmp / "x"), "/home"))
|
||||
# A missing share on this machine: an unknown server name made
|
||||
# Windows wait about 2.8 s for the network (#487).
|
||||
self.assertIsNone(
|
||||
layout_module._guard_folder("\\\\server\\share\\loras", str(self.tmp))
|
||||
layout_module._guard_folder(
|
||||
"\\\\localhost\\pm-no-such-share\\loras", str(self.tmp)
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
layout_module._guard_folder(str(self.tmp), "/nonexistent-home"),
|
||||
@@ -843,17 +859,17 @@ class FolderGuardTests(unittest.TestCase):
|
||||
def test_windows_model_path_matches_a_lower_case_root(self):
|
||||
self.assertTrue(
|
||||
layout_module._inside(
|
||||
"C:/Models/Loras/A.safetensors", ("c:\\models\\loras",), ntpath
|
||||
"C:/Models/Loras/A.safetensors", ("c:\\models\\loras",), LEXICAL_NT
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
layout_module._inside(
|
||||
"C:/Models/LorasX/A.safetensors", ("c:\\models\\loras",), ntpath
|
||||
"C:/Models/LorasX/A.safetensors", ("c:\\models\\loras",), LEXICAL_NT
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
layout_module._inside(
|
||||
"\\\\other\\share\\A.safetensors", ("c:\\models\\loras",), ntpath
|
||||
"\\\\other\\share\\A.safetensors", ("c:\\models\\loras",), LEXICAL_NT
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import email
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import ssl
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
@@ -669,6 +670,15 @@ class TestStaleDownloadSweep(unittest.TestCase):
|
||||
self.assertTrue(old.exists())
|
||||
|
||||
|
||||
def _unused_tls_context():
|
||||
"""A TLS context for the fake handlers, which never open a connection.
|
||||
|
||||
HTTPSHandler() builds a default context, and on Windows that reads the
|
||||
system certificate store: about 0.14 s per handler (#487).
|
||||
"""
|
||||
return ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
|
||||
|
||||
class _FakeHTTPSHandler(urllib.request.HTTPSHandler):
|
||||
"""Serves canned responses through a real urllib opener, recording requests.
|
||||
|
||||
@@ -677,7 +687,7 @@ class _FakeHTTPSHandler(urllib.request.HTTPSHandler):
|
||||
"""
|
||||
|
||||
def __init__(self, redirect_to):
|
||||
super().__init__()
|
||||
super().__init__(context=_unused_tls_context())
|
||||
self.redirect_to = redirect_to
|
||||
self.requests = []
|
||||
|
||||
@@ -745,7 +755,7 @@ class _ChainHTTPSHandler(urllib.request.HTTPSHandler):
|
||||
urllib opener, recording every request; an unknown URL answers 404."""
|
||||
|
||||
def __init__(self, routes):
|
||||
super().__init__()
|
||||
super().__init__(context=_unused_tls_context())
|
||||
self.routes = routes
|
||||
self.requests = []
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""The shared folder skip every media walker uses (#501)."""
|
||||
|
||||
import unittest
|
||||
|
||||
from utils.media_folders import in_skipped_folder, is_skipped_folder, is_trash_folder
|
||||
|
||||
|
||||
class TrashFolderNameTests(unittest.TestCase):
|
||||
def test_trash_folder_names_in_any_case(self):
|
||||
for name in (
|
||||
".Trash",
|
||||
".trash",
|
||||
".Trash-1000",
|
||||
".trash-0",
|
||||
".Trashes",
|
||||
"$RECYCLE.BIN",
|
||||
"$Recycle.Bin",
|
||||
):
|
||||
with self.subTest(name=name):
|
||||
self.assertTrue(is_trash_folder(name))
|
||||
self.assertTrue(is_skipped_folder(name))
|
||||
|
||||
def test_other_names_are_not_trash(self):
|
||||
for name in (".Trash-abc", ".Trash-", "trash", "Recycle", "RECYCLE.BIN", "x"):
|
||||
with self.subTest(name=name):
|
||||
self.assertFalse(is_trash_folder(name))
|
||||
|
||||
def test_thumbnails_is_skipped_exactly(self):
|
||||
self.assertTrue(is_skipped_folder("thumbnails"))
|
||||
self.assertFalse(is_skipped_folder("Thumbnails"))
|
||||
self.assertFalse(is_trash_folder("thumbnails"))
|
||||
|
||||
def test_folder_parts(self):
|
||||
self.assertTrue(in_skipped_folder(("a", ".Trash-1000", "files")))
|
||||
self.assertTrue(in_skipped_folder(("thumbnails",)))
|
||||
self.assertFalse(in_skipped_folder(("a", "b")))
|
||||
self.assertFalse(in_skipped_folder(()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,196 @@
|
||||
"""#443: the Library parent archive table, its migration and its validation."""
|
||||
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from database.models import PromptModel
|
||||
from database.operations import PromptDatabase
|
||||
from database.prompt_archive import (
|
||||
TABLE,
|
||||
migrate_prompt_archive,
|
||||
prompt_archive_schema_error,
|
||||
)
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
|
||||
def _tables(conn):
|
||||
return {
|
||||
row[0]
|
||||
for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||
}
|
||||
|
||||
|
||||
class TestPromptArchiveSchema(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.path = str(Path(self.temp.name) / "archive.db")
|
||||
self.db = PromptDatabase(self.path)
|
||||
|
||||
def tearDown(self):
|
||||
self.db.close_all()
|
||||
self.temp.cleanup()
|
||||
|
||||
def conn(self):
|
||||
return self.db.model.get_connection()
|
||||
|
||||
def save(self, text):
|
||||
return self.db.save_prompt(text, prompt_hash=generate_prompt_hash(text))
|
||||
|
||||
def test_startup_creates_the_table_and_it_validates(self):
|
||||
self.assertIn(TABLE, _tables(self.conn()))
|
||||
self.assertIsNone(prompt_archive_schema_error(self.conn(), allow_missing=False))
|
||||
|
||||
def test_migration_is_idempotent(self):
|
||||
conn = self.conn()
|
||||
migrate_prompt_archive(conn)
|
||||
migrate_prompt_archive(conn)
|
||||
self.assertIsNone(prompt_archive_schema_error(conn, allow_missing=False))
|
||||
|
||||
def test_a_missing_table_is_accepted_only_when_allowed(self):
|
||||
conn = sqlite3.connect(":memory:")
|
||||
self.addCleanup(conn.close)
|
||||
self.assertIsNone(prompt_archive_schema_error(conn))
|
||||
self.assertEqual(
|
||||
prompt_archive_schema_error(conn, allow_missing=False),
|
||||
"Missing prompt archive schema",
|
||||
)
|
||||
|
||||
def test_deleting_a_prompt_drops_its_archive_row(self):
|
||||
pid = self.save("Archived then deleted")
|
||||
conn = self.conn()
|
||||
conn.execute(
|
||||
f"INSERT INTO {TABLE}(prompt_id, archived_at) VALUES (?, ?)",
|
||||
(pid, "2026-10-28T00:00:00+00:00"),
|
||||
)
|
||||
conn.commit()
|
||||
self.db.delete_prompt(pid)
|
||||
self.assertEqual(conn.execute(f"SELECT COUNT(*) FROM {TABLE}").fetchone()[0], 0)
|
||||
|
||||
def _broken(self, *statements):
|
||||
conn = sqlite3.connect(":memory:")
|
||||
self.addCleanup(conn.close)
|
||||
conn.execute("CREATE TABLE prompts (id INTEGER PRIMARY KEY)")
|
||||
for statement in statements:
|
||||
conn.execute(statement)
|
||||
return conn
|
||||
|
||||
def test_unsupported_structures_are_refused(self):
|
||||
good = (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
|
||||
"archived_at TEXT NOT NULL, "
|
||||
"FOREIGN KEY(prompt_id) REFERENCES prompts(id) ON DELETE CASCADE)"
|
||||
)
|
||||
cases = {
|
||||
"other definition": (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
|
||||
"archived_at TEXT)",
|
||||
),
|
||||
"no cascade": (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
|
||||
"archived_at TEXT NOT NULL, "
|
||||
"FOREIGN KEY(prompt_id) REFERENCES prompts(id))",
|
||||
),
|
||||
"extra index": (good, f"CREATE INDEX x ON {TABLE}(archived_at)"),
|
||||
"trigger": (
|
||||
good,
|
||||
f"CREATE TRIGGER t AFTER INSERT ON {TABLE} BEGIN SELECT 1; END",
|
||||
),
|
||||
"view name": (f"CREATE VIEW {TABLE} AS SELECT 1",),
|
||||
}
|
||||
for name, statements in cases.items():
|
||||
with self.subTest(name):
|
||||
self.assertIsNotNone(
|
||||
prompt_archive_schema_error(self._broken(*statements))
|
||||
)
|
||||
|
||||
def test_invalid_values_and_dangling_rows_are_refused(self):
|
||||
good = (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
|
||||
"archived_at TEXT NOT NULL, "
|
||||
"FOREIGN KEY(prompt_id) REFERENCES prompts(id) ON DELETE CASCADE)"
|
||||
)
|
||||
for name, row in {
|
||||
"number": "(1, 5)",
|
||||
"blob": "(1, x'00')",
|
||||
"long": f"(1, '{'x' * 65}')",
|
||||
"dangling": "(99, '2026-10-28T00:00:00+00:00')",
|
||||
}.items():
|
||||
with self.subTest(name):
|
||||
conn = self._broken(good, "INSERT INTO prompts(id) VALUES (1)")
|
||||
conn.execute(f"INSERT INTO {TABLE} VALUES {row}")
|
||||
self.assertIsNotNone(prompt_archive_schema_error(conn))
|
||||
|
||||
def test_a_refused_structure_stops_the_migration_without_changes(self):
|
||||
conn = self._broken(
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, archived_at TEXT)"
|
||||
)
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
migrate_prompt_archive(conn)
|
||||
sql = conn.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE name=?", (TABLE,)
|
||||
).fetchone()[0]
|
||||
self.assertNotIn("NOT NULL", sql)
|
||||
|
||||
|
||||
class TestPromptArchiveBackups(unittest.TestCase):
|
||||
"""Restore validation accepts an older backup and refuses a damaged table."""
|
||||
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temp.name)
|
||||
self.db = PromptDatabase(str(self.root / "live.db"))
|
||||
|
||||
def tearDown(self):
|
||||
self.db.close_all()
|
||||
self.temp.cleanup()
|
||||
|
||||
def copy(self, name):
|
||||
target = self.root / name
|
||||
self.db.model.backup_database(str(target))
|
||||
return str(target)
|
||||
|
||||
def test_a_backup_without_the_table_verifies_and_migrates(self):
|
||||
path = self.copy("old.db")
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute(f"DROP TABLE {TABLE}")
|
||||
conn.close()
|
||||
ok, reason = self.db.model.verify_database_file(path)
|
||||
self.assertTrue(ok, reason)
|
||||
model = PromptModel(path)
|
||||
try:
|
||||
self.assertIn(TABLE, _tables(model.get_connection()))
|
||||
finally:
|
||||
model.close_all()
|
||||
|
||||
def test_a_backup_with_a_damaged_table_is_refused(self):
|
||||
path = self.copy("bad.db")
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute(f"DROP TABLE {TABLE}")
|
||||
conn.execute(f"CREATE TABLE {TABLE} (prompt_id INTEGER, archived_at TEXT)")
|
||||
conn.close()
|
||||
ok, reason = self.db.model.verify_database_file(path)
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("archive", reason.lower())
|
||||
|
||||
def test_an_archived_prompt_survives_a_backup_copy(self):
|
||||
pid = self.db.save_prompt("Kept", prompt_hash=generate_prompt_hash("Kept"))
|
||||
conn = self.db.model.get_connection()
|
||||
conn.execute(
|
||||
f"INSERT INTO {TABLE}(prompt_id, archived_at) VALUES (?, ?)",
|
||||
(pid, "2026-10-28T00:00:00+00:00"),
|
||||
)
|
||||
conn.commit()
|
||||
path = self.copy("copy.db")
|
||||
shutil.copy(path, path + ".2")
|
||||
with sqlite3.connect(path + ".2") as other:
|
||||
self.assertEqual(
|
||||
other.execute(f"SELECT prompt_id FROM {TABLE}").fetchall(), [(pid,)]
|
||||
)
|
||||
other.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -772,6 +772,13 @@ class TestMonitorHistoryRecheck(RecordedRuns, unittest.TestCase):
|
||||
run_attribution.notify_outputs_sealed("run", []) # no dead callback raises
|
||||
|
||||
|
||||
def folder_is_case_sensitive(folder):
|
||||
"""True when ``folder``'s volume tells names apart by case: the existing
|
||||
folder spelled with swapped case is not found (#476)."""
|
||||
head, name = os.path.split(folder)
|
||||
return not os.path.exists(os.path.join(head, name.swapcase()))
|
||||
|
||||
|
||||
class TestMonitorForeignRuns(RecordedRuns, unittest.TestCase):
|
||||
"""#247: outputs of workflows without a PromptManager node, in the monitor."""
|
||||
|
||||
@@ -942,11 +949,21 @@ class TestMonitorForeignRuns(RecordedRuns, unittest.TestCase):
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [])
|
||||
|
||||
def test_the_sealed_reference_spells_the_file_as_history_recorded_it(self):
|
||||
"""Windows: the watcher's spelling differs in case from the run's report."""
|
||||
Path(self.output, "Dataset").mkdir()
|
||||
recorded = self.image("IMG_GEN_00007_.png", subfolder="Dataset")
|
||||
"""Windows: the watcher's spelling differs in case from the run's report.
|
||||
|
||||
On a case-sensitive folder the two spellings are two files and the
|
||||
folded key joins them; on a case-insensitive one (Windows, default
|
||||
macOS) they already name one file, so only history's spelling and
|
||||
the watcher's differ (#476)."""
|
||||
seen = os.path.join(self.output, "dataset", "img_gen_00007_.png")
|
||||
Path(seen).write_bytes(b"png")
|
||||
recorded = os.path.join(self.output, "Dataset", "IMG_GEN_00007_.png")
|
||||
self.assertNotEqual(os.path.basename(recorded), os.path.basename(seen))
|
||||
if folder_is_case_sensitive(os.path.join(self.output, "dataset")):
|
||||
Path(self.output, "Dataset").mkdir()
|
||||
self.image("IMG_GEN_00007_.png", subfolder="Dataset")
|
||||
else:
|
||||
self.assertTrue(os.path.exists(recorded))
|
||||
folded = staticmethod(lambda path: os.path.realpath(path).lower())
|
||||
with mock.patch.object(ImageGenerationHandler, "_awaiting_key", folded):
|
||||
self.handler.process_new_image(seen)
|
||||
|
||||
+53
-2
@@ -18,11 +18,13 @@ from pathlib import Path
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SUBPROCESS_TIMEOUT = 60
|
||||
WATCHDOG_ENV = ("PM_TEST_HANG_LIMIT", "PM_TEST_SHUTDOWN_LIMIT")
|
||||
# Integer settings the runner reads; a bad value is a usage error.
|
||||
RUNNER_ENV = (*WATCHDOG_ENV, "PM_TEST_SLOWEST")
|
||||
|
||||
|
||||
def run_suite(folder, extra_env=None, *args, verbosity=("-v",)):
|
||||
"""Run the runner on ``folder`` and return the completed process."""
|
||||
env = {k: v for k, v in os.environ.items() if k not in WATCHDOG_ENV}
|
||||
env = {k: v for k, v in os.environ.items() if k not in RUNNER_ENV}
|
||||
# Windows writes a piped stderr in the ANSI codepage; ask for the UTF-8
|
||||
# this helper decodes, so non-ASCII values and paths survive.
|
||||
env["PYTHONIOENCODING"] = "utf-8"
|
||||
@@ -269,8 +271,57 @@ class RunSuiteTests(unittest.TestCase):
|
||||
self.assertIn("ok", result.stderr, detail)
|
||||
self.assertIn("Timeout (", result.stderr, detail)
|
||||
|
||||
def test_slowest_tests_are_listed_when_asked(self):
|
||||
self.write(
|
||||
"test_probe_slowest.py",
|
||||
"""
|
||||
import time
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_fast(self):
|
||||
pass
|
||||
|
||||
def test_slow(self):
|
||||
time.sleep(0.3)
|
||||
|
||||
def test_slower(self):
|
||||
time.sleep(0.6)
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder, {"PM_TEST_SLOWEST": "2"})
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
report = result.stderr.split("slowest 2 of 3 tests:\n", 1)
|
||||
self.assertEqual(len(report), 2, result.stderr)
|
||||
lines = [line.split() for line in report[1].strip().splitlines()[:2]]
|
||||
self.assertEqual(
|
||||
[line[1] for line in lines],
|
||||
[
|
||||
"test_probe_slowest.Probe.test_slower",
|
||||
"test_probe_slowest.Probe.test_slow",
|
||||
],
|
||||
result.stderr,
|
||||
)
|
||||
self.assertGreaterEqual(float(lines[0][0].rstrip("s")), 0.6)
|
||||
self.assertNotIn("test_fast", report[1])
|
||||
|
||||
def test_slowest_tests_are_not_listed_by_default(self):
|
||||
self.write(
|
||||
"test_probe_quiet.py",
|
||||
"""
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_ok(self):
|
||||
pass
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertNotIn("slowest", result.stderr)
|
||||
|
||||
def test_invalid_limit_is_a_usage_error(self):
|
||||
for name in WATCHDOG_ENV:
|
||||
for name in RUNNER_ENV:
|
||||
for value in ("abc", "0", "-3", "1.5", "\u00b2"):
|
||||
with self.subTest(name=name, value=value):
|
||||
result = run_suite(self.folder, {name: value})
|
||||
|
||||
+23
-11
@@ -48,6 +48,7 @@ LIBRARY_STATE = {
|
||||
"sort": "rating",
|
||||
"tags": [],
|
||||
"tag_mode": "untagged",
|
||||
"folder": "2026/Aug",
|
||||
}
|
||||
IMAGES_DEFAULTS = {
|
||||
"q": "",
|
||||
@@ -66,6 +67,7 @@ LIBRARY_DEFAULTS = {
|
||||
"sort": "newest",
|
||||
"tags": [],
|
||||
"tag_mode": "all",
|
||||
"folder": "",
|
||||
}
|
||||
# A row saved by an older build: no favorites key, a retired key, a sort
|
||||
# that is no longer offered and an over-long folder.
|
||||
@@ -242,14 +244,15 @@ class TestSavedViewsApi(ImageAPITestCase):
|
||||
for sort in ("newest", "oldest", "name", "rating"):
|
||||
await self.create(
|
||||
page="library",
|
||||
state={"sort": sort, "tags": ["a b"], "tag_mode": "all"},
|
||||
state={"sort": sort, "tags": ["a b"], "tag_mode": "all", "folder": ""},
|
||||
)
|
||||
untagged = await self.create(
|
||||
page="library", state={"tags": [], "tag_mode": "untagged"}
|
||||
)
|
||||
self.assertEqual(untagged["state"]["tag_mode"], "untagged")
|
||||
for mode in ("untagged", "any"):
|
||||
view = await self.create(
|
||||
page="library", state={"tags": [], "tag_mode": mode}
|
||||
)
|
||||
self.assertEqual(view["state"]["tag_mode"], mode)
|
||||
_, listed = await self.send("GET")
|
||||
self.assertEqual(len(listed["views"]), 21)
|
||||
self.assertEqual(len(listed["views"]), 22)
|
||||
|
||||
async def test_write_requires_the_collection_key(self):
|
||||
# #331 D140: a view saved on a collection page keeps the collection id.
|
||||
@@ -694,13 +697,24 @@ class TestSavedViewsSchema(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(normalize_state("history", {}), {})
|
||||
|
||||
def test_library_tag_mode_is_kept_and_never_paired_with_tags(self):
|
||||
# A row from before #444 has no tag_mode: it opens matching every tag.
|
||||
def test_library_tag_mode_and_folder_are_kept_with_defaults_for_old_rows(self):
|
||||
# A row from before #444 has neither key.
|
||||
old = {"q": "", "category": "", "sort": "rating", "tags": ["a"]}
|
||||
self.assertEqual(normalize_state("library", old), {**LIBRARY_DEFAULTS, **old})
|
||||
self.assertEqual(
|
||||
normalize_state("library", {"tag_mode": "any"}), LIBRARY_DEFAULTS
|
||||
normalize_state("library", {"tag_mode": "or", "folder": 3}),
|
||||
LIBRARY_DEFAULTS,
|
||||
)
|
||||
for mode in ("all", "any", "untagged"):
|
||||
self.assertEqual(
|
||||
normalize_state("library", {"tag_mode": mode})["tag_mode"], mode
|
||||
)
|
||||
with self.assertRaises(SavedViewInvalid):
|
||||
encode_state("library", {**LIBRARY_STATE, "tag_mode": "or"})
|
||||
with self.assertRaises(SavedViewInvalid):
|
||||
encode_state("library", {**LIBRARY_STATE, "folder": "x" * 1025})
|
||||
|
||||
def test_library_untagged_is_never_paired_with_tags(self):
|
||||
# Never written by this build: the tags win, as the listing refuses both.
|
||||
self.assertEqual(
|
||||
normalize_state("library", {"tags": ["a"], "tag_mode": "untagged"}),
|
||||
@@ -708,8 +722,6 @@ class TestSavedViewsSchema(unittest.TestCase):
|
||||
)
|
||||
with self.assertRaises(SavedViewInvalid):
|
||||
encode_state("library", {**LIBRARY_STATE, "tags": ["a"]})
|
||||
with self.assertRaises(SavedViewInvalid):
|
||||
encode_state("library", {**LIBRARY_STATE, "tag_mode": "none"})
|
||||
|
||||
def test_validation_errors_are_saved_view_invalid(self):
|
||||
for call in (
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
"""The freedesktop trash backend (#480): rename into a trash or refuse, never unlink.
|
||||
|
||||
Every test builds a disposable tree: a fake ``XDG_DATA_HOME`` and, through the
|
||||
backend's ``mounts`` seam, a fake "drive" mount point, so the real trash of the
|
||||
user running the tests is never touched. Moves are real renames on one
|
||||
filesystem.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from utils import trash # noqa: E402
|
||||
|
||||
POSIX = os.name == "posix" and sys.platform != "darwin"
|
||||
|
||||
|
||||
@unittest.skipUnless(POSIX, "freedesktop trash is the Linux/BSD backend")
|
||||
class FreedesktopTrashTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="pm-trash-")
|
||||
self.addCleanup(shutil.rmtree, self.tmp, True)
|
||||
self.data_home = os.path.join(self.tmp, "xdg")
|
||||
self.drive = os.path.join(self.tmp, "drive")
|
||||
self.root = os.path.join(self.drive, "output")
|
||||
self.home_root = os.path.join(self.tmp, "home-output")
|
||||
for folder in (self.data_home, self.root, self.home_root):
|
||||
os.makedirs(folder)
|
||||
self.mounts = ["/", os.path.realpath(self.drive)]
|
||||
self.backend = trash.FreedesktopBackend(
|
||||
data_home=self.data_home, mounts=lambda: list(self.mounts)
|
||||
)
|
||||
|
||||
def _file(self, folder, name="a.png", content=b"png-bytes"):
|
||||
path = os.path.join(folder, name)
|
||||
with open(path, "wb") as handle:
|
||||
handle.write(content)
|
||||
return path
|
||||
|
||||
def _info(self, trash_dir, name):
|
||||
with open(os.path.join(trash_dir, "info", name + ".trashinfo")) as handle:
|
||||
return handle.read()
|
||||
|
||||
# ── choosing the trash ────────────────────────────────────────────
|
||||
|
||||
def test_file_on_the_home_mount_goes_to_the_home_trash(self):
|
||||
path = self._file(self.home_root)
|
||||
label = self.backend.move(path, [self.home_root])
|
||||
home_trash = os.path.join(self.data_home, "Trash")
|
||||
self.assertEqual(label, trash.HOME_TRASH_LABEL)
|
||||
self.assertFalse(os.path.exists(path))
|
||||
with open(os.path.join(home_trash, "files", "a.png"), "rb") as handle:
|
||||
self.assertEqual(handle.read(), b"png-bytes")
|
||||
info = self._info(home_trash, "a.png")
|
||||
self.assertIn("[Trash Info]\n", info)
|
||||
self.assertIn("Path=" + os.path.realpath(path) + "\n", info)
|
||||
self.assertRegex(info, r"DeletionDate=\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\n")
|
||||
|
||||
def test_file_on_another_mount_uses_its_trash_dash_uid(self):
|
||||
path = self._file(self.root, "b c%.png")
|
||||
self.backend.move(path, [self.root])
|
||||
top = os.path.join(self.drive, f".Trash-{os.getuid()}")
|
||||
info_mode = stat.S_IMODE(os.lstat(top).st_mode)
|
||||
self.assertEqual(info_mode, 0o700)
|
||||
self.assertTrue(os.path.exists(os.path.join(top, "files", "b c%.png")))
|
||||
# A top-directory trash records the path relative to its top folder,
|
||||
# percent-encoded.
|
||||
self.assertIn("Path=output/b%20c%25.png\n", self._info(top, "b c%.png"))
|
||||
|
||||
def test_sticky_dot_trash_folder_is_used(self):
|
||||
shared = os.path.join(self.drive, ".Trash")
|
||||
os.mkdir(shared)
|
||||
os.chmod(shared, 0o1777)
|
||||
path = self._file(self.root)
|
||||
self.backend.move(path, [self.root])
|
||||
self.assertTrue(
|
||||
os.path.exists(os.path.join(shared, str(os.getuid()), "files", "a.png"))
|
||||
)
|
||||
|
||||
def test_dot_trash_without_sticky_bit_is_skipped(self):
|
||||
shared = os.path.join(self.drive, ".Trash")
|
||||
os.mkdir(shared, 0o777)
|
||||
os.chmod(shared, 0o777)
|
||||
path = self._file(self.root)
|
||||
self.backend.move(path, [self.root])
|
||||
self.assertFalse(os.path.exists(os.path.join(shared, str(os.getuid()))))
|
||||
top = os.path.join(self.drive, f".Trash-{os.getuid()}")
|
||||
self.assertTrue(os.path.exists(os.path.join(top, "files", "a.png")))
|
||||
|
||||
def test_dot_trash_that_is_a_link_is_skipped(self):
|
||||
elsewhere = os.path.join(self.tmp, "elsewhere")
|
||||
os.mkdir(elsewhere)
|
||||
os.chmod(elsewhere, 0o1777)
|
||||
os.symlink(elsewhere, os.path.join(self.drive, ".Trash"))
|
||||
path = self._file(self.root)
|
||||
self.backend.move(path, [self.root])
|
||||
self.assertEqual(os.listdir(elsewhere), [])
|
||||
|
||||
def test_trash_dash_uid_that_is_a_link_is_refused(self):
|
||||
elsewhere = os.path.join(self.tmp, "elsewhere")
|
||||
os.mkdir(elsewhere)
|
||||
os.symlink(elsewhere, os.path.join(self.drive, f".Trash-{os.getuid()}"))
|
||||
path = self._file(self.root)
|
||||
with self.assertRaises(trash.TrashRefused) as caught:
|
||||
self.backend.move(path, [self.root])
|
||||
self.assertEqual(caught.exception.code, trash.NO_TRASH)
|
||||
self.assertTrue(os.path.exists(path))
|
||||
self.assertEqual(os.listdir(elsewhere), [])
|
||||
|
||||
def test_trash_inside_an_output_root_is_refused(self):
|
||||
# The drive's top folder is itself an output root: its .Trash-uid
|
||||
# would be listed as images.
|
||||
path = self._file(self.root)
|
||||
with self.assertRaises(trash.TrashRefused) as caught:
|
||||
self.backend.move(path, [self.drive])
|
||||
self.assertEqual(caught.exception.code, trash.TRASH_IN_ROOT)
|
||||
self.assertTrue(os.path.exists(path))
|
||||
self.assertFalse(
|
||||
os.path.exists(os.path.join(self.drive, f".Trash-{os.getuid()}"))
|
||||
)
|
||||
|
||||
def test_home_trash_inside_an_output_root_is_refused(self):
|
||||
path = self._file(self.home_root)
|
||||
with self.assertRaises(trash.TrashRefused) as caught:
|
||||
self.backend.move(path, [self.home_root, self.tmp])
|
||||
self.assertEqual(caught.exception.code, trash.TRASH_IN_ROOT)
|
||||
self.assertTrue(os.path.exists(path))
|
||||
|
||||
def test_read_only_top_folder_is_no_trash(self):
|
||||
path = self._file(self.root)
|
||||
os.chmod(self.drive, 0o555)
|
||||
self.addCleanup(os.chmod, self.drive, 0o755)
|
||||
if os.access(self.drive, os.W_OK):
|
||||
self.skipTest("running as root: permissions are not enforced")
|
||||
self.assertEqual(self.backend.available(path, [self.root]), trash.NO_TRASH)
|
||||
with self.assertRaises(trash.TrashRefused) as caught:
|
||||
self.backend.move(path, [self.root])
|
||||
self.assertEqual(caught.exception.code, trash.NO_TRASH)
|
||||
self.assertTrue(os.path.exists(path))
|
||||
|
||||
def test_available_reports_without_writing(self):
|
||||
path = self._file(self.root)
|
||||
self.assertIsNone(self.backend.available(path, [self.root]))
|
||||
self.assertEqual(
|
||||
self.backend.available(path, [self.drive]), trash.TRASH_IN_ROOT
|
||||
)
|
||||
self.assertEqual(os.listdir(self.drive), ["output"])
|
||||
|
||||
# ── names and the no-replace rename ───────────────────────────────
|
||||
|
||||
def test_existing_name_in_the_trash_is_never_replaced(self):
|
||||
top = os.path.join(self.drive, f".Trash-{os.getuid()}")
|
||||
os.makedirs(os.path.join(top, "files"), mode=0o700)
|
||||
os.makedirs(os.path.join(top, "info"), mode=0o700)
|
||||
os.chmod(top, 0o700)
|
||||
# A stray trashed file with no info file (another tool crashed).
|
||||
stray = self._file(os.path.join(top, "files"), "a.png", b"older-file")
|
||||
path = self._file(self.root)
|
||||
self.backend.move(path, [self.root])
|
||||
with open(stray, "rb") as handle:
|
||||
self.assertEqual(handle.read(), b"older-file")
|
||||
self.assertTrue(os.path.exists(os.path.join(top, "files", "a.1.png")))
|
||||
self.assertTrue(os.path.exists(os.path.join(top, "info", "a.1.png.trashinfo")))
|
||||
self.assertFalse(os.path.exists(os.path.join(top, "info", "a.png.trashinfo")))
|
||||
|
||||
def test_existing_info_name_is_skipped(self):
|
||||
path = self._file(self.root)
|
||||
self.backend.move(path, [self.root])
|
||||
second = self._file(self.root)
|
||||
self.backend.move(second, [self.root])
|
||||
top = os.path.join(self.drive, f".Trash-{os.getuid()}")
|
||||
self.assertEqual(
|
||||
sorted(os.listdir(os.path.join(top, "files"))), ["a.1.png", "a.png"]
|
||||
)
|
||||
|
||||
def test_missing_no_replace_rename_is_no_trash(self):
|
||||
path = self._file(self.root)
|
||||
for error in (38, 22): # ENOSYS, EINVAL
|
||||
with self.subTest(error=error):
|
||||
backend = trash.FreedesktopBackend(
|
||||
data_home=self.data_home,
|
||||
mounts=lambda: list(self.mounts),
|
||||
rename=mock.Mock(side_effect=OSError(error, "no")),
|
||||
)
|
||||
with self.assertRaises(trash.TrashRefused) as caught:
|
||||
backend.move(path, [self.root])
|
||||
self.assertEqual(caught.exception.code, trash.NO_TRASH)
|
||||
self.assertTrue(os.path.exists(path))
|
||||
top = os.path.join(self.drive, f".Trash-{os.getuid()}")
|
||||
self.assertEqual(os.listdir(os.path.join(top, "info")), [])
|
||||
|
||||
def test_cross_device_rename_is_no_trash_and_never_copies(self):
|
||||
path = self._file(self.root)
|
||||
backend = trash.FreedesktopBackend(
|
||||
data_home=self.data_home,
|
||||
mounts=lambda: list(self.mounts),
|
||||
rename=mock.Mock(side_effect=OSError(18, "EXDEV")),
|
||||
)
|
||||
with mock.patch("shutil.move") as move, mock.patch("shutil.copy2") as copy:
|
||||
with self.assertRaises(trash.TrashRefused) as caught:
|
||||
backend.move(path, [self.root])
|
||||
self.assertEqual(caught.exception.code, trash.NO_TRASH)
|
||||
move.assert_not_called()
|
||||
copy.assert_not_called()
|
||||
self.assertTrue(os.path.exists(path))
|
||||
|
||||
def test_other_rename_failure_is_failed_and_removes_only_our_info(self):
|
||||
path = self._file(self.root)
|
||||
backend = trash.FreedesktopBackend(
|
||||
data_home=self.data_home,
|
||||
mounts=lambda: list(self.mounts),
|
||||
rename=mock.Mock(side_effect=OSError(5, "EIO")),
|
||||
)
|
||||
with self.assertRaises(trash.TrashRefused) as caught:
|
||||
backend.move(path, [self.root])
|
||||
self.assertEqual(caught.exception.code, trash.FAILED)
|
||||
top = os.path.join(self.drive, f".Trash-{os.getuid()}")
|
||||
self.assertEqual(os.listdir(os.path.join(top, "info")), [])
|
||||
self.assertTrue(os.path.exists(path))
|
||||
|
||||
def test_never_removes_or_moves_the_source(self):
|
||||
path = self._file(self.root)
|
||||
real_unlink = os.unlink
|
||||
|
||||
def guarded_unlink(target, *args, **kwargs):
|
||||
# Only our own .trashinfo may ever be removed.
|
||||
self.assertTrue(str(target).endswith(".trashinfo"), target)
|
||||
return real_unlink(target, *args, **kwargs)
|
||||
|
||||
backend = trash.FreedesktopBackend(
|
||||
data_home=self.data_home,
|
||||
mounts=lambda: list(self.mounts),
|
||||
rename=mock.Mock(side_effect=OSError(5, "EIO")),
|
||||
)
|
||||
with (
|
||||
mock.patch("os.unlink", guarded_unlink),
|
||||
mock.patch("os.remove", guarded_unlink),
|
||||
mock.patch("shutil.move") as move,
|
||||
mock.patch("os.rename") as rename,
|
||||
):
|
||||
with self.assertRaises(trash.TrashRefused):
|
||||
backend.move(path, [self.root])
|
||||
move.assert_not_called()
|
||||
rename.assert_not_called()
|
||||
self.assertTrue(os.path.exists(path))
|
||||
|
||||
def test_mountinfo_parsing_decodes_escapes(self):
|
||||
text = (
|
||||
"36 35 98:0 / / rw - ext4 /dev/root rw\n"
|
||||
"37 36 98:1 / /mnt/my\\040drive rw - ext4 /dev/sdb1 rw\n"
|
||||
)
|
||||
self.assertEqual(trash.parse_mountinfo(text), ["/", "/mnt/my drive"])
|
||||
|
||||
def test_mount_of_picks_the_longest_containing_mount(self):
|
||||
mounts = ["/", "/mnt", "/mnt/data"]
|
||||
self.assertEqual(trash.mount_of("/mnt/data/x/y.png", mounts), "/mnt/data")
|
||||
self.assertEqual(trash.mount_of("/mnt/datax/y.png", mounts), "/mnt")
|
||||
self.assertEqual(trash.mount_of("/home/u/y.png", mounts), "/")
|
||||
|
||||
|
||||
class UnavailableBackendTests(unittest.TestCase):
|
||||
def test_reports_no_trash_and_leaves_the_file(self):
|
||||
backend = trash.UnavailableBackend()
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = os.path.join(folder, "a.png")
|
||||
with open(path, "wb") as handle:
|
||||
handle.write(b"x")
|
||||
self.assertEqual(backend.available(path, [folder]), trash.NO_TRASH)
|
||||
with self.assertRaises(trash.TrashRefused) as caught:
|
||||
backend.move(path, [folder])
|
||||
self.assertEqual(caught.exception.code, trash.NO_TRASH)
|
||||
self.assertTrue(os.path.exists(path))
|
||||
|
||||
def test_windows_and_macos_have_no_trash_yet(self):
|
||||
for platform in ("win32", "darwin"):
|
||||
with self.subTest(platform=platform):
|
||||
self.assertIsInstance(
|
||||
trash.default_backend(platform), trash.UnavailableBackend
|
||||
)
|
||||
self.assertIsInstance(trash.default_backend("linux"), trash.FreedesktopBackend)
|
||||
|
||||
def test_windows_says_why_the_recycle_bin_is_not_used(self):
|
||||
# #500 ruling (a): the Recycle Bin can delete permanently, so it is never used.
|
||||
backend = trash.default_backend("win32")
|
||||
self.assertEqual(
|
||||
backend.available("C:/x/a.png", ["C:/x"]), trash.WINDOWS_RECYCLE
|
||||
)
|
||||
with self.assertRaises(trash.TrashRefused) as caught:
|
||||
backend.move("C:/x/a.png", ["C:/x"])
|
||||
self.assertEqual(caught.exception.code, trash.WINDOWS_RECYCLE)
|
||||
self.assertIn(trash.WINDOWS_RECYCLE, trash.CODES)
|
||||
self.assertEqual(
|
||||
trash.default_backend("darwin").available("/x", ["/"]), trash.NO_TRASH
|
||||
)
|
||||
|
||||
def test_module_functions_use_the_replaceable_backend(self):
|
||||
fake = mock.Mock()
|
||||
fake.available.return_value = "network"
|
||||
fake.move.return_value = "label"
|
||||
with mock.patch.object(trash, "BACKEND", fake):
|
||||
self.assertEqual(trash.trash_available("/x/a.png", ["/x"]), "network")
|
||||
self.assertEqual(trash.move_to_trash("/x/a.png", ["/x"]), "label")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Vendored
+27
-3
@@ -212,11 +212,15 @@ test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () =
|
||||
assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards');
|
||||
});
|
||||
|
||||
test('page workflows run each browser in two shards', () => {
|
||||
// #473: two shards split by test count put the slow concept specs in shard 1, which reached the 18-minute step limit.
|
||||
test('page workflows run each browser in three shards', () => {
|
||||
const workflows = allJobs.get('browser-workflows');
|
||||
assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']);
|
||||
assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']);
|
||||
assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/);
|
||||
assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2', '3']);
|
||||
assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/3/);
|
||||
assert.match(workflows, /^ {4}name: page workflows \(\$\{\{ matrix\.browser \}\}, shard \$\{\{ matrix\.shard \}\}\/3\)$/m);
|
||||
assert.match(workflows, /name: Page workflows in \$\{\{ matrix\.browser \}\}, shard \$\{\{ matrix\.shard \}\}\/3$/m);
|
||||
assert.doesNotMatch(workflows, /\/2\b/, 'no step still names two shards');
|
||||
assert.match(workflows, /^ {6}fail-fast: false$/m);
|
||||
assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/);
|
||||
});
|
||||
@@ -349,6 +353,26 @@ test('Windows and macOS run 3.12-3.14 on main, nightly and on demand, never on a
|
||||
assert.doesNotMatch(text, /cancel-in-progress: true|cancel-in-progress: \$/, 'no portability run is cancelled');
|
||||
});
|
||||
|
||||
// #493: the portability limit follows the rule in its comment: ceil(1.5 x the slowest of three
|
||||
// green main runs) + 3 minutes, never under 5, with those runs and times written above jobs:.
|
||||
// The times are whole-job wall times. That the runs were green and on main is the author's word.
|
||||
test('the portability limit is computed from three measured green main runs', () => {
|
||||
const text = readWorkflow('portability.yml');
|
||||
const comment = text.slice(0, text.indexOf('\njobs:'));
|
||||
const runs = [...comment.matchAll(/^# Measured \d{4}-\d{2}-\d{2} on main runs (\d+) \([0-9a-f]{7}\), (\d+) \([0-9a-f]{7}\), (\d+) \([0-9a-f]{7}\):$/gm)];
|
||||
assert.equal(runs.length, 1, 'the comment names the three runs it measured, once');
|
||||
assert.equal(new Set(runs[0].slice(1)).size, 3, 'three different runs');
|
||||
const time = '(\\d+)m([0-5]\\d)s \\(([^)]+)\\)';
|
||||
const slowest = new RegExp(`^#\\s+slowest leg: ${time} / ${time} / ${time} -> (\\d+) min$`, 'm').exec(comment);
|
||||
assert.ok(slowest, 'the comment gives the slowest leg of each run and the limit');
|
||||
const seconds = [1, 4, 7].map(i => Number(slowest[i]) * 60 + Number(slowest[i + 1]));
|
||||
const expected = Math.max(5, Math.ceil(1.5 * Math.max(...seconds) / 60) + 3);
|
||||
assert.equal(Number(slowest[10]), expected, 'the written limit follows the rule');
|
||||
assert.equal(comment.match(/-> \d+ min/g).length, 1, 'no older measurement stays beside it');
|
||||
const limit = Number(/^ {4}timeout-minutes: (\d+)\s*$/m.exec(jobs(text).get('test'))[1]);
|
||||
assert.equal(limit, expected, 'the job uses the written limit');
|
||||
});
|
||||
|
||||
test('real ComfyUI runs only outside pull requests; the other test.yml checks run on every event', () => {
|
||||
const found = testJobs();
|
||||
assert.match(found.get('comfyui'), QUEUE_ONLY_IF);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user