Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c1b5fd3fc0 | ||
|
|
917aa838cb | ||
|
|
c36ea7a0fb | ||
|
|
1153de26fc | ||
|
|
e14ec03945 | ||
|
|
990ebee79b | ||
|
|
d44cd198c4 | ||
|
|
07c16dca58 | ||
|
|
a6927489a8 | ||
|
|
53cfd8b035 | ||
|
|
567b06ac48 | ||
|
|
4888427a1e | ||
|
|
9f55f0cc79 | ||
|
|
6372bc7c63 | ||
|
|
d4d99b7e18 |
@@ -21,19 +21,15 @@ env:
|
||||
PYTHONFAULTHANDLER: "1"
|
||||
|
||||
# Time limit rule as in test.yml: ceil(1.5 x the slowest of three recent green runs on main) + 3
|
||||
# minutes, never under 5, timed as whole-job wall time. One matrix job, so the slowest leg of each
|
||||
# run counts (tests/vendor/ci-workflow.test.mjs recomputes it, #493).
|
||||
# Measured 2026-10-29 on main runs 40935337099 (dd247e3), 40938407252 (dd247e3), 40941240624 (dd247e3):
|
||||
# slowest leg: 7m24s (py3.12 windows) / 7m31s (py3.13 windows) / 7m32s (py3.14 windows) -> 15 min
|
||||
# (windows 4m56s-7m32s, macos 4m39s-6m14s)
|
||||
# native-trash (#499) has no green main runs yet: its probe, run 40949370943 (2026-10-29), took
|
||||
# about 1 minute per macOS leg, so ceil(1.5 x 2) + 3 gives it 6 minutes (step 4). Recalibrate
|
||||
# after three green runs.
|
||||
# minutes, never under 5. Measured 2026-10-26 in test.yml on runs 40257217681 (3be0ada),
|
||||
# 40240706568 (fb9134f), 40228167328 (f9835aa), before these legs moved here in #420:
|
||||
# slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 27
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -55,52 +51,5 @@ jobs:
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
# The runner's TEMP is on C:, the OS disk, where a flush took about 6 ms
|
||||
# and a test database's create-to-delete about 111 ms; on the D: work
|
||||
# disk the same took 0.5 ms and 23 ms (macOS: 19 ms). Tests write their
|
||||
# databases under TEMP, so it moves to a folder on D: (measured in #487,
|
||||
# run 40915866444). The runner's TEMP was an 8.3 short name
|
||||
# (C:\Users\RUNNER~1\...), so the new one is spelled by its short name too
|
||||
# and paths still have to be resolved. D: makes no short names by default
|
||||
# (run 40922502209), so they are switched on for it first.
|
||||
- name: Keep test files on the work disk (Windows)
|
||||
if: runner.os == 'Windows'
|
||||
shell: pwsh
|
||||
run: |
|
||||
fsutil 8dot3name set D: 0
|
||||
$folder = New-Item -ItemType Directory -Force -Path (Join-Path $env:RUNNER_TEMP "PromptManagerTests")
|
||||
$short = (New-Object -ComObject Scripting.FileSystemObject).GetFolder($folder.FullName).ShortPath
|
||||
Write-Output "test TEMP: $short (long name $($folder.FullName))"
|
||||
if ($short -eq $folder.FullName) { Write-Output "::warning::D: gave the test TEMP no 8.3 short name" }
|
||||
"TEMP=$short" >> $env:GITHUB_ENV
|
||||
"TMP=$short" >> $env:GITHUB_ENV
|
||||
|
||||
- name: Run tests
|
||||
env:
|
||||
PM_TEST_SLOWEST: "40"
|
||||
run: python -m tests.run_suite -v
|
||||
|
||||
# The real Trash on both macOS CPUs (#499 design §10.2). It runs apart from the suite, so a red
|
||||
# suite never skips the native proof.
|
||||
native-trash:
|
||||
name: native trash on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 6
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [macos-latest, macos-15-intel]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Native trash proof
|
||||
timeout-minutes: 4
|
||||
env:
|
||||
PM_NATIVE_TRASH: macos
|
||||
run: python -m unittest tests.test_trash_macos_native tests.test_trash_macos -v
|
||||
|
||||
@@ -28,8 +28,8 @@ env:
|
||||
# py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
# 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs.
|
||||
# Windows and macOS moved to portability.yml (#420), which has its own measured limit (#493);
|
||||
# this Linux-only matrix keeps 27 minutes until it is re-measured.
|
||||
# Windows and macOS moved to portability.yml (#420), which keeps this 27-minute limit; this
|
||||
# Linux-only matrix keeps it too until it is re-measured.
|
||||
# real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min
|
||||
# coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min
|
||||
# perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s,
|
||||
|
||||
@@ -36,11 +36,6 @@ concurrency:
|
||||
# 12m53s, node units and the viewer suite before 11.8m of workflows), so coverage is the tight one
|
||||
# in its 18 minutes. This holds only when the npm cache hits: a cold cache adds up to 5m to npm ci,
|
||||
# and then the job limit can still end the job before the step limit does.
|
||||
# #473: firefox/webkit shard 1/2 reached its 18-minute step limit (queue run 40866252975: webkit 622 tests
|
||||
# passed in 17m56s, 48.5 worker-minutes vs 28.0 in shard 2/2; recent shard 1/2 jobs 13.7m-18.9m, shard 2/2
|
||||
# 7.3m-12.2m), because --shard splits by test count and the slow concept specs sort into shard 1. The page
|
||||
# workflows now run in three shards per browser with the limits unchanged; the per-shard times are
|
||||
# re-measured after three green main runs.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
@@ -180,7 +175,7 @@ jobs:
|
||||
retention-days: 14
|
||||
|
||||
browser-workflows:
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/3)
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
|
||||
# Merge queue, main and manual runs only (#420); pull requests run the Chromium/touch coverage shards.
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
@@ -199,7 +194,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: [firefox, webkit]
|
||||
shard: [1, 2, 3]
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -226,9 +221,9 @@ jobs:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/3
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
|
||||
timeout-minutes: 18
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/3
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
|
||||
# Traces and the blob report (test-results/blob/) of a failed, timed-out or cancelled step.
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
|
||||
+13
-310
@@ -251,29 +251,12 @@ the process exits non-zero.
|
||||
stacks.
|
||||
- When the watchdog fires under coverage, that run's coverage data is lost; the
|
||||
job fails anyway and the dump is what matters.
|
||||
- `PM_TEST_SLOWEST=N` (off by default) lists the N slowest tests after the
|
||||
summary, each timed from its `start:` line to the next one like the hang
|
||||
limit. The Windows and macOS jobs (`portability.yml`) set it to 40, so every
|
||||
run shows where its time went (#487). The list prints only when the run
|
||||
ends: a job GitHub cancels or a watchdog exit prints none, so read the
|
||||
timestamps between `start:` lines instead.
|
||||
- On the Windows runners `TEMP` sits on the OS disk C:, where a flush took about
|
||||
6 ms and a test database's create-to-delete about 111 ms (D: 0.5 ms and 23 ms,
|
||||
macOS 19 ms). `portability.yml` points `TEMP`/`TMP` at a folder under
|
||||
`RUNNER_TEMP` on D: before the suite. It switches 8.3 short names on for D:
|
||||
and spells the folder by its short name, as the runner's own TEMP was
|
||||
(`C:\Users\RUNNER~1\...`), logs it, and warns when there is none.
|
||||
A test that waits on a fixed busy timeout, a network name (`\\server\share`:
|
||||
use `\\localhost\<missing share>`) or a default TLS context (it reads the
|
||||
Windows certificate store) costs seconds there; the slowest list shows it.
|
||||
|
||||
Every job in `test.yml`, `code-quality.yml`, `viewerjs-check.yml` and
|
||||
`portability.yml` has `timeout-minutes` (`tests/vendor/ci-workflow.test.mjs`
|
||||
checks it). The rule is ceil(1.5 x the slowest of three recent green runs on
|
||||
`main`) + 3 minutes, never under 5, timed as whole-job wall time; the three run
|
||||
ids, their times and the resulting limits are written in a comment above
|
||||
`jobs:`. For `portability.yml` the tooling test recomputes the limit from that
|
||||
comment (#493); its one matrix job takes the slowest Windows or macOS leg. The 3 minutes leave room for one watchdog firing, so
|
||||
Every job in `test.yml`, `code-quality.yml` and `viewerjs-check.yml` has
|
||||
`timeout-minutes` (`tests/vendor/ci-workflow.test.mjs` checks it). The rule is
|
||||
ceil(1.5 x the slowest of three recent green runs on `main`) + 3 minutes, never
|
||||
under 5; the three run ids, their times and the resulting limits are written
|
||||
in a comment above `jobs:`. The 3 minutes leave room for one watchdog firing, so
|
||||
the dump prints before GitHub cancels the job. A job that hits its limit, or a
|
||||
watchdog dump, is a failure like any other: read the dump, find the blocking
|
||||
call, fix it. Never raise a limit or rerun to get green; a limit changes only
|
||||
@@ -1847,9 +1830,8 @@ Inspect plan keeps D60–D66 (plan D60 → D83, D61 → D84, D65 → D88, D68
|
||||
blocks' context ("Current file image tags" in History). An empty source draws no
|
||||
section. Repeats are removed within a source only, so a linked tag that is also an
|
||||
image tag shows in both sections and each count is true. D21's filter rules are
|
||||
unchanged. History's current-file read carries the same `image_tags` facet as the Images
|
||||
read (#457), so History's inline current file and its viewer show the image tags section
|
||||
too (`test_current_inspection_carries_the_files_image_tags`). `tests/browser/inspector-concept.spec.mjs` wraps the
|
||||
unchanged. History's current-file read carries no `image_tags` facet (D88), so there
|
||||
only the linked section shows. `tests/browser/inspector-concept.spec.mjs` wraps the
|
||||
concept's row in the same sections at runtime (`withTagSections`, the D154 rule
|
||||
injected) and compares the measured sections (frame, heading, label, row); a
|
||||
mutation of the label color, the row margin or the top rule fails it. Browser tests:
|
||||
@@ -2416,102 +2398,6 @@ npm run --prefix tools/vendor test:workflows -- workspace-collections.spec.mjs c
|
||||
images-concept.spec.mjs inspector-concept.spec.mjs shell-concept.spec.mjs --project chromium --project touch
|
||||
```
|
||||
|
||||
### Deleting images (#480)
|
||||
|
||||
Images › Delete… removes the checked images (or "Select all N matching", at most 500, or
|
||||
the one file of the side panel or the viewer) from the library, and on request moves the
|
||||
files to the system trash. **No code path unlinks or overwrites a media file.**
|
||||
|
||||
- **D159** (#480, maintainer request): the concept's "Remove links…" (`danger`, app.js
|
||||
432) returns as **Delete…** in the same place (after Export metadata, before Clear), and
|
||||
as the last action of the side panel and after Download in the Images viewer. Its
|
||||
review is the concept's `maintenancePlan` (`dl.meta-grid`, Cancel and a primary) with a
|
||||
radio group first, "What should happen to the files?", nothing checked (the ruling: ask
|
||||
every time): "Remove from the library only" or "Also move the files to the trash". The
|
||||
rows follow the chosen mode (Scope, Not changed, Prompt links, Image details, Backup,
|
||||
Media on disk) and the primary is `danger`: "Remove N images from the library" / "Move
|
||||
N images to the trash". `tests/browser/images-delete-concept.spec.mjs` compares the bar
|
||||
button and the review's grid and actions with the concept.
|
||||
- **Routes.** `POST /prompt_manager/images/delete/preview {items | filter}` reviews both
|
||||
modes at once under the maintenance preview lease and answers a reviewed plan
|
||||
(`_plan_response`, operation `delete_images`): `library.count`, `trash.count`, `refused`
|
||||
and `trash_refused` (`{count, counts, rows ≤ 50}` with coded reasons), `records`,
|
||||
`orphaned_prompts` and the backup. Each target is matched to its entry in the current
|
||||
listing (`not-listed` otherwise). `POST /prompt_manager/maintenance/apply {token, mode}`
|
||||
(`mode` required for this plan, refused for any other) starts job kind `image_delete`
|
||||
(admission `records`, not cancellable; groups `trashed`, `removed`, `not_changed`,
|
||||
`errors`): sources unchanged, the promised safety backup, then per file a recheck
|
||||
(containment and fingerprint) and
|
||||
- library mode: one transaction deletes the file's records and writes its removal mark;
|
||||
- trash mode: the file moves first (`utils/trash.py`); a refused move changes nothing;
|
||||
then its records go. A records failure stops the job (`records-failed`); later files
|
||||
are "Not attempted".
|
||||
- **What goes** (`database/image_removal.py`): image links under every spelling (pair
|
||||
links by cascade, workflow documents released), image tags, auto-tag marks, pending
|
||||
undecided Needs review items, the own category, collection memberships and the
|
||||
favorite. A decided or revised review item keeps its decision and the annotation rows it
|
||||
cites. Prompts, recorded runs and the vocabulary stay. Previews are removed.
|
||||
- **Removal marks** (`image_removals`, `image_removals_meta`): a library-only removal
|
||||
hides exactly that file (locator, size, `mtime_ns`) from the workspace listing, the
|
||||
output scan, the duplicate find, Tag output images and linking. A modified or replaced
|
||||
file shows again. An explicit tag, category, favorite or collection write clears the
|
||||
mark. Older builds ignore the tables (removed files show there); restoring an older
|
||||
backup shows every file again; restoring a pre-delete backup after trash mode brings
|
||||
back records of files that are in the trash (Missing-file links cleans them up). The
|
||||
legacy gallery lists every file on disk, removed or not. Hard links have one mark per
|
||||
path, and on FAT/exFAT/SMB a same-size replacement can stay hidden.
|
||||
Since #501 the thumbnail jobs (System › Thumbnails' counts and its missing, rebuild and
|
||||
convert runs, and the previews after a scan) and the legacy `GET /scan_output_dir`
|
||||
listing skip hidden files too; an on-demand preview the legacy gallery asks for is still
|
||||
made, and a one-image auto-tag (`/autotag/single` + `/autotag/apply`) is an explicit
|
||||
write that shows the file again. System › Maintenance **Missing-file links** also
|
||||
forgets the marks that hide nothing any more (the file is gone from a folder that still
|
||||
exists, or its bytes changed), counted in its scope ("K removed images whose files are
|
||||
gone or changed") and job line, judged only under a source that is readable and lists
|
||||
something (an unplugged drive's empty mount point keeps its marks), never for a source
|
||||
no longer configured, and re-checked at apply (`plan-changed`). The sidebar's Images,
|
||||
Favorites and Needs review counts and the collection counts are read again when an
|
||||
`image_delete` job ends in the tab that followed it (`JOB_END_COUNTS`, `state.mjs`).
|
||||
Tests: `tests/test_image_removal_gaps.py`, the sidebar case in
|
||||
`tests/browser/workspace-images-delete.spec.mjs`.
|
||||
- **Trash folders inside an output root** (#501, `utils/media_folders.py`): no walker
|
||||
descends a folder below a root named `.Trash`, `.Trash-<digits>`, `.Trashes` or
|
||||
`$RECYCLE.BIN` (any case), as none descends `thumbnails`: the workspace and legacy
|
||||
listings (`_walk_media`, `_settled_entry`), the metadata scan and duplicate find
|
||||
(`_collect_media_files`), the legacy auto-tag listing (`_collect_scan_files`) and the
|
||||
delete review (`trash-folder`). The image monitor tests every folder of the new file's
|
||||
absolute path, so it also ignores outputs written into a source that itself lies
|
||||
inside such a folder. Tests: `tests/test_media_folders.py`,
|
||||
`tests/test_image_removal_gaps.py`.
|
||||
- **Trash backends** (`utils/trash.py`, no dependency): Linux and BSD implement the
|
||||
freedesktop Trash specification (home trash on the same mount, else the mount's
|
||||
`.Trash/$uid` or `.Trash-$uid`), reserve the name with `O_EXCL` and move with
|
||||
`renameat2(RENAME_NOREPLACE)`; without it, across devices, on a read-only mount or with
|
||||
a trash inside an output root the move is refused (`no-trash`, `trash-in-root`). macOS
|
||||
(#499, `utils/trash_macos.py`) calls `-[NSFileManager trashItemAtURL:resultingItemURL:error:]`
|
||||
through `ctypes` (ten fixed `objc_msgSend` prototypes, nil checks before every send, one
|
||||
autorelease pool per call, loaded on first use). It answers in the review dialog, without
|
||||
writing: `network` (not a local volume), `no-trash` (read-only, or no trash folder
|
||||
macOS would use), `cloud-sync` (iCloud Drive, `~/Library/CloudStorage`, or ubiquitous),
|
||||
`trash-in-root` (the trash folder is inside an output root, compared by `(st_dev,
|
||||
st_ino)` of every ancestor) and `no-permission` (folder not writable, an immutable or
|
||||
append flag, a foreign file in a sticky folder). A fresh volume's `.Trashes/<uid>` is
|
||||
predicted (`URLForDirectory` answers 3328/`ENOENT` until macOS makes it). A refused move
|
||||
keeps the file and logs the `NSError`; a move macOS reports done but whose place cannot be
|
||||
shown (no result URL, or one inside an output root) goes under `errors` as
|
||||
`trash-unconfirmed`, keeps its records and stops the job. Windows
|
||||
never uses the Recycle Bin (#500, maintainer ruling (a)): files larger than the bin's
|
||||
capacity are deleted permanently and IFileOperation's `PreDeleteItem` guard cannot detect
|
||||
it (native probe runs 40951037575, 40953347904), so every file answers `windows-recycle`
|
||||
("Moving files to the Recycle Bin isn't available on Windows: files larger than the bin's
|
||||
capacity would be deleted permanently."). Library-only removal works everywhere. Tests use `tests/fake_trash.py`; the
|
||||
browser test server installs it, so no test touches a real trash. The native proof,
|
||||
`tests/test_trash_macos_native.py`, runs only with `PM_NATIVE_TRASH=macos` in
|
||||
portability.yml's `native-trash` job (`macos-latest` and `macos-15-intel`): the startup
|
||||
volume, a name clash, fresh APFS and FAT32 disk images, an image used as an output root,
|
||||
a read-only image and an unwritable folder (design and probe:
|
||||
`docs/egm/499-macos-trash/`, local).
|
||||
|
||||
### Prompt library cards and recorded deviations (#249 PR E)
|
||||
|
||||
The shelf uses the shared concept toolbar and caption (E1) and the concept's
|
||||
@@ -2637,8 +2523,7 @@ Category". It is built only from the concept's Images parts:
|
||||
input[type=checkbox]`); Home, End and the arrow keys still move between titles.
|
||||
- `div#library-bulkbar` sits between the toolbar and the caption, empty until something
|
||||
is checked, then the concept's `.bulkbar`: "N selected", Set category, Clear (D151
|
||||
adds Add tags before Set category, in the Images bar's order; D152 adds Delete… before
|
||||
Clear, and Restore alone on the archived shelf). The
|
||||
adds Add tags before Set category, in the Images bar's order). The
|
||||
selection is a set of prompt ids over the loaded shelf: windowed-out cards keep their
|
||||
state, Load more keeps it, and any shelf reload (search, category, sort, tags, a
|
||||
write) and leaving the page clear it. Clicking a checkbox toggles it and sets the
|
||||
@@ -2719,97 +2604,6 @@ during a write); `tests/browser/library-concept.spec.mjs` (the bar comparison no
|
||||
Add tags); units `tests/js/workspace-library-selection.test.js`; Python
|
||||
`tests/test_bulk_library_tags.py`. A Python change: ComfyUI needs a restart.
|
||||
|
||||
### Deleting and archiving selected prompts (#443)
|
||||
|
||||
**D152** (beyond the concept, on D148's bar; the concept archives a prompt from its recipe
|
||||
editor's "Archive…", `app.js` 976, reviewed in `maintenancePlan`, 953–958, and maps the legacy
|
||||
delete to a "reference-safe archive", 783): the Prompt library deletes or archives the
|
||||
selected prompts in one reviewed step, replacing the legacy admin page's bulk and single
|
||||
Delete, which refuses a whole batch when any prompt belongs to a saved pair (maintainer
|
||||
rulings of 2026-10-28):
|
||||
|
||||
- The bar reads "N selected", [Select all N loaded], Add tags, Set category, **Delete…**
|
||||
(the concept's `danger` class, as its Images bar's "Remove links…"), Clear.
|
||||
- Delete… posts `{prompt_ids, expected_epoch}` to `POST
|
||||
/prompt_manager/library/delete/preview`, which classifies each id in this order: not
|
||||
found (missing or not a Library parent), protected (tagged `__protected__`), already
|
||||
archived, archive (the positive or negative of any saved pair, archived pair or not),
|
||||
delete (everything else). From the press until the review opens, every bar action, Clear
|
||||
too, is `aria-disabled` and Delete… reads "Reviewing…" (a double press sends one preview).
|
||||
- The review is the shared record-changing review (`openReviewDialog`), titled "Delete
|
||||
selected prompts", "Archive selected prompts" or "Delete or archive selected prompts", its
|
||||
`dl.meta-grid` rows Delete permanently ("N prompts not in a saved pair") and Archive ("N prompts in a saved
|
||||
pair: hidden, restorable", never negative variants: a pair's prompt with none is archived too) (always), Protected (kept) with up to
|
||||
three titles, Already archived, No longer exist, and, when something is deleted, Image
|
||||
links removed and Backup ("Create and verify before deleting", or "Off: no safety backup
|
||||
(System › Backup & export › Recovery policy)"); then `p.small` sentences: why paired
|
||||
prompts are archived and how to restore them, what a delete removes and that it cannot be
|
||||
undone except by restoring the safety backup (which returns the whole library to that
|
||||
moment; kept among the last 10), that a later scan, new image or node run with the same
|
||||
text adds a deleted prompt again, and how to unprotect. The primary is `danger`: "Delete D
|
||||
prompts", "Archive A prompts" or "Delete D and archive A prompts". When nothing can change,
|
||||
or the disk has no room for the promised backup, the review is close-only with the reason.
|
||||
The concept's "Review maintenance plan" title, its demo note, Scope/Protected/Media on
|
||||
disk rows and `ca-primary` start are replaced by these.
|
||||
- Start posts the reviewed lists `{archive_ids, delete_ids, expected_epoch, backup}` to `POST
|
||||
/prompt_manager/library/delete` while Delete… reads "Deleting…" (or "Archiving…"). The
|
||||
server takes a `records` hold ("Delete prompts (Library)", refused 409 `busy` beside a scan,
|
||||
restore, maintenance, move or Optimize), checks the disk (507 `insufficient-space`) and,
|
||||
when a delete remains and the reviewed `backup` was "on" or the policy is on now, takes a
|
||||
verified safety backup (500 `backup-failed`, nothing changed). One `BEGIN IMMEDIATE`
|
||||
transaction then deletes only reviewed deletes still unpaired whose prompt row, tags,
|
||||
title and image links are unchanged since the backup (exactly the legacy rows: image
|
||||
link rows, cascaded tags, Library state and pair links of images, released workflow
|
||||
documents; files never), archives the paired ones and any reviewed delete now paired, and
|
||||
counts the rest (`already_archived`, `not_found`, `changed`). Archiving writes a
|
||||
`prompt_archive` row, the prompt's `updated_at` and, when its stored hash is stale or
|
||||
missing and no other row holds the canonical one, the canonical hash, so scans, the
|
||||
monitor and node saves find the archived row instead of adding a copy. A changed epoch
|
||||
answers 409 `storage-changed`; nothing is written.
|
||||
- Outcomes: "Deleted D prompts." "Archived A prompts." with already archived, no longer
|
||||
exist, changed and kept-protected sentences and "Safety backup saved: NAME (System ›
|
||||
Backup & export)."; the shelf, categories, cached Library views and sidebar counts reload.
|
||||
Storage restored: "The library was restored or moved since this page loaded. Nothing was
|
||||
deleted or archived."; busy: "NAME is running. Nothing was changed; try again when it
|
||||
finishes."; a failed backup or a short disk: the server's sentence; no answer, 502, 503 or
|
||||
504: "The server did not confirm the change. The Library was reloaded: check the shelf and
|
||||
Show archived prompts."; other refusals: "The selected prompts could not be changed. Try
|
||||
again.", the selection kept.
|
||||
- **Show archived prompts (N)** (heading actions, before Export, `aria-pressed`; N the
|
||||
archived prompts matching the filters, `archived_total`; the shelf counterpart of #235's
|
||||
Show archived) lists only archived prompts (`GET /prompt_manager/library?shelf=archived`,
|
||||
a cursor never crosses shelves), each card with an "Archived" `.chip`, the caption "N
|
||||
archived prompts · M negative variants hidden with them", the bar "N selected", Restore,
|
||||
Clear, and "No archived prompts." / "No archived prompts match." when empty. The pressed
|
||||
state stays with the shelf visit, never in a saved view; Browse and a saved view open the
|
||||
shelf. Restore posts `{prompt_ids, expected_epoch}` to `POST
|
||||
/prompt_manager/library/restore` (every requested archived id, protected or not): "Restored
|
||||
N prompts to the Prompt library."
|
||||
- An archived prompt's page shows the concept's `.notice` ("Archived" / "This prompt is archived
|
||||
because it is in a saved pair. It and its N negative variants are hidden from the Prompt library; nothing was
|
||||
removed.", or "It is hidden from…" when it has none) with Restore;
|
||||
everything else on the page works. New prompt or Add negative on an archived prompt's
|
||||
text keeps it archived and toasts "This text is an archived prompt. Restore it to show it
|
||||
in the Prompt library again."
|
||||
- Archived prompts: hidden from the shelf, its search, `total`, the sidebar count, Tags &
|
||||
categories' prompt counts (a value only archived prompts hold stays listed with 0) and
|
||||
System › Overview and the restore review's positive counts; still counted by rename and
|
||||
delete impacts and usage statistics; still shown as another prompt's negative variant, on
|
||||
recorded runs and Images, by the legacy pages and the ComfyUI node's search; still reached
|
||||
by run counting, scans and the monitor; never pruned by maintenance (a new protection,
|
||||
"archived in the Library"); exported with `archived: true`. Older builds ignore the table
|
||||
and show them again; the rows stay.
|
||||
- Deviation from legacy: a selected `__protected__` prompt is never deleted or archived (the
|
||||
legacy delete removed it); it is listed in the review and the toast.
|
||||
|
||||
Tests: `tests/browser/workspace-library-delete.spec.mjs` (mixed review and result on the
|
||||
server, Cancel, Show archived prompts and Restore from the bar and the notice, double press,
|
||||
failed backup, busy, unconfirmed, restore fence); `tests/browser/library-delete-concept.spec.mjs`
|
||||
(the review against the concept's `maintenancePlan`, the bar's Delete… against the concept's
|
||||
"Remove links…"); units `tests/js/workspace-library-delete.test.js`; Python
|
||||
`tests/test_prompt_archive_schema.py`, `tests/test_bulk_library_delete.py`,
|
||||
`tests/test_library_delete_routes.py`. A Python change: ComfyUI needs a restart.
|
||||
|
||||
### No 1–5 prompt rating in the workspace (#447)
|
||||
|
||||
**D153** (maintainer ruling 2026-10-28: the star is only for favorites; the concept keeps
|
||||
@@ -3074,12 +2868,6 @@ mapped differences"):
|
||||
change within the workspace waits the same way: the shell renders the next route once a
|
||||
video still starting on Inspect settles (`leaveGuard`; the latest route wins). Route
|
||||
changes at `loadstart` + 0–7 ms hung 4–5 of 120 pages before and 0 of 120 after.
|
||||
The shared viewer's own video gets the same guard (#483): a new selection while the
|
||||
viewer D155 opened is still starting its video leaves the viewer open, untouched, until
|
||||
that video reports metadata or an error (or 15 s), then closes it; the URL is revoked
|
||||
after that, and a viewer asked for meanwhile (the new file's D155 open) opens then. The
|
||||
merge queue hung once without it (run 40903238387: the PNG selection 12 ms after the
|
||||
inline video showed, with the viewer's new video still starting).
|
||||
A real page leave cannot wait. With no page code releasing anything, navigating
|
||||
away hung 8 of 120 pages and reload 22 of 120 (3 and 8 of 120 with the page's own
|
||||
release), so that WebKit teardown stays a known limitation; dispose still releases at
|
||||
@@ -3511,8 +3299,7 @@ until this browser dismissed the 3.2.0 update notice (`pm_update_notice_dismisse
|
||||
the legacy admin page's key); Later dismisses it without scanning, and Review scan
|
||||
dismisses it as the legacy Scan does. "+ Add source" asks validate-source, then adds
|
||||
the folder to the host draft of `gallery_root_paths`; Remove takes one out; "Watch for
|
||||
new outputs" (`monitoring_enabled`), "Save prompts from other workflows"
|
||||
(`save_foreign_runs`, D158) and Worker threads (`worker_threads`) are host
|
||||
new outputs" (`monitoring_enabled`) and Worker threads (`worker_threads`) are host
|
||||
drafts too, and Save writes them with the revision. A save the server refuses because a
|
||||
scan runs toasts "Another job is running: <title>." with a link to that job (D57) and
|
||||
keeps the drafts; a save refused for one source says the server's message as the status
|
||||
@@ -3761,7 +3548,7 @@ the connection's `total_changes` is 0 at close and the data reads back unchanged
|
||||
`-shm` bytes are not compared: every WAL reader rewrites its read marks there, so no
|
||||
reader can keep it byte-identical (`test_health_check_never_writes`).
|
||||
|
||||
Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107, D109–D110 and D111–D114; D121–D134 and D158 assigned). D94,
|
||||
Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107, D109–D110 and D111–D114; D121–D134 assigned). D94,
|
||||
the per-section hand-off panel, was retired by S12 (#329): every section is native.
|
||||
|
||||
- **D92** (shell): no "Sample configuration" chip. The Save status is truthful: "No
|
||||
@@ -4034,8 +3821,8 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
- **D104** (Backup & export, S9, Q4): Recovery policy keeps the concept's "Proposed" chip.
|
||||
"Create a safety backup before maintenance" is the server's `safety_backup`
|
||||
(`maintenance.safety_backup`) policy. Its line says what it does ("Keep a verified copy
|
||||
of the database on disk before maintenance, cleanups, LoRA imports and prompt deletes
|
||||
change records. A restore always keeps one."); the concept's "Preview a verified backup before applying
|
||||
of the database on disk before maintenance, cleanups and LoRA imports change records. A
|
||||
restore always keeps one."); the concept's "Preview a verified backup before applying
|
||||
operations that change records." describes nothing the server does. It is drafted behind the Save row and saved with
|
||||
`expected_revision` (a configuration changed elsewhere is the conflict status); it is
|
||||
disabled with "Server settings could not be read." when the read failed. "Scheduled
|
||||
@@ -4281,26 +4068,6 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
Cancel-only review matches the concept review shell` (system-tagging-concept.spec) and
|
||||
the batch tests of workspace-system-tagging.spec; the mutation check renames each grid
|
||||
label in turn.
|
||||
- **D158** (Sources, #247 D-Q4): the concept has no switch for prompts from workflows
|
||||
without a PromptManager node; the maintainer chose a visible switch over a config key.
|
||||
A `.setting-row` in the concept's toggle markup follows "Watch for new outputs":
|
||||
"Save prompts from other workflows" (`#foreign-runs`, host key `save_foreign_runs`,
|
||||
config.json `gallery.monitoring.save_foreign_runs`, default on), "New outputs of
|
||||
workflows without a PromptManager node add their prompt to the library when it is
|
||||
certain.". It is saved with the revision like the other host rows and applied at once
|
||||
(the monitor reads it for every file), so a save names no restart. While "Watch for
|
||||
new outputs" is drafted off, a `p.small` under the description says "Applies while
|
||||
Watch for new outputs is on." (the switch's `aria-describedby`; the value is kept) and
|
||||
the switch is disabled, except while a source still reads "watching until restart"
|
||||
(D128): the running observer still delivers files then, so the value can be set. The
|
||||
server stores no foreign prompt while `monitoring_enabled` is saved off, whatever the
|
||||
observer does.
|
||||
Pinned by `sources matches the concept` (the mapping inserts the row after the
|
||||
monitoring row), `sources comparison detects mutations` (a missing row and a changed
|
||||
label are caught), `D158: Save prompts from other workflows saves save_foreign_runs
|
||||
live; off while monitoring is off (#247)` (keyboard on desktop, a tap on touch) and
|
||||
`D158: saved off while the monitor still watches until a restart, the foreign row can
|
||||
still be set (D128)` (workspace-system-sources.spec) and the system-sources node test.
|
||||
- **D109** (Logs, S12, #329): the concept's Logs surface on the live server. The toolbar
|
||||
keeps "Level", Refresh and the download (renamed "Download log": the active log,
|
||||
`/logs/download/prompt_manager.log`, through the api client), with all six minimum
|
||||
@@ -6228,70 +5995,6 @@ Tests: `tests/test_file_prompt.py` (one test per ladder step and junk class),
|
||||
saver, a legacy-linked file left as it is, `_BatchHistory.recorded`) and
|
||||
`tests/test_recorded_saver_anchor.py` `TestInspectionResolution`.
|
||||
|
||||
**In the monitor.** An output whose workflow has no positive PromptManager node
|
||||
(`_find_prompt_from_metadata` says `foreign`) used to be logged and skipped. It now
|
||||
goes through the same ladder with the `MONITOR` policy (`utils/foreign_runs.py`, the
|
||||
handler's `_ingest_foreign`, `_recheck_foreign`, `_store_foreign`):
|
||||
|
||||
- **At settle.** The ladder runs without history. Certain (`pair` or `positive`): the
|
||||
prompt is stored and the file linked at once. `history` (an undecided graph inside
|
||||
ComfyUI's output folder): the file is registered in its own waiting set, then the
|
||||
ladder runs again at once with the file's recorded output, because the run may have
|
||||
been sealed before the file settled. When that lookup finds a record, its answer is
|
||||
final: certain is stored, anything else is logged "No certain prompt for <file>;
|
||||
left unlinked" and the file stops waiting (no seal is coming). When it finds none,
|
||||
only the inspector's or the recorded saver's answer is stored; anything else (the
|
||||
legacy floor included) keeps waiting for the re-check. Anything else at the first
|
||||
pass is logged "No certain prompt for <file>; left unlinked".
|
||||
- **After the run ends.** When the run's outputs are sealed, each waiting foreign file
|
||||
gets one re-check (`ForeignRecheck(reference)`) with the sealed output reference,
|
||||
so history is asked with the spelling it recorded (a Windows folder whose case
|
||||
differs from the disk still links). Not certain: "No certain prompt for <file>
|
||||
after its run ended; left unlinked". An ambiguous inspector answer is never given
|
||||
the legacy floor in the monitor (D-Q3); an untraceable one still is.
|
||||
- **Waiting sets.** `_awaiting_foreign` and `_foreign_due` sit beside the keyless sets
|
||||
with the same bounds (1000 files, one hour) but are separate, so a large foreign run
|
||||
never evicts a waiting keyless file. An expired entry is logged "No recorded run
|
||||
for <file>; left unlinked", an evicted one (waiting or due) "Too many files wait for
|
||||
their run; <file> left unlinked", both after the lock is released. A due re-check
|
||||
is dropped only when it was scheduled, never a newer reference for the same file. `awaiting_foreign()` lists them;
|
||||
`shutdown()` and `close()` clear them, and a closed handler re-checks nothing.
|
||||
- **Store.** One write through `survive_switch` (rerun after a Move library): a pair
|
||||
through `ingest_resolved_pair(..., marks=RUN_MARKS)`, a positive through
|
||||
`store_scanned_prompt_or_raise(..., RUN_MARKS)`. A created prompt carries category
|
||||
`scanned`, tag `auto-scanned` and the note `Saved from a workflow run: <file>`; a
|
||||
file linked to another prompt is `conflict` and keeps its link. Logged at info
|
||||
"Foreign run: <status> prompt <id> for <file> (<source>)"; a failure is logged and
|
||||
the file stays unlinked.
|
||||
- **Never.** No tracker FIFO entry is read or consumed, nothing is counted (run count
|
||||
stays 0, no `last_used_at`), and nothing is queued for Needs review.
|
||||
- **The setting.** `GalleryConfig.SAVE_FOREIGN_RUNS` (`gallery.monitoring.
|
||||
save_foreign_runs`, default true, only a bool applies) is read for every file, at
|
||||
settle, at the re-check and again just before the store, together with
|
||||
`MONITORING_ENABLED`: either saved off stops foreign writes at once (the running
|
||||
observer watches until a restart, D128); off logs "No PromptManager prompt in
|
||||
workflow, skipping image" as before. System › Sources & scans "Save prompts from other workflows" (D158)
|
||||
saves it through `POST /prompt_manager/settings` with no restart field; the
|
||||
diagnostics report lists it.
|
||||
|
||||
Tests: `tests/test_run_attribution.py` `TestMonitorForeignRuns` (dataset branches
|
||||
linked after the seal, a seal before settle, a name two runs reported, the setting
|
||||
off at settle and at the re-check, a plain workflow linked at settle with no run
|
||||
count and the tracker queue untouched, the sealed reference's spelling, the separate
|
||||
bound, expiry, a seal while the pending set is full, shutdown, a failed store, no
|
||||
record at settle keeps waiting, a record without an answer ends the wait, monitoring
|
||||
saved off, the setting read before the store, a seal between registering and the
|
||||
settle lookup, a newer due reference kept, evictions logged, one link per file),
|
||||
`tests/test_image_linking.py` `TestForeignWorkflows`, `tests/test_image_monitor_singleton.py`
|
||||
(close drops a foreign re-check), `tests/test_config.py`,
|
||||
`tests/test_settings_sources.py` `TestSaveForeignRunsSetting`, and the real-host
|
||||
`tests/comfyui/image-attribution.spec.mjs` case "an output of a workflow without a
|
||||
PromptManager node creates and links its prompt; a second run reuses it" (ComfyUI's
|
||||
own `CLIPTextEncode`, `KSampler` and `SaveImage`), and "each branch of a
|
||||
shared-prefix workflow without a PromptManager node links to its own prompt after the
|
||||
run" (two branches, one save prefix: the recorded saver names each file's branch).
|
||||
The fixture route's GET also lists `foreign`, the waiting foreign files.
|
||||
|
||||
## Linked prompt text (#343)
|
||||
|
||||
A link drives `text` on every PromptManager node, and `negative_text` on the Pair,
|
||||
@@ -6572,7 +6275,7 @@ The versioned public reads are:
|
||||
| `GET /prompt_manager/library/{positive_id}/history/pairs/{pair_id}/newest` | The pair's newest recorded run for the history run dot (#249 D26): `{run: {run_id, anchor_id, context_id}}` or `{run: null}` |
|
||||
| `GET /prompt_manager/history/runs/{run_id}` | Frozen fields, per-field provenance, capture and outcome for a known UUID |
|
||||
| `GET /prompt_manager/history/runs/{run_id}/outputs` | Ordered host-reported occurrences, including repeated locations |
|
||||
| `GET /prompt_manager/history/runs/{run_id}/output/current/{media,thumbnail,inspection}` | Authorized current file at that reported location, using the returned epoch/scope URL; `inspection` carries the shared inspector envelope including the file's `image_tags` facet (#457) |
|
||||
| `GET /prompt_manager/history/runs/{run_id}/output/current/{media,thumbnail,inspection}` | Authorized current file at that reported location, using the returned epoch/scope URL |
|
||||
|
||||
The newest-pair read takes no query. It answers 404 when the positive is not an
|
||||
eligible saved prompt or the pair is not one of its pairs. It walks the positive's exact
|
||||
|
||||
@@ -40,7 +40,6 @@ from .annotation_schema import (
|
||||
METADATA_TABLE,
|
||||
)
|
||||
from .autotag_schema import AUTOTAG_TABLE
|
||||
from .image_removal_schema import clear_marks
|
||||
from .library_commands import _UUID, LibraryCommandError, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .vocabulary import validate_vocabulary_tag_name
|
||||
@@ -237,9 +236,6 @@ class AnnotationOperations:
|
||||
|
||||
@staticmethod
|
||||
def _annotation_owner(conn: sqlite3.Connection, locator: str, now: str) -> str:
|
||||
# A file with image tags or Needs review state is in the library: a
|
||||
# removal mark never hides it (#480).
|
||||
clear_marks(conn, [locator])
|
||||
row = conn.execute(_OWNER, (locator,)).fetchone()
|
||||
if row is not None:
|
||||
return row[0]
|
||||
|
||||
@@ -7,11 +7,6 @@ from typing import Callable, ContextManager, Iterator
|
||||
|
||||
try:
|
||||
from ..utils.execution_history_events import RunFinish, encode_command
|
||||
from ..utils.hashing import (
|
||||
generate_prompt_hash,
|
||||
stored_prompt_text,
|
||||
stored_prompt_text_sql,
|
||||
)
|
||||
from ..utils.execution_contexts import (
|
||||
CAPTURE_REASONS,
|
||||
ContextBundle,
|
||||
@@ -20,11 +15,6 @@ try:
|
||||
)
|
||||
except ImportError:
|
||||
from utils.execution_history_events import RunFinish, encode_command
|
||||
from utils.hashing import (
|
||||
generate_prompt_hash,
|
||||
stored_prompt_text,
|
||||
stored_prompt_text_sql,
|
||||
)
|
||||
from utils.execution_contexts import (
|
||||
CAPTURE_REASONS,
|
||||
ContextBundle,
|
||||
@@ -144,8 +134,7 @@ def _insert_exact(
|
||||
|
||||
def _role_hints(role: str) -> str:
|
||||
return (
|
||||
f"SELECT r.{role}_prompt_id, tx.text_value, "
|
||||
f"{stored_prompt_text_sql('tx.text_value')} "
|
||||
f"SELECT r.{role}_prompt_id, tx.text_value "
|
||||
"FROM execution_run_contexts r "
|
||||
"JOIN execution_contexts c ON c.context_id = r.context_id "
|
||||
f"JOIN execution_prompt_snapshots s ON s.snapshot_id = c.{role}_snapshot_id "
|
||||
@@ -157,78 +146,57 @@ def _role_hints(role: str) -> str:
|
||||
|
||||
# Set form of the read-time prompt-hint rule in get_execution_contexts():
|
||||
# a stored role hint is live only in the current storage epoch, and only when
|
||||
# the hinted prompt is the one match of _prompt_hint: by text (the raw
|
||||
# snapshot text or its stored form, outer whitespace stripped, #438), or, only
|
||||
# when no prompt has either text, by dedup hash (case folded too, #460).
|
||||
# The hash is a fallback, so a hint #438 kept is never made ambiguous.
|
||||
# Needs register_prompt_functions (pm_prompt_hash). The hint rows are the
|
||||
# outer loop (CROSS JOIN), so each prompts lookup is by index.
|
||||
# its exact raw snapshot text matches exactly one prompt, which is the hinted
|
||||
# one (_prompt_hint). Evaluated once per statement, never per prompt row.
|
||||
# Defines the CTE ``live_hint_prompts(prompt_id)``; prefix it with WITH.
|
||||
LIVE_HINT_CTES = (
|
||||
"stored_hints(prompt_id, raw, stored) AS ("
|
||||
"stored_hints(prompt_id, raw) AS ("
|
||||
+ _role_hints("positive")
|
||||
+ " UNION "
|
||||
+ _role_hints("negative")
|
||||
+ "), hint_matches(prompt_id, raw, stored, match_id) AS ("
|
||||
"SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h "
|
||||
"CROSS JOIN prompts p ON p.text COLLATE BINARY IN (h.raw, h.stored) "
|
||||
"UNION ALL "
|
||||
"SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h "
|
||||
"CROSS JOIN prompts p ON p.hash = pm_prompt_hash(h.raw) "
|
||||
"WHERE NOT EXISTS (SELECT 1 FROM prompts q "
|
||||
"WHERE q.text COLLATE BINARY IN (h.raw, h.stored))"
|
||||
+ "), unique_hint_texts(prompt_id, raw) AS ("
|
||||
"SELECT MIN(id), text COLLATE BINARY FROM prompts "
|
||||
"WHERE text COLLATE BINARY IN (SELECT raw FROM stored_hints) "
|
||||
"GROUP BY text COLLATE BINARY HAVING COUNT(*) = 1"
|
||||
"), live_hint_prompts(prompt_id) AS ("
|
||||
"SELECT prompt_id FROM hint_matches GROUP BY prompt_id, raw, stored "
|
||||
"HAVING COUNT(*) = 1 AND MIN(match_id) = prompt_id)"
|
||||
"SELECT h.prompt_id FROM stored_hints h JOIN unique_hint_texts u "
|
||||
"ON u.prompt_id = h.prompt_id AND u.raw = h.raw COLLATE BINARY)"
|
||||
)
|
||||
LIVE_HINT_PROMPT_IDS_SQL = (
|
||||
"WITH " + LIVE_HINT_CTES + " SELECT DISTINCT prompt_id FROM live_hint_prompts"
|
||||
)
|
||||
|
||||
_BY_TEXT = "SELECT id FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 2"
|
||||
|
||||
|
||||
def _text_matches(conn: sqlite3.Connection, raw: str) -> list:
|
||||
return [row[0] for row in conn.execute(_BY_TEXT, (raw, stored_prompt_text(raw)))]
|
||||
|
||||
|
||||
def _prompt_hint(conn: sqlite3.Connection, raw: str | None) -> int | None:
|
||||
"""The one prompt that stores ``raw`` (#438), else the one it counts toward.
|
||||
|
||||
By text first: ``raw`` or ``raw.strip()`` (the exact form keeps hints of
|
||||
older rows that kept outer whitespace). Only when no prompt has either
|
||||
text, by the dedup hash, which also folds case (#460).
|
||||
"""
|
||||
if raw is None:
|
||||
return None
|
||||
matches = _text_matches(conn, raw)
|
||||
if not matches:
|
||||
matches = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT id FROM prompts WHERE hash=? LIMIT 2",
|
||||
(generate_prompt_hash(raw),),
|
||||
)
|
||||
]
|
||||
return matches[0] if len(matches) == 1 else None
|
||||
rows = conn.execute(
|
||||
"SELECT id FROM prompts WHERE text=? COLLATE BINARY LIMIT 2", (raw,)
|
||||
).fetchall()
|
||||
return rows[0][0] if len(rows) == 1 else None
|
||||
|
||||
|
||||
def _hints(
|
||||
conn: sqlite3.Connection, positive: str | None, negative: str | None
|
||||
) -> tuple:
|
||||
pos_id, neg_id = _prompt_hint(conn, positive), _prompt_hint(conn, negative)
|
||||
stored = None if negative is None else stored_prompt_text(negative)
|
||||
pair_id = None
|
||||
# A known negative that names no prompt because several share its text
|
||||
negative_ambiguous = (
|
||||
stored is not None and neg_id is None and bool(_text_matches(conn, negative))
|
||||
negative is not None
|
||||
and neg_id is None
|
||||
and conn.execute(
|
||||
"SELECT 1 FROM prompts WHERE text=? COLLATE BINARY LIMIT 1", (negative,)
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
if pos_id is not None and stored is not None and not negative_ambiguous:
|
||||
if pos_id is not None and negative is not None and not negative_ambiguous:
|
||||
rows = conn.execute(
|
||||
"SELECT pp.id FROM prompt_pairs pp "
|
||||
"SELECT pp.id FROM prompt_pairs pp LEFT JOIN prompts n ON "
|
||||
"n.id=pp.negative_prompt_id "
|
||||
"WHERE pp.positive_prompt_id=? AND ((pp.negative_prompt_id IS NULL AND "
|
||||
"?='') OR pp.negative_prompt_id=?) LIMIT 2",
|
||||
(pos_id, stored, neg_id),
|
||||
"?='') "
|
||||
"OR (pp.negative_prompt_id=? AND n.text=? COLLATE BINARY)) LIMIT 2",
|
||||
(pos_id, negative, neg_id, negative),
|
||||
).fetchall()
|
||||
if len(rows) == 1:
|
||||
pair_id = rows[0][0]
|
||||
|
||||
+7
-3
@@ -23,10 +23,10 @@ from pathlib import PurePosixPath, PureWindowsPath
|
||||
from urllib.parse import quote
|
||||
|
||||
try:
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
from .models import normalize_image_path
|
||||
except ImportError:
|
||||
from utils.hashing import register_prompt_functions
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from database.models import normalize_image_path
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.health")
|
||||
@@ -125,8 +125,12 @@ def _foreign_keys(conn, _roots):
|
||||
)
|
||||
|
||||
|
||||
def _prompt_hash(text):
|
||||
return generate_prompt_hash(text) if isinstance(text, str) else None
|
||||
|
||||
|
||||
def _canonical_hashes(conn, _roots):
|
||||
register_prompt_functions(conn) # pm_prompt_hash, shared with history (#460)
|
||||
conn.create_function("pm_prompt_hash", 1, _prompt_hash, deterministic=True)
|
||||
return _counted(
|
||||
f"Prompt {row[0]}: the stored hash does not match its text"
|
||||
for row in conn.execute(
|
||||
|
||||
+28
-66
@@ -15,11 +15,6 @@ from .execution_history import HistoryEpochChanged
|
||||
from .execution_outputs import _capture, _REFERENCE_COLUMNS
|
||||
from .library import _ELIGIBLE
|
||||
|
||||
try:
|
||||
from ..utils.hashing import prompt_key, stored_prompt_text
|
||||
except ImportError:
|
||||
from utils.hashing import prompt_key, stored_prompt_text
|
||||
|
||||
RunKey = tuple[int, str, int, str]
|
||||
|
||||
|
||||
@@ -105,43 +100,14 @@ _ORDER = (
|
||||
)
|
||||
# Stale-hint rows a newest-pair lookup inspects before it reports none.
|
||||
_PAIR_RUN_CANDIDATES = 16
|
||||
# The recorded texts of one scope, as a JSON array parameter (_scope_text_ids).
|
||||
_IN_SCOPE = "IN (SELECT value FROM json_each(?))"
|
||||
_MATCHING = (
|
||||
"SELECT DISTINCT rc.run_id FROM execution_prompt_snapshots s "
|
||||
"JOIN execution_contexts c ON c.positive_snapshot_id=s.snapshot_id "
|
||||
"JOIN execution_run_contexts rc ON rc.context_id=c.context_id "
|
||||
f"WHERE s.raw_text_id {_IN_SCOPE}"
|
||||
"WHERE s.raw_text_id=?"
|
||||
)
|
||||
|
||||
|
||||
def _scope_text_ids(conn: sqlite3.Connection, key: str) -> str:
|
||||
"""Every recorded raw text whose prompt key is ``key``, as a JSON array.
|
||||
|
||||
A run keeps its exact raw text, while the Library saves and counts it as
|
||||
the prompt with its key (outer whitespace #438, case #460). The values are
|
||||
compared, never only their digests; ``pm_prompt_key`` is Python's key.
|
||||
"""
|
||||
return json.dumps(
|
||||
[
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
# Only snapshot raw texts: effective and affix texts are skipped
|
||||
"SELECT t.text_id FROM (SELECT DISTINCT raw_text_id AS text_id "
|
||||
"FROM execution_prompt_snapshots) s CROSS JOIN "
|
||||
"execution_context_texts t WHERE t.text_id=s.text_id AND "
|
||||
"pm_prompt_key(t.text_value)=? COLLATE BINARY",
|
||||
(key,),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _scope_text_id(raw: str | None) -> str | None:
|
||||
"""The scope digest of a raw text: that of its prompt key."""
|
||||
return None if raw is None else content_digest("text", (prompt_key(raw),))
|
||||
|
||||
|
||||
def _limit(limit: int) -> None:
|
||||
if type(limit) is not int or not 1 <= limit <= 100:
|
||||
raise ValueError("History limit must be an integer from 1 to 100")
|
||||
@@ -289,15 +255,8 @@ class HistoryReadOperations:
|
||||
if saved is None:
|
||||
return None
|
||||
text = saved["text"]
|
||||
raw_id = _scope_text_id(text)
|
||||
# A position from before #460 names the scope by the stripped text
|
||||
# alone; it is the same scope, so a page open across the upgrade resumes.
|
||||
legacy_id = (
|
||||
None
|
||||
if text is None
|
||||
else content_digest("text", (stored_prompt_text(text),))
|
||||
)
|
||||
if after is not None and after.raw_text_id not in (raw_id, legacy_id):
|
||||
raw_id = None if text is None else content_digest("text", (text,))
|
||||
if after is not None and after.raw_text_id != raw_id:
|
||||
raise ValueError("History position text scope changed")
|
||||
group = {
|
||||
"positive_id": positive_id,
|
||||
@@ -314,11 +273,18 @@ class HistoryReadOperations:
|
||||
next_position=None,
|
||||
resume_position=None,
|
||||
)
|
||||
if raw_id is None:
|
||||
if (
|
||||
raw_id is None
|
||||
or conn.execute(
|
||||
"SELECT 1 FROM execution_context_texts WHERE text_id=? "
|
||||
"AND text_value=? COLLATE BINARY",
|
||||
(raw_id, text),
|
||||
).fetchone()
|
||||
is None
|
||||
):
|
||||
return result
|
||||
scope = _scope_text_ids(conn, prompt_key(text))
|
||||
where = f"r.run_id IN ({_MATCHING})"
|
||||
params = [scope]
|
||||
params = [raw_id]
|
||||
if after is not None:
|
||||
where += f" AND {_KEY}<=(?,?,?,?)"
|
||||
params.extend(after.upper)
|
||||
@@ -372,9 +338,9 @@ class HistoryReadOperations:
|
||||
"LEFT JOIN execution_prompt_snapshots n "
|
||||
"ON n.snapshot_id=c.negative_snapshot_id "
|
||||
"LEFT JOIN execution_context_texts t ON t.text_id=n.raw_text_id "
|
||||
f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id {_IN_SCOPE} "
|
||||
f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id=? "
|
||||
"ORDER BY rc.anchor_id COLLATE BINARY,rc.run_id COLLATE BINARY",
|
||||
(*by_run, scope),
|
||||
(*by_run, raw_id),
|
||||
)
|
||||
summaries = {}
|
||||
evidence_by_context = {}
|
||||
@@ -472,7 +438,6 @@ class HistoryReadOperations:
|
||||
if saved["text"] is None:
|
||||
return {"run": None}
|
||||
epoch = self.get_execution_epoch()
|
||||
key = prompt_key(saved["text"])
|
||||
candidates = conn.execute(
|
||||
"SELECT r.run_id,rc.anchor_id,rc.context_id,"
|
||||
"c.positive_snapshot_id,c.negative_snapshot_id "
|
||||
@@ -480,11 +445,10 @@ class HistoryReadOperations:
|
||||
"ON rc.run_id=r.run_id JOIN execution_contexts c "
|
||||
"ON c.context_id=rc.context_id JOIN execution_prompt_snapshots s "
|
||||
"ON s.snapshot_id=c.positive_snapshot_id "
|
||||
f"WHERE s.raw_text_id {_IN_SCOPE} AND rc.pair_id=? "
|
||||
"AND rc.reference_epoch=? "
|
||||
"WHERE s.raw_text_id=? AND rc.pair_id=? AND rc.reference_epoch=? "
|
||||
f"ORDER BY {_ORDER},rc.anchor_id COLLATE BINARY LIMIT ?",
|
||||
(
|
||||
_scope_text_ids(conn, key),
|
||||
content_digest("text", (saved["text"],)),
|
||||
pair_id,
|
||||
epoch,
|
||||
_PAIR_RUN_CANDIDATES,
|
||||
@@ -493,11 +457,7 @@ class HistoryReadOperations:
|
||||
for row in candidates:
|
||||
positive = _values(conn, row["positive_snapshot_id"])
|
||||
negative = _values(conn, row["negative_snapshot_id"])
|
||||
if (
|
||||
positive is None
|
||||
or positive["raw"] is None
|
||||
or prompt_key(positive["raw"]) != key
|
||||
):
|
||||
if positive is None or positive["raw"] != saved["text"]:
|
||||
continue
|
||||
hints = _hints(conn, positive["raw"], negative["raw"] if negative else None)
|
||||
if hints[2] == pair_id:
|
||||
@@ -517,9 +477,11 @@ class HistoryReadOperations:
|
||||
snapshots = {
|
||||
row["anchor_id"]: row
|
||||
for row in conn.execute(
|
||||
"SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id "
|
||||
"SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id,"
|
||||
"p.raw_text_id AS positive_raw_text_id "
|
||||
"FROM execution_run_contexts rc JOIN execution_contexts c "
|
||||
"ON c.context_id=rc.context_id WHERE rc.run_id=?",
|
||||
"ON c.context_id=rc.context_id LEFT JOIN execution_prompt_snapshots p "
|
||||
"ON p.snapshot_id=c.positive_snapshot_id WHERE rc.run_id=?",
|
||||
(run_id,),
|
||||
)
|
||||
}
|
||||
@@ -529,12 +491,12 @@ class HistoryReadOperations:
|
||||
context.update(
|
||||
{
|
||||
key: row[key]
|
||||
for key in ("positive_snapshot_id", "negative_snapshot_id")
|
||||
},
|
||||
# The history scope this context belongs to (#438)
|
||||
positive_raw_text_id=_scope_text_id(
|
||||
(context["positive"] or {}).get("raw")
|
||||
),
|
||||
for key in (
|
||||
"positive_snapshot_id",
|
||||
"negative_snapshot_id",
|
||||
"positive_raw_text_id",
|
||||
)
|
||||
}
|
||||
)
|
||||
pair = context["pair_id"]
|
||||
context["navigation"] = {
|
||||
|
||||
@@ -221,7 +221,6 @@ class ImageCategoryOperations:
|
||||
sources. Returns ``{updated, unchanged}``; a file that already had
|
||||
exactly this value is ``unchanged`` and keeps its ``updated_at``.
|
||||
"""
|
||||
from .image_removal_schema import clear_marks_of_keys
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
keys = list(dict.fromkeys(tuple(key) for key in file_keys))
|
||||
@@ -235,7 +234,6 @@ class ImageCategoryOperations:
|
||||
continue
|
||||
conn.execute(UPSERT_SQL, (*key, category, now))
|
||||
counts["updated"] += 1
|
||||
clear_marks_of_keys(conn, keys) # #480: a categorized file is listed
|
||||
return counts
|
||||
|
||||
def clear_image_categories(self, file_keys) -> int:
|
||||
|
||||
@@ -402,7 +402,6 @@ class ImageCollectionOperations:
|
||||
is created only when at least one item is inserted. Returns
|
||||
``{collection, created, added, already}``.
|
||||
"""
|
||||
from .image_removal_schema import clear_marks_of_keys
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
if (collection_id is None) == (name is None):
|
||||
@@ -439,7 +438,6 @@ class ImageCollectionOperations:
|
||||
).rowcount
|
||||
for root, path in pairs
|
||||
)
|
||||
clear_marks_of_keys(conn, pairs) # #480: a collected file is listed
|
||||
return {
|
||||
"collection": collection,
|
||||
"created": created,
|
||||
|
||||
@@ -104,14 +104,11 @@ class ImageFavoriteOperations:
|
||||
"""Mark one file as a favorite; repeating it is a no-op. Returns True."""
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
from .image_removal_schema import clear_marks_of_keys
|
||||
|
||||
with self.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO image_favorites VALUES (?, ?, ?)",
|
||||
(root_key, relative_path, utc_now_iso()),
|
||||
)
|
||||
clear_marks_of_keys(conn, [(root_key, relative_path)]) # #480
|
||||
return True
|
||||
|
||||
def clear_image_favorite(self, root_key: str, relative_path: str) -> bool:
|
||||
|
||||
@@ -1,322 +0,0 @@
|
||||
"""Remove a file's records from the library, and its removal mark (#480).
|
||||
|
||||
A *removal* describes one physical file the way the API resolved it from the
|
||||
walked listing (``py/api/image_delete.py``):
|
||||
|
||||
- ``spellings``: every path string records may name it by
|
||||
(``duplicate_spellings.Spellings.of``);
|
||||
- ``locator``: its private canonical locator;
|
||||
- ``favorites``: ``(root key, relative key)`` under every listed root holding
|
||||
it, the keys of its favorite and its collection memberships;
|
||||
- ``category``: its own category's file key, or None.
|
||||
|
||||
``remove_image_records`` deletes, in one ``BEGIN IMMEDIATE`` transaction:
|
||||
|
||||
- its image records (``generated_images``) under every spelling, plus legacy
|
||||
rows matched by ``image_path``; their saved-pair links go by cascade and the
|
||||
workflow documents they held are released;
|
||||
- the image tags and auto-tag marks of its byte records. The annotation file
|
||||
and byte rows go too, unless a Needs review item still cites them
|
||||
(``media_review_items.byte_id`` is ``ON DELETE RESTRICT``): a decided or
|
||||
revised item keeps its decision;
|
||||
- its pending, undecided Needs review items (settled as duplicate removal
|
||||
settles them);
|
||||
- its own category, favorite and collection memberships.
|
||||
|
||||
With ``mark`` it also writes the file's removal mark (library-only removal):
|
||||
the file stays on disk, hidden until its bytes change. Prompts, recorded runs
|
||||
and the media file itself are never touched here.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from .annotation_schema import BYTES_TABLE, FILES_TABLE
|
||||
from .annotations import _BUMP_INDEX
|
||||
from .autotag_schema import AUTOTAG_TABLE
|
||||
from .duplicate_copies import (
|
||||
DELETE_FAVORITE_SQL,
|
||||
DELETE_LINK_SQL,
|
||||
DELETE_MEMBERSHIPS_SQL,
|
||||
_file_link_rows,
|
||||
)
|
||||
from .image_categories import DELETE_SQL as DELETE_CATEGORY_SQL
|
||||
from .image_removal_schema import (
|
||||
META_TABLE,
|
||||
TABLE,
|
||||
clear_marks,
|
||||
has_marks_table,
|
||||
mark_of,
|
||||
write_mark as _write_mark,
|
||||
)
|
||||
from .library import display_title
|
||||
from .models import utc_now_iso
|
||||
from .review import _BUMP_REVIEW_INDEX, _PENDING_OF_LOCATOR, _SETTLE_ITEM
|
||||
from .review_changes import notify_review_change
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
BYTES_OF_LOCATOR_SQL = (
|
||||
f"SELECT b.id FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b " # nosec B608
|
||||
"ON b.file_id=f.id WHERE f.locator=?"
|
||||
)
|
||||
TAG_COUNT_SQL = "SELECT COUNT(*) FROM media_image_tags WHERE byte_id=?"
|
||||
AUTOTAG_COUNT_SQL = (
|
||||
f"SELECT COUNT(*) FROM {AUTOTAG_TABLE} WHERE byte_id=?" # nosec B608
|
||||
)
|
||||
CLEAR_TAGS_SQL = "DELETE FROM media_image_tags WHERE byte_id=?"
|
||||
CLEAR_AUTOTAG_SQL = f"DELETE FROM {AUTOTAG_TABLE} WHERE byte_id=?" # nosec B608
|
||||
BUMP_TAG_REVISION_SQL = (
|
||||
f"UPDATE {BYTES_TABLE} SET tag_revision=tag_revision+1 WHERE id=?" # nosec B608
|
||||
)
|
||||
REVIEW_REFS_SQL = (
|
||||
f"SELECT COUNT(*) FROM {FILES_TABLE} f " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
"JOIN media_review_items i ON i.byte_id=b.id WHERE f.locator=?"
|
||||
)
|
||||
DELETE_FILE_SQL = f"DELETE FROM {FILES_TABLE} WHERE locator=?" # nosec B608
|
||||
FILE_EXISTS_SQL = f"SELECT 1 FROM {FILES_TABLE} WHERE locator=?" # nosec B608
|
||||
FAVORITE_EXISTS_SQL = (
|
||||
"SELECT COUNT(*) FROM image_favorites WHERE root_key=? AND relative_path=?"
|
||||
)
|
||||
MEMBERSHIP_COUNT_SQL = (
|
||||
"SELECT COUNT(*) FROM image_collection_items WHERE root_key=? AND relative_path=?"
|
||||
)
|
||||
CATEGORY_EXISTS_SQL = (
|
||||
"SELECT COUNT(*) FROM image_categories WHERE root_key=? AND relative_path=?"
|
||||
)
|
||||
PAIR_LINK_COUNT_SQL = "SELECT COUNT(*) FROM image_prompt_pairs WHERE image_id=?"
|
||||
PROMPT_LINKS_SQL = "SELECT id FROM generated_images WHERE prompt_id=?"
|
||||
PROMPT_TITLE_SQL = (
|
||||
"SELECT p.text, s.title FROM prompts p "
|
||||
"LEFT JOIN prompt_library_state s ON s.prompt_id=p.id WHERE p.id=?"
|
||||
)
|
||||
MARKS_SQL = f"SELECT locator, size, mtime_ns FROM {TABLE}" # nosec B608
|
||||
REVISION_SQL = f"SELECT revision FROM {META_TABLE} WHERE singleton_id=1" # nosec B608
|
||||
MAX_ORPHAN_TITLES = 3
|
||||
RECORD_KINDS = (
|
||||
"prompt_links",
|
||||
"pair_links",
|
||||
"image_tags",
|
||||
"auto_tagged",
|
||||
"categories",
|
||||
"collections",
|
||||
"favorites",
|
||||
"review_items",
|
||||
)
|
||||
|
||||
|
||||
def _link_ids(conn, removal):
|
||||
"""``{image id: prompt id}`` of the records naming the file, any spelling."""
|
||||
found = {}
|
||||
for spelled in dict.fromkeys(removal["spellings"]):
|
||||
for image_id, prompt_id in _file_link_rows(conn, spelled):
|
||||
found[image_id] = prompt_id
|
||||
return found
|
||||
|
||||
|
||||
def _byte_ids(conn, locator):
|
||||
return [row[0] for row in conn.execute(BYTES_OF_LOCATOR_SQL, (locator,))]
|
||||
|
||||
|
||||
def file_locator(path) -> str:
|
||||
"""The locator of an absolute path (``normcase(realpath)``)."""
|
||||
return os.path.normcase(os.path.realpath(str(path)))
|
||||
|
||||
|
||||
def removed_from_library(conn, path) -> bool:
|
||||
"""Whether *path* is hidden by a removal mark whose bytes still match.
|
||||
|
||||
A mark whose file changed (other size or ``mtime_ns``) or is gone is
|
||||
stale: it is deleted on *conn* (the caller's write transaction), and the
|
||||
file is not hidden.
|
||||
"""
|
||||
locator = file_locator(path)
|
||||
mark = mark_of(conn, locator)
|
||||
if mark is None:
|
||||
return False
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except OSError:
|
||||
clear_marks(conn, [locator])
|
||||
return False
|
||||
if (info.st_size, info.st_mtime_ns) == tuple(mark):
|
||||
return True
|
||||
clear_marks(conn, [locator])
|
||||
return False
|
||||
|
||||
|
||||
def mark_hides(marks: dict, locator: str, path) -> bool:
|
||||
"""Whether a removal mark in *marks* hides the file at *path* (blocking).
|
||||
|
||||
*marks* is ``{locator: (size, mtime_ns)}``. The mark of *locator* hides
|
||||
the file while ``os.lstat(path)`` still has the size and ``mtime_ns`` it
|
||||
was removed with; a file that cannot be looked at is not hidden.
|
||||
"""
|
||||
mark = marks.get(locator)
|
||||
if mark is None:
|
||||
return False
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except OSError:
|
||||
return False
|
||||
return (info.st_size, info.st_mtime_ns) == tuple(mark)
|
||||
|
||||
|
||||
def write_mark(conn, locator, size, mtime_ns):
|
||||
_write_mark(conn, locator, size, mtime_ns, utc_now_iso())
|
||||
|
||||
|
||||
class ImageRemovalOperations:
|
||||
"""PromptDatabase mixin: remove a file's records; removal marks."""
|
||||
|
||||
def count_removal_records(self, removals) -> dict:
|
||||
"""Counts of what removing *removals* deletes, in one read snapshot.
|
||||
|
||||
``{records: {kind: n}, orphaned_prompts: {count, titles}}``: prompts
|
||||
whose every image record is among these files lose their last image
|
||||
(they stay in the Library).
|
||||
"""
|
||||
from .maintenance import _read_snapshot
|
||||
|
||||
def read(conn):
|
||||
totals = dict.fromkeys(RECORD_KINDS, 0)
|
||||
links = {}
|
||||
for removal in removals:
|
||||
found = _link_ids(conn, removal)
|
||||
links.update(found)
|
||||
for image_id in found:
|
||||
totals["pair_links"] += conn.execute(
|
||||
PAIR_LINK_COUNT_SQL, (image_id,)
|
||||
).fetchone()[0]
|
||||
tagged = autotagged = False
|
||||
for byte_id in _byte_ids(conn, removal["locator"]):
|
||||
tagged |= bool(
|
||||
conn.execute(TAG_COUNT_SQL, (byte_id,)).fetchone()[0]
|
||||
)
|
||||
autotagged |= bool(
|
||||
conn.execute(AUTOTAG_COUNT_SQL, (byte_id,)).fetchone()[0]
|
||||
)
|
||||
totals["image_tags"] += int(tagged)
|
||||
totals["auto_tagged"] += int(autotagged)
|
||||
totals["review_items"] += len(
|
||||
conn.execute(_PENDING_OF_LOCATOR, (removal["locator"],)).fetchall()
|
||||
)
|
||||
totals["favorites"] += int(
|
||||
any(
|
||||
conn.execute(FAVORITE_EXISTS_SQL, key).fetchone()[0]
|
||||
for key in removal["favorites"]
|
||||
)
|
||||
)
|
||||
totals["collections"] += sum(
|
||||
conn.execute(MEMBERSHIP_COUNT_SQL, key).fetchone()[0]
|
||||
for key in removal["favorites"]
|
||||
)
|
||||
if removal.get("category"):
|
||||
totals["categories"] += conn.execute(
|
||||
CATEGORY_EXISTS_SQL, removal["category"]
|
||||
).fetchone()[0]
|
||||
totals["prompt_links"] = len(links)
|
||||
return {
|
||||
"records": totals,
|
||||
"orphaned_prompts": self._orphaned_prompts(conn, links),
|
||||
}
|
||||
|
||||
return _read_snapshot(self.model.db_path, read)
|
||||
|
||||
@staticmethod
|
||||
def _orphaned_prompts(conn, links):
|
||||
removed = set(links)
|
||||
orphans = []
|
||||
for prompt_id in dict.fromkeys(links.values()):
|
||||
remaining = [
|
||||
row[0]
|
||||
for row in conn.execute(PROMPT_LINKS_SQL, (prompt_id,))
|
||||
if row[0] not in removed
|
||||
]
|
||||
if not remaining:
|
||||
orphans.append(prompt_id)
|
||||
titles = []
|
||||
for prompt_id in orphans[:MAX_ORPHAN_TITLES]:
|
||||
row = conn.execute(PROMPT_TITLE_SQL, (prompt_id,)).fetchone()
|
||||
if row is not None:
|
||||
titles.append(display_title((row[1],), row[0]) or "")
|
||||
return {"count": len(orphans), "titles": titles}
|
||||
|
||||
def remove_image_records(self, removal, *, mark=None) -> dict:
|
||||
"""Delete every record of one file (one transaction); see the module.
|
||||
|
||||
``mark``: ``(size, mtime_ns)`` to write the file's removal mark, or
|
||||
None (the file was moved to the trash). Returns the counts deleted.
|
||||
"""
|
||||
counts = dict.fromkeys(RECORD_KINDS, 0)
|
||||
locator = removal["locator"]
|
||||
with self._pair_transaction() as conn:
|
||||
links = _link_ids(conn, removal)
|
||||
if links:
|
||||
documents = documents_of_rows(conn, "id", list(links))
|
||||
for image_id in links:
|
||||
counts["pair_links"] += conn.execute(
|
||||
PAIR_LINK_COUNT_SQL, (image_id,)
|
||||
).fetchone()[0]
|
||||
conn.execute(DELETE_LINK_SQL, (image_id,))
|
||||
release_documents(conn, documents)
|
||||
counts["prompt_links"] = len(links)
|
||||
counts.update(self._remove_annotations(conn, locator))
|
||||
for key in removal["favorites"]:
|
||||
counts["favorites"] += conn.execute(DELETE_FAVORITE_SQL, key).rowcount
|
||||
counts["collections"] += conn.execute(
|
||||
DELETE_MEMBERSHIPS_SQL, key
|
||||
).rowcount
|
||||
if removal.get("category"):
|
||||
counts["categories"] += conn.execute(
|
||||
DELETE_CATEGORY_SQL, tuple(removal["category"])
|
||||
).rowcount
|
||||
if mark is not None:
|
||||
write_mark(conn, locator, *mark)
|
||||
if counts["review_items"]:
|
||||
notify_review_change(self.model.db_path)
|
||||
return counts
|
||||
|
||||
def _remove_annotations(self, conn, locator) -> dict:
|
||||
"""Tags, auto-tag marks, pending review items and, when free, the rows."""
|
||||
counts = {"image_tags": 0, "auto_tagged": 0, "review_items": 0}
|
||||
bytes_ = _byte_ids(conn, locator)
|
||||
for byte_id in bytes_:
|
||||
tags = conn.execute(CLEAR_TAGS_SQL, (byte_id,)).rowcount
|
||||
autotags = conn.execute(CLEAR_AUTOTAG_SQL, (byte_id,)).rowcount
|
||||
if tags:
|
||||
conn.execute(BUMP_TAG_REVISION_SQL, (byte_id,))
|
||||
counts["image_tags"] |= bool(tags)
|
||||
counts["auto_tagged"] |= bool(autotags)
|
||||
for (item_id,) in conn.execute(_PENDING_OF_LOCATOR, (locator,)).fetchall():
|
||||
counts["review_items"] += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
||||
if counts["review_items"]:
|
||||
conn.execute(_BUMP_REVIEW_INDEX)
|
||||
if conn.execute(FILE_EXISTS_SQL, (locator,)).fetchone():
|
||||
if not conn.execute(REVIEW_REFS_SQL, (locator,)).fetchone()[0]:
|
||||
conn.execute(DELETE_FILE_SQL, (locator,))
|
||||
conn.execute(_BUMP_INDEX)
|
||||
counts["image_tags"] = int(counts["image_tags"])
|
||||
counts["auto_tagged"] = int(counts["auto_tagged"])
|
||||
return counts
|
||||
|
||||
def removal_marks(self) -> dict:
|
||||
"""``{locator: (size, mtime_ns)}`` of every removal mark (one read)."""
|
||||
conn = self.model.get_connection()
|
||||
if not has_marks_table(conn):
|
||||
return {}
|
||||
return {row[0]: (row[1], row[2]) for row in conn.execute(MARKS_SQL)}
|
||||
|
||||
def removal_revision(self) -> int:
|
||||
conn = self.model.get_connection()
|
||||
if not has_marks_table(conn):
|
||||
return 0
|
||||
row = conn.execute(REVISION_SQL).fetchone()
|
||||
return row[0] if row else 0
|
||||
|
||||
def clear_removal_marks(self, locators) -> int:
|
||||
"""Show these files again (their marks go); the count removed."""
|
||||
locators = list(locators)
|
||||
if not locators:
|
||||
return 0
|
||||
with self._pair_transaction() as conn:
|
||||
return clear_marks(conn, locators)
|
||||
@@ -1,208 +0,0 @@
|
||||
"""Removal marks (#480): files removed from the library that stay on disk.
|
||||
|
||||
``image_removals`` holds one row per hidden file: its private canonical
|
||||
locator (the ``media_annotation_files.locator`` form) and the ``size`` and
|
||||
``mtime_ns`` it had when it was removed. A file at that locator with other
|
||||
bytes is a new file and shows again. ``image_removals_meta`` is one row, the
|
||||
marks revision, bumped in the same transaction as every mark change so the
|
||||
workspace listing caches can tell marks changed.
|
||||
|
||||
The group installs like ``image_favorites``: optional in old backups,
|
||||
required once a database is current. Older builds ignore it.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
from .annotation_schema import MAX_LOCATOR_LENGTH
|
||||
|
||||
TABLE = "image_removals"
|
||||
META_TABLE = "image_removals_meta"
|
||||
TABLES = (TABLE, META_TABLE)
|
||||
_TABLE_DDL = {
|
||||
TABLE: (
|
||||
f"CREATE TABLE {TABLE} (locator TEXT PRIMARY KEY, "
|
||||
"size INTEGER NOT NULL CHECK(size>=0), mtime_ns INTEGER NOT NULL, "
|
||||
"removed_at TEXT NOT NULL) WITHOUT ROWID"
|
||||
),
|
||||
META_TABLE: (
|
||||
f"CREATE TABLE {META_TABLE} (singleton_id INTEGER PRIMARY KEY "
|
||||
"CHECK(singleton_id=1), revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(revision>=0))"
|
||||
),
|
||||
}
|
||||
_SEED_META = (
|
||||
f"INSERT OR IGNORE INTO {META_TABLE} (singleton_id, revision) VALUES (1, 0)"
|
||||
)
|
||||
_COLUMNS = {
|
||||
TABLE: (
|
||||
("locator", "TEXT", 1, 1),
|
||||
("size", "INTEGER", 1, 0),
|
||||
("mtime_ns", "INTEGER", 1, 0),
|
||||
("removed_at", "TEXT", 1, 0),
|
||||
),
|
||||
META_TABLE: (
|
||||
("singleton_id", "INTEGER", 0, 1),
|
||||
("revision", "INTEGER", 1, 0),
|
||||
),
|
||||
}
|
||||
_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection, allow_missing: bool) -> str | None:
|
||||
rows = conn.execute(
|
||||
"SELECT type,name,tbl_name,sql FROM sqlite_master WHERE name IN (?,?) "
|
||||
"OR tbl_name IN (?,?)",
|
||||
TABLES + TABLES,
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return None if allow_missing else "Missing image removal marks schema"
|
||||
tables = {name: sql for kind, name, _tbl, sql in rows if kind == "table"}
|
||||
others = {(kind, name) for kind, name, _tbl, _sql in rows if kind != "table"}
|
||||
if set(tables) != set(TABLES) or others - {("index", _AUTOINDEX)}:
|
||||
return "Image removal marks name is reserved by an unsupported schema object"
|
||||
for table in TABLES:
|
||||
if _compact(tables[table]) != _compact(_TABLE_DDL[table]):
|
||||
return "Image removal marks have an invalid table definition"
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({table})")
|
||||
)
|
||||
if columns != _COLUMNS[table]:
|
||||
return "Image removal marks have invalid columns"
|
||||
return None
|
||||
|
||||
|
||||
def _values_error(conn: sqlite3.Connection) -> str | None:
|
||||
limit = MAX_LOCATOR_LENGTH * 4
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {TABLE} WHERE typeof(locator) != 'text' " # nosec B608
|
||||
f"OR length(CAST(locator AS BLOB)) > {limit} OR locator = '' "
|
||||
"OR typeof(size) != 'integer' OR typeof(mtime_ns) != 'integer' "
|
||||
"OR typeof(removed_at) != 'text' OR length(CAST(removed_at AS BLOB)) > 64 "
|
||||
"LIMIT 1"
|
||||
).fetchone():
|
||||
return "Image removal marks contain invalid values"
|
||||
meta = conn.execute(
|
||||
f"SELECT singleton_id, typeof(revision) FROM {META_TABLE}" # nosec B608
|
||||
).fetchall()
|
||||
if meta != [(1, "integer")]:
|
||||
return "Image removal marks have an invalid revision"
|
||||
return None
|
||||
|
||||
|
||||
def image_removals_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> str | None:
|
||||
"""Reject unsupported schemas and invalid persisted values (reads only)."""
|
||||
error = _structure_error(conn, allow_missing)
|
||||
if error:
|
||||
return error
|
||||
exists = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone()
|
||||
return _values_error(conn) if exists else None
|
||||
|
||||
|
||||
def migrate_image_removals(conn: sqlite3.Connection) -> None:
|
||||
"""Install both tables atomically; validate before and after."""
|
||||
conn.execute("SAVEPOINT image_removals_schema")
|
||||
try:
|
||||
error = image_removals_schema_error(conn)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
for sql in _TABLE_DDL.values():
|
||||
conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
conn.execute(_SEED_META)
|
||||
error = image_removals_schema_error(conn, allow_missing=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE image_removals_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO image_removals_schema")
|
||||
conn.execute("RELEASE image_removals_schema")
|
||||
raise
|
||||
|
||||
|
||||
_UNMARK_SQL = f"DELETE FROM {TABLE} WHERE locator=?" # nosec B608
|
||||
_MARK_SQL = (
|
||||
f"INSERT OR REPLACE INTO {TABLE} " # nosec B608
|
||||
"(locator, size, mtime_ns, removed_at) VALUES (?, ?, ?, ?)"
|
||||
)
|
||||
_MARK_OF_SQL = f"SELECT size, mtime_ns FROM {TABLE} WHERE locator=?" # nosec B608
|
||||
BUMP_MARKS_SQL = (
|
||||
f"UPDATE {META_TABLE} SET revision=revision+1 WHERE singleton_id=1" # nosec B608
|
||||
)
|
||||
|
||||
|
||||
# Bumped whenever this process writes or clears a mark (even in a transaction
|
||||
# that later rolls back): readers that cache the marks reload when it moves.
|
||||
_generation = 0
|
||||
|
||||
|
||||
def generation() -> int:
|
||||
"""The marks generation of this process (see ``_generation``)."""
|
||||
return _generation
|
||||
|
||||
|
||||
def _bump() -> None:
|
||||
global _generation
|
||||
_generation += 1
|
||||
|
||||
|
||||
def has_marks_table(conn: sqlite3.Connection) -> bool:
|
||||
return (
|
||||
conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def locator_of_key(root_key: str, relative_key: str) -> str:
|
||||
"""The locator of a favorite-style file key (root key + relative key)."""
|
||||
return os.path.normcase(os.path.join(root_key, *relative_key.split("/")))
|
||||
|
||||
|
||||
def mark_of(conn: sqlite3.Connection, locator: str):
|
||||
"""``(size, mtime_ns)`` of the file's removal mark, or None."""
|
||||
if not has_marks_table(conn):
|
||||
return None
|
||||
row = conn.execute(_MARK_OF_SQL, (locator,)).fetchone()
|
||||
return (row[0], row[1]) if row else None
|
||||
|
||||
|
||||
def write_mark(
|
||||
conn: sqlite3.Connection, locator: str, size: int, mtime_ns: int, now: str
|
||||
):
|
||||
conn.execute(_MARK_SQL, (locator, int(size), int(mtime_ns), now))
|
||||
conn.execute(BUMP_MARKS_SQL)
|
||||
_bump()
|
||||
|
||||
|
||||
def clear_marks(conn: sqlite3.Connection, locators) -> int:
|
||||
"""Delete the marks of *locators* on *conn* (no commit); the count removed.
|
||||
|
||||
Every record write for a file calls this (#480): a file with records is
|
||||
in the library, so it is never hidden.
|
||||
"""
|
||||
locators = list(dict.fromkeys(locators))
|
||||
if not locators or not has_marks_table(conn):
|
||||
return 0
|
||||
removed = 0
|
||||
for locator in locators:
|
||||
removed += conn.execute(_UNMARK_SQL, (locator,)).rowcount
|
||||
if removed:
|
||||
conn.execute(BUMP_MARKS_SQL)
|
||||
_bump()
|
||||
return removed
|
||||
|
||||
|
||||
def clear_marks_of_keys(conn: sqlite3.Connection, file_keys) -> int:
|
||||
"""``clear_marks`` for favorite-style ``(root key, relative key)`` pairs."""
|
||||
return clear_marks(conn, [locator_of_key(*key) for key in file_keys])
|
||||
+12
-41
@@ -7,7 +7,6 @@ from contextlib import contextmanager
|
||||
|
||||
from .library_folders import folder_condition, list_library_folders
|
||||
from .library_tokens import library_edit_token
|
||||
from .prompt_archive import ARCHIVED, SHELVED
|
||||
|
||||
_ELIGIBLE = """(EXISTS(SELECT 1 FROM prompt_pairs ep WHERE ep.positive_prompt_id=p.id)
|
||||
OR EXISTS(SELECT 1 FROM generated_images eg WHERE eg.prompt_id=p.id)
|
||||
@@ -107,12 +106,6 @@ ARCHIVE_SCOPES = {
|
||||
}
|
||||
|
||||
|
||||
# Shelves (#443): the parents on the shelf, or the archived ones.
|
||||
PARENT_SCOPES = {"active": SHELVED, "archived": ARCHIVED}
|
||||
# A positive pair owner that is on the shelf, as a condition on alias pp.
|
||||
SHELVED_PAIR = SHELVED.replace("p.id", "pp.positive_prompt_id")
|
||||
|
||||
|
||||
def _escape(value: str) -> str:
|
||||
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
@@ -277,8 +270,6 @@ class LibraryOperations:
|
||||
rows = conn.execute(
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,p.created_at,"
|
||||
"p.run_count,p.last_used_at,"
|
||||
+ ARCHIVED
|
||||
+ " AS archived,"
|
||||
+ (f"{sort_expression} AS _sort," if sort_expression else "")
|
||||
+ "(SELECT COUNT(*) FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
|
||||
"AND v.archived=0) AS variant_count,"
|
||||
@@ -302,7 +293,6 @@ class LibraryOperations:
|
||||
# Usage is shown, never compared: the token hashes owner fields only.
|
||||
row["run_count"] = row["run_count"] if type(row["run_count"]) is int else 0
|
||||
row["last_used_at"] = utc_iso(row["last_used_at"])
|
||||
row["archived"] = bool(row["archived"])
|
||||
row["edit_token"] = library_edit_token(epoch, row)
|
||||
return result
|
||||
|
||||
@@ -320,13 +310,10 @@ class LibraryOperations:
|
||||
sort: str = "newest",
|
||||
limit: int = 40,
|
||||
after: dict | None = None,
|
||||
shelf: str = "active",
|
||||
) -> dict:
|
||||
_limit(limit)
|
||||
if sort not in _ORDERS:
|
||||
raise ValueError("Unknown library sort order")
|
||||
if not isinstance(shelf, str) or shelf not in PARENT_SCOPES:
|
||||
raise ValueError("shelf must be active or archived")
|
||||
where, params = _filter(
|
||||
text, category, [] if tags is None else tags, tag_mode, folder_prefixes
|
||||
)
|
||||
@@ -341,18 +328,15 @@ class LibraryOperations:
|
||||
ceiling = after["ceiling"]
|
||||
where += " AND p.id<=?"
|
||||
params.append(ceiling)
|
||||
# One SELECT counts the filtered parents of this scope, their active
|
||||
# variants, and the filtered archived parents (Show archived, #443).
|
||||
scope = PARENT_SCOPES[shelf]
|
||||
total, variant_total, archived_total = conn.execute(
|
||||
f"SELECT COALESCE(SUM({scope}),0),"
|
||||
f"COALESCE(SUM(CASE WHEN {scope} THEN (SELECT COUNT(*) "
|
||||
"FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
|
||||
"AND v.archived=0) ELSE 0 END),0),"
|
||||
f"COALESCE(SUM({ARCHIVED}),0) FROM " + _FROM + " WHERE " + where,
|
||||
# One SELECT counts the filtered parents and their active variants.
|
||||
total, variant_total = conn.execute(
|
||||
"SELECT COUNT(*),COALESCE(SUM((SELECT COUNT(*) FROM prompt_pairs v "
|
||||
"WHERE v.positive_prompt_id=p.id AND v.archived=0)),0) FROM "
|
||||
+ _FROM
|
||||
+ " WHERE "
|
||||
+ where,
|
||||
params,
|
||||
).fetchone()
|
||||
where += f" AND {scope}"
|
||||
expression, direction = _ORDERS[sort]
|
||||
if after is not None:
|
||||
operator = ">" if direction == "ASC" else "<"
|
||||
@@ -385,7 +369,6 @@ class LibraryOperations:
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"variant_total": variant_total,
|
||||
"archived_total": archived_total,
|
||||
# Echoed so a client can tell this read applied its tag match.
|
||||
"tag_mode": tag_mode,
|
||||
"has_more": has_more,
|
||||
@@ -403,7 +386,6 @@ class LibraryOperations:
|
||||
sort: str = "newest",
|
||||
limit: int = 40,
|
||||
after: dict | None = None,
|
||||
shelf: str = "active",
|
||||
) -> dict:
|
||||
with self.library_snapshot(include_ceiling=True) as (conn, epoch, ceiling):
|
||||
return self._list_library(
|
||||
@@ -418,7 +400,6 @@ class LibraryOperations:
|
||||
sort=sort,
|
||||
limit=limit,
|
||||
after=after,
|
||||
shelf=shelf,
|
||||
)
|
||||
|
||||
def _is_library_positive(self, conn: sqlite3.Connection, positive_id: int) -> bool:
|
||||
@@ -663,9 +644,7 @@ class LibraryOperations:
|
||||
owners = {}
|
||||
order = []
|
||||
for row in conn.execute(
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,"
|
||||
+ ARCHIVED
|
||||
+ " AS archived FROM "
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes FROM "
|
||||
+ _FROM
|
||||
+ " WHERE "
|
||||
+ _ELIGIBLE
|
||||
@@ -678,7 +657,6 @@ class LibraryOperations:
|
||||
"tags": [],
|
||||
"rating": row["rating"],
|
||||
"notes": row["notes"],
|
||||
"archived": bool(row["archived"]),
|
||||
"variants": [],
|
||||
}
|
||||
order.append(row["id"])
|
||||
@@ -723,9 +701,8 @@ class LibraryOperations:
|
||||
"""System › Overview counts from one snapshot, writing nothing.
|
||||
|
||||
``positive_prompts`` and ``negative_variants`` are the unfiltered
|
||||
Library's ``total`` and ``variant_total``: eligible parents on the
|
||||
shelf (not archived, #443) and their active pairs (every pair's
|
||||
positive is eligible). ``images`` counts indexed
|
||||
Library's ``total`` and ``variant_total``: eligible parents and active
|
||||
pairs (every pair's positive is eligible). ``images`` counts indexed
|
||||
``generated_images`` rows, not files on disk. An image has no rating
|
||||
of its own (it shows its linked prompt's), so ``rated_images`` counts
|
||||
the indexed images whose linked prompt is rated. ``review_pending`` is
|
||||
@@ -733,14 +710,8 @@ class LibraryOperations:
|
||||
"""
|
||||
with self.library_snapshot() as (conn, _epoch, _ceiling):
|
||||
row = conn.execute(
|
||||
"SELECT (SELECT COUNT(*) FROM prompts p WHERE "
|
||||
+ _ELIGIBLE
|
||||
+ " AND "
|
||||
+ SHELVED
|
||||
+ "),"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs pp WHERE pp.archived=0 AND "
|
||||
+ SHELVED_PAIR
|
||||
+ "),"
|
||||
"SELECT (SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE + "),"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs WHERE archived=0),"
|
||||
"(SELECT COUNT(*) FROM generated_images),"
|
||||
"(SELECT COUNT(*) FROM generated_images gi JOIN prompts r "
|
||||
"ON r.id=gi.prompt_id WHERE r.rating IS NOT NULL),"
|
||||
|
||||
@@ -1,289 +0,0 @@
|
||||
"""Delete selected Library prompts; archive the paired ones (#443, D152).
|
||||
|
||||
A selected Library parent is classified, in this order, as ``not_found``
|
||||
(missing, or not a Library parent), ``protected`` (tagged ``__protected__``;
|
||||
never changed), ``already_archived``, ``archive`` (the positive or negative of
|
||||
any saved pair, archived pair or not: archiving hides it from the shelf and
|
||||
removes nothing) or ``delete`` (in no pair: deleted exactly as the legacy
|
||||
delete does, through ``delete_prompt_rows``). Every command checks the
|
||||
storage epoch first and runs in one ``BEGIN IMMEDIATE`` transaction.
|
||||
|
||||
The write executes what the user reviewed: it is given the reviewed archive
|
||||
and delete lists, and deletes only reviewed deletes the route backed up
|
||||
(``candidates``) whose fingerprint is unchanged since that backup.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
from typing import Iterable
|
||||
|
||||
from .library import _ELIGIBLE
|
||||
from .library_commands import _validate_bulk_ids, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .prompt_archive import ARCHIVED as _ARCHIVED
|
||||
from .prompt_archive import TABLE as ARCHIVE
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
# Ids bound per statement: the pair check binds each id twice, which stays
|
||||
# under SQLite's historical 999-variable limit (maintenance.py ID_BATCH).
|
||||
ID_BATCH = 400
|
||||
_PROTECTED = (
|
||||
"EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.prompt_id=p.id AND t.name='__protected__')"
|
||||
)
|
||||
_PAIRED = (
|
||||
"EXISTS(SELECT 1 FROM prompt_pairs pp "
|
||||
"WHERE pp.positive_prompt_id=p.id OR pp.negative_prompt_id=p.id)"
|
||||
)
|
||||
# Classes in the order a prompt is tested; the first that holds wins.
|
||||
CLASSES = ("not_found", "protected", "already_archived", "archive", "delete")
|
||||
|
||||
|
||||
def _batches(ids: list) -> Iterable[list]:
|
||||
for start in range(0, len(ids), ID_BATCH):
|
||||
yield ids[start : start + ID_BATCH]
|
||||
|
||||
|
||||
def delete_prompt_rows(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
|
||||
"""Delete prompts and their image link rows; return the prompts deleted.
|
||||
|
||||
The legacy delete's statements (``delete_prompt``, ``bulk_delete_prompts``)
|
||||
in the caller's transaction: image link rows first (``image_prompt_pairs``
|
||||
cascades, review decisions keep a NULL image), then the prompts (tags and
|
||||
Library state cascade), then the workflow documents no row references.
|
||||
Image files are never touched. The caller checks pairs first.
|
||||
"""
|
||||
documents = documents_of_rows(conn, "prompt_id", prompt_ids)
|
||||
deleted = 0
|
||||
for chunk in _batches(list(prompt_ids)):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
conn.execute(
|
||||
f"DELETE FROM generated_images WHERE prompt_id IN ({marks})", # nosec
|
||||
chunk,
|
||||
)
|
||||
deleted += conn.execute(
|
||||
f"DELETE FROM prompts WHERE id IN ({marks})", chunk # nosec B608
|
||||
).rowcount
|
||||
release_documents(conn, documents)
|
||||
return deleted
|
||||
|
||||
|
||||
class LibraryArchiveOperations:
|
||||
"""PromptDatabase mixin: classify, archive or delete, and restore parents."""
|
||||
|
||||
def _classify_library_delete(
|
||||
self, conn: sqlite3.Connection, prompt_ids: list[int]
|
||||
) -> dict:
|
||||
found = {}
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for row in conn.execute(
|
||||
f"SELECT p.id,{_PROTECTED},{_ARCHIVED},{_PAIRED} " # nosec B608
|
||||
f"FROM prompts p WHERE p.id IN ({marks}) AND {_ELIGIBLE}",
|
||||
chunk,
|
||||
):
|
||||
protected, archived, paired = row[1], row[2], row[3]
|
||||
if protected:
|
||||
found[row[0]] = "protected"
|
||||
elif archived:
|
||||
found[row[0]] = "already_archived"
|
||||
else:
|
||||
found[row[0]] = "archive" if paired else "delete"
|
||||
classes = {name: [] for name in CLASSES}
|
||||
for pid in prompt_ids:
|
||||
classes[found.get(pid, "not_found")].append(pid)
|
||||
return classes
|
||||
|
||||
@staticmethod
|
||||
def _image_links(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
|
||||
total = 0
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
total += conn.execute(
|
||||
"SELECT COUNT(*) FROM generated_images " # nosec B608
|
||||
f"WHERE prompt_id IN ({marks})",
|
||||
chunk,
|
||||
).fetchone()[0]
|
||||
return total
|
||||
|
||||
@staticmethod
|
||||
def _fingerprints(conn: sqlite3.Connection, prompt_ids: list[int]) -> dict:
|
||||
"""What a safety backup holds of each prompt a delete would remove.
|
||||
|
||||
The prompt row (usage included), its tag ids, its Library title and
|
||||
its image link rows (count and newest id), as a JSON-safe list.
|
||||
"""
|
||||
prints = {}
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for row in conn.execute(
|
||||
"SELECT p.id,p.text,p.hash,p.category,p.rating,p.notes," # nosec
|
||||
"p.updated_at,p.run_count,p.last_used_at,"
|
||||
"(SELECT group_concat(tag_id) FROM (SELECT tag_id FROM prompt_tags "
|
||||
"WHERE prompt_id=p.id ORDER BY tag_id)),"
|
||||
"(SELECT title FROM prompt_library_state WHERE prompt_id=p.id),"
|
||||
"(SELECT COUNT(*) FROM generated_images WHERE prompt_id=p.id),"
|
||||
"(SELECT MAX(id) FROM generated_images WHERE prompt_id=p.id) "
|
||||
f"FROM prompts p WHERE p.id IN ({marks})",
|
||||
chunk,
|
||||
):
|
||||
prints[row[0]] = [
|
||||
value if isinstance(value, (int, str, type(None))) else repr(value)
|
||||
for value in tuple(row)[1:]
|
||||
]
|
||||
return prints
|
||||
|
||||
@staticmethod
|
||||
def _checked_ids(prompt_ids: list[int], expected_epoch: str) -> list[int]:
|
||||
_validate_epoch(expected_epoch)
|
||||
_validate_bulk_ids(prompt_ids)
|
||||
return list(dict.fromkeys(prompt_ids))
|
||||
|
||||
def classify_library_delete(
|
||||
self, prompt_ids: list[int], *, expected_epoch: str
|
||||
) -> dict:
|
||||
"""What a delete of ``prompt_ids`` would do now (reads only).
|
||||
|
||||
Returns the id lists per class, in request order, ``image_links``
|
||||
(the image link rows the ``delete`` class would remove) and
|
||||
``fingerprints`` of the ``delete`` class (``_fingerprints``).
|
||||
"""
|
||||
prompt_ids = self._checked_ids(prompt_ids, expected_epoch)
|
||||
with self.library_snapshot() as (conn, _epoch, _ceiling):
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
classes = self._classify_library_delete(conn, prompt_ids)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
**classes,
|
||||
"image_links": self._image_links(conn, classes["delete"]),
|
||||
"fingerprints": self._fingerprints(conn, classes["delete"]),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _repair_hash(conn: sqlite3.Connection, prompt_id: int) -> None:
|
||||
"""Give an archived prompt its canonical hash, so ingestion finds it.
|
||||
|
||||
Scans, the monitor and node saves resolve prompts by hash; a stale or
|
||||
missing one would make them insert a visible copy (Q3). When another
|
||||
row already holds the canonical hash, lookups find that row and no new
|
||||
copy is made, so nothing is written.
|
||||
"""
|
||||
text, stored = conn.execute(
|
||||
"SELECT text,hash FROM prompts WHERE id=?", (prompt_id,)
|
||||
).fetchone()
|
||||
canonical = generate_prompt_hash(text) if isinstance(text, str) else None
|
||||
if canonical is None or stored == canonical:
|
||||
return
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM prompts WHERE hash=? AND id!=?", (canonical, prompt_id)
|
||||
).fetchone():
|
||||
return
|
||||
conn.execute("UPDATE prompts SET hash=? WHERE id=?", (canonical, prompt_id))
|
||||
|
||||
def archive_or_delete_library_prompts(
|
||||
self,
|
||||
archive_ids: list[int],
|
||||
delete_ids: list[int],
|
||||
*,
|
||||
expected_epoch: str,
|
||||
candidates: dict | None,
|
||||
) -> dict:
|
||||
"""Archive and delete what the user reviewed, all or nothing.
|
||||
|
||||
``archive_ids`` and ``delete_ids`` are the reviewed classes.
|
||||
``candidates`` maps each reviewed delete the route backed up to its
|
||||
fingerprint then (None: every reviewed delete, unchecked; tests and
|
||||
callers without a backup). A reviewed delete is deleted only when it
|
||||
still classifies ``delete`` and its fingerprint is unchanged; one that
|
||||
is now paired, and a reviewed archive now unpaired, are archived (the
|
||||
restorable direction); anything else is kept and counted.
|
||||
"""
|
||||
_validate_epoch(expected_epoch)
|
||||
for ids in (archive_ids, delete_ids):
|
||||
if not isinstance(ids, list) or any(
|
||||
type(pid) is not int or not 1 <= pid <= 2**63 - 1 for pid in ids
|
||||
):
|
||||
raise ValueError("prompt ids must be lists of positive integers")
|
||||
reviewed_delete = set(delete_ids)
|
||||
if not archive_ids and not delete_ids:
|
||||
raise ValueError("Nothing to delete or archive")
|
||||
if reviewed_delete & set(archive_ids):
|
||||
raise ValueError("A prompt cannot be both archived and deleted")
|
||||
prompt_ids = list(dict.fromkeys([*archive_ids, *delete_ids]))
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
classes = self._classify_library_delete(conn, prompt_ids)
|
||||
now_prints = self._fingerprints(conn, classes["delete"])
|
||||
deletes, archives, changed = [], list(classes["archive"]), 0
|
||||
for pid in classes["delete"]:
|
||||
if pid not in reviewed_delete:
|
||||
archives.append(pid) # reviewed as archive, now unpaired
|
||||
elif candidates is None or candidates.get(pid) == now_prints[pid]:
|
||||
deletes.append(pid)
|
||||
else:
|
||||
changed += 1
|
||||
changed += len(classes["protected"])
|
||||
image_links = self._image_links(conn, deletes)
|
||||
now = utc_now_iso()
|
||||
for pid in archives:
|
||||
self._repair_hash(conn, pid)
|
||||
conn.executemany(
|
||||
f"INSERT INTO {ARCHIVE}(prompt_id, archived_at) VALUES (?, ?)",
|
||||
[(pid, now) for pid in archives],
|
||||
)
|
||||
# A changed row is the newest edit the restore fingerprint reads.
|
||||
conn.executemany(
|
||||
"UPDATE prompts SET updated_at=? WHERE id=?",
|
||||
[(now, pid) for pid in archives],
|
||||
)
|
||||
deleted = delete_prompt_rows(conn, deletes)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"deleted": deleted,
|
||||
"archived": len(archives),
|
||||
"already_archived": len(classes["already_archived"]),
|
||||
"not_found": len(classes["not_found"]),
|
||||
"changed": changed,
|
||||
"image_links": image_links,
|
||||
}
|
||||
|
||||
def restore_library_prompts(
|
||||
self, prompt_ids: list[int], *, expected_epoch: str
|
||||
) -> dict:
|
||||
"""Put archived prompts back on the shelf, all or nothing.
|
||||
|
||||
Every requested id with an archive row is restored, whatever its
|
||||
protection or Library parent state: restoring only un-hides it.
|
||||
"""
|
||||
prompt_ids = self._checked_ids(prompt_ids, expected_epoch)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
restored, existing = [], set()
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for pid, archived in conn.execute(
|
||||
f"SELECT p.id,{_ARCHIVED} FROM prompts p " # nosec B608
|
||||
f"WHERE p.id IN ({marks})",
|
||||
chunk,
|
||||
):
|
||||
existing.add(pid)
|
||||
if archived:
|
||||
restored.append(pid)
|
||||
now = utc_now_iso()
|
||||
conn.executemany(
|
||||
f"DELETE FROM {ARCHIVE} WHERE prompt_id=?", [(pid,) for pid in restored]
|
||||
)
|
||||
conn.executemany(
|
||||
"UPDATE prompts SET updated_at=? WHERE id=?",
|
||||
[(now, pid) for pid in restored],
|
||||
)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"restored": len(restored),
|
||||
"unchanged": len(existing) - len(restored),
|
||||
"not_found": len(prompt_ids) - len(existing),
|
||||
}
|
||||
+26
-163
@@ -15,8 +15,6 @@ change:
|
||||
``cleanup_orphaned_images``), counted only under a source that can be read
|
||||
now, as the health check's ``missing_files`` judges them
|
||||
(``health.link_states``). It deletes database rows only, never a file.
|
||||
It also forgets the removal marks (#480) that hide nothing any more: the
|
||||
file is gone or changed, under a readable source (#501, ``_stale_marks``).
|
||||
|
||||
``preview_maintenance`` reads one snapshot on its own ``mode=ro`` connection
|
||||
(``health.readonly_uri``) and closes it in ``finally``; files are looked at
|
||||
@@ -43,16 +41,9 @@ from pathlib import Path
|
||||
|
||||
from .execution_contexts import LIVE_HINT_CTES
|
||||
from .health import BATCH_ROWS, link_states, readable_roots, readonly_uri
|
||||
from .image_removal_schema import TABLE as MARKS_TABLE
|
||||
from .image_removal_schema import clear_marks, has_marks_table
|
||||
from .pairs import PairDependencyError
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
try:
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
except ImportError:
|
||||
from utils.hashing import register_prompt_functions
|
||||
|
||||
MAINTENANCE_OPERATIONS = (
|
||||
"duplicate_prompts",
|
||||
"optimize",
|
||||
@@ -89,12 +80,6 @@ PROTECTIONS = (
|
||||
"with Library state (a title or an explicit parent)",
|
||||
"with Library state (a title or an explicit parent)",
|
||||
),
|
||||
(
|
||||
# #443: an archived prompt is restorable, never pruned.
|
||||
"EXISTS (SELECT 1 FROM prompt_archive pa WHERE pa.prompt_id = p.id)",
|
||||
"archived in the Library",
|
||||
"archived in the Library",
|
||||
),
|
||||
(
|
||||
"p.id IN (SELECT prompt_id FROM live_hint_prompts)",
|
||||
"used by recorded runs",
|
||||
@@ -103,8 +88,7 @@ PROTECTIONS = (
|
||||
)
|
||||
|
||||
# Exempt references: images, pairs, the __protected__ tag, Library parent
|
||||
# state, the Library archive, and live recorded-run hints (computed once,
|
||||
# see LIVE_HINT_CTES).
|
||||
# state, and live recorded-run hints (computed once, see LIVE_HINT_CTES).
|
||||
ORPHANED_PROMPTS_SQL = (
|
||||
"WITH "
|
||||
+ LIVE_HINT_CTES
|
||||
@@ -198,8 +182,6 @@ class MaintenanceDraft:
|
||||
protected: dict
|
||||
media: dict
|
||||
details: list = field(default_factory=list)
|
||||
# missing_links only: locators of removal marks whose file is gone (#501).
|
||||
marks: list = field(default_factory=list)
|
||||
|
||||
def public(self):
|
||||
"""The plan fields a response may carry (no fingerprint, no targets)."""
|
||||
@@ -302,96 +284,11 @@ def _delete_ids(conn, table, ids):
|
||||
return removed
|
||||
|
||||
|
||||
def _mark_rows(conn, only=None):
|
||||
"""``[(locator, size, mtime_ns)]`` of the removal marks (all, or those
|
||||
among *only*), sorted by locator."""
|
||||
if not has_marks_table(conn):
|
||||
return []
|
||||
sql = f"SELECT locator, size, mtime_ns FROM {MARKS_TABLE}" # nosec B608
|
||||
if only is None:
|
||||
return sorted(tuple(row) for row in conn.execute(sql))
|
||||
found = []
|
||||
for batch in _batches(sorted(only)):
|
||||
found.extend(
|
||||
tuple(row)
|
||||
for row in conn.execute(
|
||||
f"{sql} WHERE locator IN ({_marks(batch)})", batch # nosec B608
|
||||
)
|
||||
)
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def _lists_something(folder):
|
||||
"""Whether *folder* holds at least one entry (an unplugged drive's mount
|
||||
point, or an unbound volume, is an empty folder that reads fine)."""
|
||||
try:
|
||||
with os.scandir(folder) as entries:
|
||||
return next(entries, None) is not None
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _name_listed(path):
|
||||
"""Whether *path*'s folder lists its name, compared under ``normcase``.
|
||||
|
||||
A locator is ``normcase``d; on a case-sensitive NTFS folder ``lstat`` of
|
||||
the lower-cased spelling fails for a file that is there.
|
||||
"""
|
||||
folder, name = os.path.split(path)
|
||||
wanted = os.path.normcase(name)
|
||||
try:
|
||||
with os.scandir(folder) as entries:
|
||||
return any(os.path.normcase(entry.name) == wanted for entry in entries)
|
||||
except OSError:
|
||||
return True # cannot tell: keep the mark
|
||||
|
||||
|
||||
def _stale_mark(locator, size, mtime_ns):
|
||||
"""Whether the file of a mark is gone or has other bytes (blocking)."""
|
||||
try:
|
||||
info = os.lstat(locator)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return os.path.isdir(os.path.dirname(locator)) and not _name_listed(locator)
|
||||
except OSError:
|
||||
return False
|
||||
return (info.st_size, info.st_mtime_ns) != (size, mtime_ns)
|
||||
|
||||
|
||||
def _stale_marks(marks, roots):
|
||||
"""``[(locator, owner)]`` of the marks that hide nothing any more (#501).
|
||||
|
||||
*marks* are ``(locator, size, mtime_ns)`` rows. A mark is stale when its
|
||||
file was changed, or is gone from a folder that still exists. Only marks
|
||||
under a source that can be read now and lists something are judged (an
|
||||
unplugged drive's marks, or its empty mount point's, stay); any doubt
|
||||
keeps the mark. *owner* is the index of the longest source holding it.
|
||||
Locators are ``normcase(realpath)`` paths, so the sources are too.
|
||||
"""
|
||||
readable = readable_roots(roots)
|
||||
prefixes = [
|
||||
os.path.normcase(os.path.realpath(root)).rstrip("\\/") + os.sep
|
||||
for root in roots
|
||||
]
|
||||
judged = {}
|
||||
stale = []
|
||||
for locator, size, mtime_ns in marks:
|
||||
owners = [i for i, prefix in enumerate(prefixes) if locator.startswith(prefix)]
|
||||
if not owners:
|
||||
continue # under no source now (a source removed from the settings)
|
||||
owner = max(owners, key=lambda i: len(prefixes[i]))
|
||||
if owner not in judged:
|
||||
judged[owner] = readable[owner] and _lists_something(roots[owner])
|
||||
if judged[owner] and _stale_mark(locator, size, mtime_ns):
|
||||
stale.append((locator, owner))
|
||||
return stale
|
||||
|
||||
|
||||
def _read_snapshot(db_path, read):
|
||||
"""``read(conn)`` on one ``mode=ro`` snapshot; the connection always closes."""
|
||||
conn = sqlite3.connect(readonly_uri(db_path), uri=True)
|
||||
try:
|
||||
conn.row_factory = sqlite3.Row
|
||||
register_prompt_functions(conn) # LIVE_HINT_CTES calls them (#460)
|
||||
conn.execute("BEGIN") # one read snapshot for the whole plan
|
||||
return read(conn)
|
||||
finally:
|
||||
@@ -528,11 +425,8 @@ class MaintenanceOperations:
|
||||
if operation == "workflow_storage":
|
||||
return self._plan_workflow_storage()
|
||||
if operation == "missing_links":
|
||||
rows, marks = _read_snapshot(
|
||||
self.model.db_path,
|
||||
lambda conn: (self._link_rows(conn), _mark_rows(conn)),
|
||||
)
|
||||
return self._plan_missing_links(rows, roots, _stale_marks(marks, roots))
|
||||
rows = _read_snapshot(self.model.db_path, self._link_rows)
|
||||
return self._plan_missing_links(rows, roots)
|
||||
reader = (
|
||||
self._plan_duplicate_prompts
|
||||
if operation == "duplicate_prompts"
|
||||
@@ -663,15 +557,13 @@ class MaintenanceOperations:
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _links_fingerprint(candidates, stale=()):
|
||||
def _links_fingerprint(candidates):
|
||||
fingerprint = _Fingerprint("missing_links")
|
||||
for row, owner in candidates:
|
||||
fingerprint.add([row[0], row[1], row[3], owner])
|
||||
for locator, owner in stale:
|
||||
fingerprint.add(["mark", locator, owner])
|
||||
return fingerprint.hexdigest()
|
||||
|
||||
def _plan_missing_links(self, rows, roots, stale=()):
|
||||
def _plan_missing_links(self, rows, roots):
|
||||
readable = readable_roots(roots)
|
||||
candidates, unavailable, outside = [], 0, 0
|
||||
for row, owner, state in link_states(rows, roots, readable):
|
||||
@@ -699,47 +591,29 @@ class MaintenanceOperations:
|
||||
)
|
||||
)
|
||||
count = len(candidates)
|
||||
parts = []
|
||||
if count:
|
||||
parts.append(
|
||||
_plural(count, "link to a missing file", "links to missing files")
|
||||
)
|
||||
if stale:
|
||||
parts.append(
|
||||
_plural(
|
||||
len(stale),
|
||||
"removed image whose file is gone or changed",
|
||||
"removed images whose files are gone or changed",
|
||||
)
|
||||
)
|
||||
details = [
|
||||
{
|
||||
"label": _relative(row[1], roots[owner]),
|
||||
"detail": f"{_source_name(roots[owner], owner)} · the file is missing",
|
||||
}
|
||||
for row, owner in candidates
|
||||
] + [
|
||||
{
|
||||
"label": _relative(locator, os.path.realpath(roots[owner])),
|
||||
"detail": (
|
||||
f"{_source_name(roots[owner], owner)} · removed image, "
|
||||
"the file is gone or changed"
|
||||
),
|
||||
}
|
||||
for locator, owner in stale
|
||||
]
|
||||
return MaintenanceDraft(
|
||||
operation="missing_links",
|
||||
fingerprint=self._links_fingerprint(candidates, stale),
|
||||
fingerprint=self._links_fingerprint(candidates),
|
||||
targets=[row[0] for row, _owner in candidates],
|
||||
scope={
|
||||
"count": count + len(stale),
|
||||
"label": "; ".join(parts) or "No links to missing files",
|
||||
"count": count,
|
||||
"label": (
|
||||
_plural(count, "link to a missing file", "links to missing files")
|
||||
if count
|
||||
else "No links to missing files"
|
||||
),
|
||||
},
|
||||
protected={"count": unavailable + outside, "reasons": reasons},
|
||||
media={"deletes_files": False, "label": LINKS_ONLY},
|
||||
details=details[:MAX_DETAILS],
|
||||
marks=[locator for locator, _owner in stale],
|
||||
details=[
|
||||
{
|
||||
"label": _relative(row[1], roots[owner]),
|
||||
"detail": (
|
||||
f"{_source_name(roots[owner], owner)} · the file is missing"
|
||||
),
|
||||
}
|
||||
for row, owner in candidates[:MAX_DETAILS]
|
||||
],
|
||||
)
|
||||
|
||||
def _plan_optimize(self):
|
||||
@@ -789,18 +663,14 @@ class MaintenanceOperations:
|
||||
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
def apply_maintenance(
|
||||
self, operation, fingerprint, targets, roots=(), marks=()
|
||||
) -> dict:
|
||||
def apply_maintenance(self, operation, fingerprint, targets, roots=()) -> dict:
|
||||
"""Apply a reviewed plan exactly, or raise PlanChanged (blocking).
|
||||
|
||||
Inside ``BEGIN IMMEDIATE`` the fingerprint is computed again on the
|
||||
write connection; when it differs nothing is written. ``targets`` and
|
||||
``roots`` are the plan's. Returns ``{changed, rows}``: how many
|
||||
records changed and up to ``MAX_RESULT_ROWS`` descriptions of them
|
||||
(``{prompt_id[, kept_id]}`` or ``{image_path}``). ``marks`` are a
|
||||
missing-links plan's stale removal marks; that apply also returns
|
||||
``marks``, how many it forgot.
|
||||
(``{prompt_id[, kept_id]}`` or ``{image_path}``).
|
||||
"""
|
||||
if operation not in MAINTENANCE_OPERATIONS or operation == "workflow_storage":
|
||||
raise ValueError(f"Unknown maintenance operation {operation!r}")
|
||||
@@ -813,12 +683,9 @@ class MaintenanceOperations:
|
||||
appliers = {
|
||||
"duplicate_prompts": self._apply_duplicate_prompts,
|
||||
"unused_components": self._apply_unused_components,
|
||||
"missing_links": self._apply_missing_links,
|
||||
}
|
||||
with self._pair_transaction() as conn:
|
||||
if operation == "missing_links":
|
||||
return self._apply_missing_links(
|
||||
conn, fingerprint, targets, roots, list(marks)
|
||||
)
|
||||
return appliers[operation](conn, fingerprint, targets, roots)
|
||||
|
||||
def _apply_duplicate_prompts(self, conn, fingerprint, targets, _roots):
|
||||
@@ -847,7 +714,7 @@ class MaintenanceOperations:
|
||||
]
|
||||
return {"changed": changed, "rows": rows}
|
||||
|
||||
def _apply_missing_links(self, conn, fingerprint, targets, roots, marks):
|
||||
def _apply_missing_links(self, conn, fingerprint, targets, roots):
|
||||
rows = []
|
||||
for batch in _batches(sorted(targets)):
|
||||
rows.extend(
|
||||
@@ -864,14 +731,10 @@ class MaintenanceOperations:
|
||||
if state != "missing": # back, or its source went away
|
||||
raise PlanChanged("missing_links")
|
||||
candidates.append((row, owner))
|
||||
stale = _stale_marks(_mark_rows(conn, marks) if marks else [], roots)
|
||||
if [locator for locator, _owner in stale] != marks: # a mark went or is valid
|
||||
raise PlanChanged("missing_links")
|
||||
if self._links_fingerprint(candidates, stale) != fingerprint:
|
||||
if self._links_fingerprint(candidates) != fingerprint:
|
||||
raise PlanChanged("missing_links")
|
||||
changed = _delete_ids(conn, "generated_images", [row[0] for row in rows])
|
||||
return {
|
||||
"changed": changed,
|
||||
"marks": clear_marks(conn, marks),
|
||||
"rows": [{"image_path": row[1]} for row in rows[:MAX_RESULT_ROWS]],
|
||||
}
|
||||
|
||||
@@ -38,14 +38,7 @@ from .review_schema import (
|
||||
review_without_annotations_error,
|
||||
)
|
||||
from .saved_views import migrate_saved_views, saved_views_schema_error
|
||||
|
||||
try:
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
except ImportError:
|
||||
from utils.hashing import register_prompt_functions
|
||||
from .image_categories import image_categories_schema_error, migrate_image_categories
|
||||
from .image_removal_schema import image_removals_schema_error, migrate_image_removals
|
||||
from .prompt_archive import migrate_prompt_archive, prompt_archive_schema_error
|
||||
from .workflow_documents import (
|
||||
DOCUMENT_ID_COLUMNS,
|
||||
migrate_workflow_documents,
|
||||
@@ -160,15 +153,12 @@ def configure_connection(conn: sqlite3.Connection) -> None:
|
||||
(common on Windows) let other writers' polling busy handlers starve past
|
||||
BUSY_TIMEOUT_SECONDS and fail with "database is locked" (#242). Without
|
||||
WAL (for example an in-memory database) SQLite's FULL default is kept.
|
||||
It also registers the prompt-key SQL functions (register_prompt_functions).
|
||||
"""
|
||||
journal_mode = conn.execute("PRAGMA journal_mode = WAL").fetchone()[0]
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}")
|
||||
if str(journal_mode).lower() == "wal":
|
||||
conn.execute("PRAGMA synchronous = NORMAL")
|
||||
# History reads and the live-hint rule call Python's prompt key (#460)
|
||||
register_prompt_functions(conn)
|
||||
|
||||
|
||||
class PromptModel:
|
||||
@@ -368,10 +358,6 @@ class PromptModel:
|
||||
migrate_autotag_state(conn)
|
||||
# Image categories (#332): independent of every table above.
|
||||
migrate_image_categories(conn)
|
||||
# The Library parent archive (#443): independent of every table above.
|
||||
migrate_prompt_archive(conn)
|
||||
# Removal marks (#480): independent of every table above.
|
||||
migrate_image_removals(conn)
|
||||
# Last: the older generated_images rebuilds run only before file_path.
|
||||
migrate_workflow_documents(conn)
|
||||
|
||||
@@ -1248,16 +1234,6 @@ class PromptModel:
|
||||
)
|
||||
if categories_error:
|
||||
return False, categories_error
|
||||
archive_error = prompt_archive_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if archive_error:
|
||||
return False, archive_error
|
||||
removals_error = image_removals_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if removals_error:
|
||||
return False, removals_error
|
||||
views_error = saved_views_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
|
||||
+8
-11
@@ -18,14 +18,12 @@ from .models import PromptModel, normalize_image_path, utc_now_iso
|
||||
from .pairs import PairDependencyError, PairOperations # noqa: F401 (re-exported)
|
||||
from .library import LibraryOperations, titled
|
||||
from .image_favorites import ImageFavoriteOperations
|
||||
from .image_removal import ImageRemovalOperations, removed_from_library
|
||||
from .image_collections import ImageCollectionOperations
|
||||
from .annotations import AnnotationOperations
|
||||
from .review import ReviewOperations
|
||||
from .review_decisions import ReviewDecisionOperations
|
||||
from .saved_views import SavedViewOperations
|
||||
from .library_commands import LibraryCommandOperations
|
||||
from .library_archive import LibraryArchiveOperations, delete_prompt_rows
|
||||
from .vocabulary import VocabularyOperations
|
||||
from .execution_history import ExecutionHistoryOperations
|
||||
from .execution_contexts import ExecutionContextOperations
|
||||
@@ -450,7 +448,6 @@ class PromptDatabase(
|
||||
PairOperations,
|
||||
LibraryOperations,
|
||||
LibraryCommandOperations,
|
||||
LibraryArchiveOperations,
|
||||
VocabularyOperations,
|
||||
ExecutionHistoryOperations,
|
||||
ExecutionContextOperations,
|
||||
@@ -468,7 +465,6 @@ class PromptDatabase(
|
||||
WorkflowStorageOperations,
|
||||
CategoryImportOperations,
|
||||
ImageCategoryOperations,
|
||||
ImageRemovalOperations,
|
||||
):
|
||||
"""Database operations class for managing prompts."""
|
||||
|
||||
@@ -1810,11 +1806,6 @@ class PromptDatabase(
|
||||
"""
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
if removed_from_library(conn, image_path):
|
||||
# #480: the user removed this exact file from the library; its
|
||||
# bytes are unchanged, so nothing links it again.
|
||||
self.logger.debug("Not linking a file removed from the library")
|
||||
return 0
|
||||
filename = os.path.basename(image_path)
|
||||
file_path = normalize_image_path(image_path)
|
||||
file_size, width, height, image_format = _coerce_file_info(metadata)
|
||||
@@ -2353,10 +2344,16 @@ class PromptDatabase(
|
||||
Returns:
|
||||
Number of prompts actually deleted.
|
||||
"""
|
||||
count = 0
|
||||
with self._pair_transaction() as conn:
|
||||
self._ensure_prompts_unpaired(conn, prompt_ids)
|
||||
# The workspace Library's delete runs the same statements (#443).
|
||||
count = delete_prompt_rows(conn, list(dict.fromkeys(prompt_ids)))
|
||||
documents = documents_of_rows(conn, "prompt_id", prompt_ids)
|
||||
for pid in prompt_ids:
|
||||
conn.execute("DELETE FROM generated_images WHERE prompt_id = ?", (pid,))
|
||||
cursor = conn.execute("DELETE FROM prompts WHERE id = ?", (pid,))
|
||||
if cursor.rowcount > 0:
|
||||
count += 1
|
||||
release_documents(conn, documents)
|
||||
conn.commit()
|
||||
return count
|
||||
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
"""Library parent archive (#443, D152): prompts hidden from the shelf, restorable.
|
||||
|
||||
A row means the Library parent is archived; no row means it is on the shelf.
|
||||
Nothing else about the prompt changes: its text, metadata, tags, run count,
|
||||
image links, pairs and recorded runs stay. Restore deletes the row. Deleting
|
||||
the prompt (any path) drops the row by cascade.
|
||||
|
||||
A table of its own, not a column: ``prompt_library_state`` is validated by its
|
||||
exact definition, so a new column there would make older builds refuse the
|
||||
upgraded database, while older builds ignore an unknown table (they show the
|
||||
archived prompts again and keep the rows). Optional in old backups, required
|
||||
once a database is current.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
TABLE = "prompt_archive"
|
||||
MAX_STAMP_BYTES = 64
|
||||
# What ``utc_now_iso`` writes starts with a date; anything else was not written here.
|
||||
_STAMP_GLOB = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T*"
|
||||
_DDL = (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
|
||||
"archived_at TEXT NOT NULL, "
|
||||
"FOREIGN KEY(prompt_id) REFERENCES prompts(id) ON DELETE CASCADE)"
|
||||
)
|
||||
_COLUMNS = (("prompt_id", "INTEGER", 0, 1), ("archived_at", "TEXT", 1, 0))
|
||||
_FOREIGN_KEYS = {("prompt_id", "prompts", "id", "CASCADE")}
|
||||
# A predicate on ``p`` (the prompts alias): the parent is not archived.
|
||||
SHELVED = f"NOT EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)"
|
||||
ARCHIVED = f"EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection, allow_missing: bool) -> str | None:
|
||||
objects = conn.execute(
|
||||
"SELECT type,sql FROM sqlite_master WHERE name=? "
|
||||
"OR (type IN ('index','trigger') AND tbl_name=?)",
|
||||
(TABLE, TABLE),
|
||||
).fetchall()
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing prompt archive schema"
|
||||
tables = [sql for kind, sql in objects if kind == "table"]
|
||||
if len(tables) != 1 or len(objects) != 1:
|
||||
return "Prompt archive name is reserved by an unsupported schema object"
|
||||
if _compact(tables[0]) != _compact(_DDL):
|
||||
return "Prompt archive has an invalid table definition"
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({TABLE})")
|
||||
)
|
||||
if columns != _COLUMNS:
|
||||
return "Prompt archive has invalid columns"
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({TABLE})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS:
|
||||
return "Prompt archive has invalid foreign key definitions"
|
||||
return None
|
||||
|
||||
|
||||
def prompt_archive_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> str | None:
|
||||
"""Reject an unsupported archive schema or values; reads only.
|
||||
|
||||
A missing table is acceptable for backups that predate it unless
|
||||
``allow_missing`` is false.
|
||||
"""
|
||||
error = _structure_error(conn, allow_missing)
|
||||
if error is not None:
|
||||
return error
|
||||
if not conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone():
|
||||
return None
|
||||
if conn.execute(f"PRAGMA foreign_key_check({TABLE})").fetchone():
|
||||
return "Prompt archive contains foreign key violations"
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {TABLE} WHERE typeof(archived_at)!='text' " # nosec B608
|
||||
f"OR length(CAST(archived_at AS BLOB))>{MAX_STAMP_BYTES} "
|
||||
f"OR archived_at NOT GLOB '{_STAMP_GLOB}' LIMIT 1"
|
||||
).fetchone():
|
||||
return "Prompt archive contains invalid values"
|
||||
return None
|
||||
|
||||
|
||||
def migrate_prompt_archive(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table atomically; validate before and after (no backfill)."""
|
||||
conn.execute("SAVEPOINT prompt_archive_schema")
|
||||
try:
|
||||
error = _structure_error(conn, True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
error = _structure_error(conn, False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE prompt_archive_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO prompt_archive_schema")
|
||||
conn.execute("RELEASE prompt_archive_schema")
|
||||
raise
|
||||
@@ -149,7 +149,6 @@ def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
|
||||
collections (#331).
|
||||
"""
|
||||
from .library import _ELIGIBLE
|
||||
from .prompt_archive import SHELVED, TABLE as ARCHIVE
|
||||
|
||||
has_state = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND "
|
||||
@@ -158,15 +157,9 @@ def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
|
||||
has_pairs = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='prompt_pairs'"
|
||||
).fetchone()
|
||||
has_archive = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (ARCHIVE,)
|
||||
).fetchone()
|
||||
if has_state and has_pairs:
|
||||
# Archived parents (#443) are off the shelf, as in library_counts.
|
||||
positives = conn.execute(
|
||||
"SELECT COUNT(*) FROM prompts p WHERE "
|
||||
+ _ELIGIBLE
|
||||
+ (" AND " + SHELVED if has_archive else "")
|
||||
"SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE
|
||||
).fetchone()[0]
|
||||
else:
|
||||
positives = _count(conn, "prompts")
|
||||
|
||||
+12
-29
@@ -20,8 +20,7 @@ counted, hashed into the token and renamed with the prompts and pairs.
|
||||
|
||||
import sqlite3
|
||||
|
||||
from .library import _ELIGIBLE, SHELVED_PAIR, UNTAGGED, _escape, _limit
|
||||
from .prompt_archive import SHELVED
|
||||
from .library import _ELIGIBLE, UNTAGGED, _escape, _limit
|
||||
from .library_commands import _TOKEN, LibraryCommandError, _validate_epoch
|
||||
from .library_tokens import canonical_token
|
||||
from .models import utc_now_iso
|
||||
@@ -41,21 +40,17 @@ _IMAGE_FILES = (
|
||||
|
||||
def _tag_rows(page: str, order: str = "page.id") -> str:
|
||||
"""Count rows for the registry rows selected by ``page`` (id, name)."""
|
||||
# Eligible parent components on the shelf carrying a tag in the page,
|
||||
# plus the positive parent of every active pair carrying it (always
|
||||
# eligible) when that parent is on the shelf: archived parents (#443)
|
||||
# are hidden with their pairs, so Browse lists what was counted.
|
||||
# Eligible parent components carrying a tag in the page, plus the
|
||||
# positive parent of every active pair carrying it (always eligible).
|
||||
return (
|
||||
"WITH page AS (" + page + "),"
|
||||
"owners(tag_id,pid) AS ("
|
||||
"SELECT pt.tag_id,p.id FROM prompts p "
|
||||
"JOIN prompt_tags pt ON pt.prompt_id=p.id "
|
||||
"WHERE pt.tag_id IN (SELECT id FROM page) AND " + _ELIGIBLE + " "
|
||||
"AND " + SHELVED + " "
|
||||
"UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id "
|
||||
"WHERE pp.archived=0 AND " + SHELVED_PAIR + " "
|
||||
"AND ppt.tag_id IN (SELECT id FROM page)),"
|
||||
"WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)),"
|
||||
"images(tag_id,n) AS (" + _IMAGE_FILES + " "
|
||||
"WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) "
|
||||
"SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count,"
|
||||
@@ -84,10 +79,8 @@ _RANKED_TAG_PAGE = (
|
||||
"WITH owners(tag_id,pid) AS ("
|
||||
"SELECT pt.tag_id,p.id FROM prompts p "
|
||||
"JOIN prompt_tags pt ON pt.prompt_id=p.id WHERE " + _ELIGIBLE + " "
|
||||
"AND " + SHELVED + " "
|
||||
"UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id WHERE pp.archived=0 "
|
||||
"AND " + SHELVED_PAIR + "),"
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id WHERE pp.archived=0),"
|
||||
"counted AS (SELECT t.id,t.name,COUNT(DISTINCT o.pid) AS prompt_count "
|
||||
"FROM tags t LEFT JOIN owners o ON o.tag_id=t.id WHERE {search} "
|
||||
"GROUP BY t.id),"
|
||||
@@ -101,13 +94,7 @@ _RANKED_TAG_PAGE = (
|
||||
"ORDER BY page.prompt_count {direction},page.id"
|
||||
)
|
||||
_UNTAGGED_COUNT = (
|
||||
"SELECT COUNT(*) FROM prompts p WHERE ("
|
||||
+ _ELIGIBLE
|
||||
+ ") AND "
|
||||
+ UNTAGGED
|
||||
# Untagged prompts opens the shelf, which hides archived parents (#443).
|
||||
+ " AND "
|
||||
+ SHELVED
|
||||
"SELECT COUNT(*) FROM prompts p WHERE (" + _ELIGIBLE + ") AND " + UNTAGGED
|
||||
)
|
||||
_TAG_ONE = _tag_rows("SELECT id,name FROM tags WHERE id=?")
|
||||
# Every owner kind a tag can sit on, for the delete confirmation.
|
||||
@@ -147,9 +134,7 @@ def _validate_tag_id(tag_id: object) -> None:
|
||||
|
||||
|
||||
# Category values in Tags & categories (#332 §7): the prompt-backed values
|
||||
# (nonblank values on eligible parents or on active pairs; ``prompt_count``
|
||||
# counts only parents on the shelf, so a value only archived parents (#443)
|
||||
# hold stays listed with 0, as a tag does; grouped exactly, so
|
||||
# (nonblank values on eligible parents or on active pairs, grouped exactly, so
|
||||
# whitespace and case variants stay distinct; /library/options lists the same
|
||||
# prompt-backed set with its own query) plus the values only image rows hold.
|
||||
# A prompt-backed value is ordered by its oldest owning positive; an image-only
|
||||
@@ -158,17 +143,15 @@ def _validate_tag_id(tag_id: object) -> None:
|
||||
# files missing on disk. No FULL OUTER JOIN, so it runs on SQLite 3.31+.
|
||||
_IMAGE_ONLY_FIRST_PID = 2**63 - 1
|
||||
_CATEGORY_ROWS = (
|
||||
"WITH owners(value,pid,shelved) AS ("
|
||||
"SELECT p.category,p.id," + SHELVED + " FROM prompts p "
|
||||
"WHERE p.category IS NOT NULL AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT pp.category,pp.positive_prompt_id," + SHELVED_PAIR + " "
|
||||
"FROM prompt_pairs pp "
|
||||
"WITH owners(value,pid) AS ("
|
||||
"SELECT p.category,p.id FROM prompts p WHERE p.category IS NOT NULL "
|
||||
"AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT pp.category,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"WHERE pp.archived=0 AND pp.category IS NOT NULL AND TRIM(pp.category)!=''),"
|
||||
"image_values(value,image_count) AS ("
|
||||
"SELECT category,COUNT(*) FROM image_categories GROUP BY category),"
|
||||
"grouped(value,first_pid,prompt_count) AS ("
|
||||
"SELECT value,MIN(pid),COUNT(DISTINCT CASE WHEN shelved THEN pid END) "
|
||||
"FROM owners GROUP BY value "
|
||||
"SELECT value,MIN(pid),COUNT(DISTINCT pid) FROM owners GROUP BY value "
|
||||
"UNION ALL SELECT value," + str(_IMAGE_ONLY_FIRST_PID) + ",0 FROM image_values "
|
||||
"WHERE value NOT IN (SELECT value FROM owners)),"
|
||||
"counted AS (SELECT g.value,g.first_pid,g.prompt_count,"
|
||||
|
||||
+13
-39
@@ -19,7 +19,6 @@ Split into domain-specific mixins for maintainability:
|
||||
- SystemRoutesMixin: System health check, Overview and usage statistics
|
||||
- LibraryLocationRoutesMixin: where the library lives, Move library, Keep it here
|
||||
- MaintenanceRoutesMixin: reviewed maintenance plans (preview, apply)
|
||||
- LibraryDeleteRoutesMixin: the Library's Delete… (archive paired) and Restore
|
||||
- WorkflowStorageRoutesMixin: store each workflow once, then Reclaim space
|
||||
- DuplicateFilesMixin: duplicate image files as find and removal jobs
|
||||
- LoraImportRoutesMixin: the reviewed, staged LoRA reference import job
|
||||
@@ -41,10 +40,8 @@ from aiohttp import web
|
||||
from PIL import Image
|
||||
|
||||
from .prompts import PromptRoutesMixin
|
||||
from .revalidation import revalidated_response
|
||||
from .pairs import PairRoutesMixin
|
||||
from .library import LibraryRoutesMixin
|
||||
from .library_delete import LibraryDeleteRoutesMixin
|
||||
from .vocabulary import VocabularyRoutesMixin
|
||||
from .annotations import AnnotationRoutesMixin
|
||||
from .review import ReviewRoutesMixin
|
||||
@@ -77,7 +74,6 @@ from .autotag_routes import AutotagRoutesMixin
|
||||
from .lora_integration import LoraIntegrationMixin
|
||||
from .lora_import_routes import LoraImportRoutesMixin
|
||||
from .image_categories import ImageCategoryRoutesMixin
|
||||
from .image_delete import ImageDeleteMixin
|
||||
from .source_rescope import SourceRescopeMixin
|
||||
|
||||
try:
|
||||
@@ -317,7 +313,6 @@ async def _gzip_middleware(request, handler):
|
||||
class PromptManagerAPI(
|
||||
PairRoutesMixin,
|
||||
LibraryRoutesMixin,
|
||||
LibraryDeleteRoutesMixin,
|
||||
VocabularyRoutesMixin,
|
||||
AnnotationRoutesMixin,
|
||||
ReviewRoutesMixin,
|
||||
@@ -348,7 +343,6 @@ class PromptManagerAPI(
|
||||
LoraIntegrationMixin,
|
||||
LoraImportRoutesMixin,
|
||||
ImageCategoryRoutesMixin,
|
||||
ImageDeleteMixin,
|
||||
SourceRescopeMixin,
|
||||
):
|
||||
"""REST API handler for PromptManager operations and web interface.
|
||||
@@ -429,12 +423,6 @@ class PromptManagerAPI(
|
||||
# Reviewed maintenance plans under single-use tokens; a restart
|
||||
# forgets them (py/api/maintenance.py).
|
||||
self._maintenance_plans = PlanStore()
|
||||
# #480: the removal marks the workspace listing hides, read once now so
|
||||
# a listing page reads no table until a mark or the database changes.
|
||||
try:
|
||||
self._removal_marks()
|
||||
except Exception:
|
||||
self.logger.warning("Removal marks could not be read", exc_info=True)
|
||||
|
||||
# A safety backup interrupted by a crash leaves its temp copy behind.
|
||||
try:
|
||||
@@ -665,45 +653,40 @@ class PromptManagerAPI(
|
||||
|
||||
@routes.get("/prompt_manager/workspace")
|
||||
async def serve_workspace_ui(request: web.Request) -> web.Response:
|
||||
return self._serve_html_page(request, "workspace.html", "Workspace")
|
||||
return self._serve_html_page("workspace.html", "Workspace")
|
||||
|
||||
@routes.get("/prompt_manager/web")
|
||||
async def serve_web_ui(request):
|
||||
return self._serve_html_page(request, "index.html", "Web UI", cache=False)
|
||||
return self._serve_html_page("index.html", "Web UI", cache=False)
|
||||
|
||||
@routes.get("/prompt_manager/gallery.html")
|
||||
async def serve_gallery_ui(request):
|
||||
return self._serve_html_page(request, "metadata.html", "Gallery")
|
||||
return self._serve_html_page("metadata.html", "Gallery")
|
||||
|
||||
@routes.get("/prompt_manager/admin")
|
||||
async def serve_admin_ui(request):
|
||||
return self._serve_html_page(request, "admin.html", "Admin UI")
|
||||
return self._serve_html_page("admin.html", "Admin UI")
|
||||
|
||||
@routes.get("/prompt_manager/gallery")
|
||||
async def serve_gallery_admin_ui(request):
|
||||
return self._serve_html_page(request, "gallery.html", "Gallery")
|
||||
return self._serve_html_page("gallery.html", "Gallery")
|
||||
|
||||
# ── Static file serving ───────────────────────────────────────
|
||||
|
||||
@routes.get("/prompt_manager/lib/{filepath:.*}")
|
||||
async def serve_lib_static(request):
|
||||
"""Serve static library files (JS, CSS) from web/lib directory."""
|
||||
return self._serve_static_file(
|
||||
request, "lib", request.match_info.get("filepath")
|
||||
)
|
||||
return self._serve_static_file("lib", request.match_info.get("filepath"))
|
||||
|
||||
@routes.get("/prompt_manager/js/{filepath:.*}")
|
||||
async def serve_js_static(request):
|
||||
"""Serve static JavaScript files from web/js directory."""
|
||||
return self._serve_static_file(
|
||||
request, "js", request.match_info.get("filepath")
|
||||
)
|
||||
return self._serve_static_file("js", request.match_info.get("filepath"))
|
||||
|
||||
# ── Register domain-specific routes from mixins ───────────────
|
||||
|
||||
self._register_pair_routes(routes)
|
||||
self._register_library_routes(routes)
|
||||
self._register_library_delete_routes(routes)
|
||||
self._register_vocabulary_routes(routes)
|
||||
self._register_annotation_routes(routes)
|
||||
self._register_review_routes(routes)
|
||||
@@ -727,7 +710,6 @@ class PromptManagerAPI(
|
||||
self._register_lora_routes(routes)
|
||||
self._register_lora_import_routes(routes)
|
||||
self._register_image_category_routes(routes)
|
||||
self._register_image_delete_routes(routes)
|
||||
self._register_rescope_routes(routes)
|
||||
|
||||
# Register gzip compression middleware (once)
|
||||
@@ -744,11 +726,8 @@ class PromptManagerAPI(
|
||||
|
||||
self.logger.info("All routes registered with decorator pattern")
|
||||
|
||||
def _serve_static_file(self, request, subdir, filepath):
|
||||
"""Serve ``web/<subdir>/<filepath>``; 403 outside it, 404 when missing.
|
||||
|
||||
A served file is revalidated on every reuse (#478, ``revalidation``).
|
||||
"""
|
||||
def _serve_static_file(self, subdir, filepath):
|
||||
"""Serve ``web/<subdir>/<filepath>``; 403 outside it, 404 when missing."""
|
||||
root = os.path.join(_get_project_root(), "web", subdir)
|
||||
exact = STATIC_EXACT_FILES.get((subdir, filepath))
|
||||
# An exact file passes the same containment check, its extension only.
|
||||
@@ -763,13 +742,10 @@ class PromptManagerAPI(
|
||||
return web.Response(text="Not Found", status=404)
|
||||
with open(file_path, "rb") as f:
|
||||
content = f.read()
|
||||
return revalidated_response(request, content, content_type)
|
||||
return web.Response(body=content, content_type=content_type)
|
||||
|
||||
def _serve_html_page(self, request, filename, title, cache=True):
|
||||
"""Serve ``web/<filename>``; 404 when missing, 500 when unreadable.
|
||||
|
||||
A served page is revalidated on every reuse (#478, ``revalidation``).
|
||||
"""
|
||||
def _serve_html_page(self, filename, title, cache=True):
|
||||
"""Serve ``web/<filename>``; 404 when missing, 500 when unreadable."""
|
||||
html_path = os.path.join(_get_project_root(), "web", filename)
|
||||
try:
|
||||
if cache and html_path in self._html_cache:
|
||||
@@ -786,9 +762,7 @@ class PromptManagerAPI(
|
||||
content_type="text/html",
|
||||
status=404,
|
||||
)
|
||||
return revalidated_response(
|
||||
request, content.encode("utf-8"), "text/html", charset="utf-8"
|
||||
)
|
||||
return web.Response(text=content, content_type="text/html", charset="utf-8")
|
||||
except Exception:
|
||||
self.logger.exception(f"Failed to load {title}")
|
||||
return web.Response(
|
||||
|
||||
+7
-22
@@ -36,13 +36,10 @@ except ImportError:
|
||||
|
||||
try:
|
||||
from ...utils import exit_signal
|
||||
from ...utils.media_folders import is_skipped_folder
|
||||
except ImportError:
|
||||
from utils import exit_signal
|
||||
from utils.media_folders import is_skipped_folder
|
||||
|
||||
try:
|
||||
from ...database.image_removal import mark_hides
|
||||
from ...database.pair_ingestion import SCAN_MARKS, ingest_resolved_pair
|
||||
from ...database.run_attribution import (
|
||||
RecordedOutput,
|
||||
@@ -63,7 +60,6 @@ try:
|
||||
decide_file_prompt,
|
||||
)
|
||||
except ImportError:
|
||||
from database.image_removal import mark_hides
|
||||
from database.pair_ingestion import SCAN_MARKS, ingest_resolved_pair
|
||||
from database.run_attribution import (
|
||||
RecordedOutput,
|
||||
@@ -122,11 +118,10 @@ DUPLICATE_FIND_LEGACY_TITLE = "Find duplicate image files (gallery page)"
|
||||
def _collect_media_files(output_dirs, extensions):
|
||||
"""Media files under ``output_dirs`` (blocking), bounded and symlink-free.
|
||||
|
||||
Walks each root without following symlinks, skips ``thumbnails`` and
|
||||
trash folders (``utils.media_folders``) and symlinked files, never
|
||||
descends more than MAX_SCAN_DEPTH levels and stops after MAX_SCAN_FILES
|
||||
files in total. Extensions match case-insensitively and each path is
|
||||
listed once.
|
||||
Walks each root without following symlinks, skips ``thumbnails``
|
||||
directories and symlinked files, never descends more than MAX_SCAN_DEPTH
|
||||
levels and stops after MAX_SCAN_FILES files in total. Extensions match
|
||||
case-insensitively and each path is listed once.
|
||||
"""
|
||||
wanted = {ext.lower() for ext in extensions}
|
||||
found = []
|
||||
@@ -141,7 +136,7 @@ def _collect_media_files(output_dirs, extensions):
|
||||
dirnames[:] = sorted(
|
||||
d
|
||||
for d in dirnames
|
||||
if not is_skipped_folder(d)
|
||||
if d != "thumbnails"
|
||||
and depth < MAX_SCAN_DEPTH
|
||||
and not os.path.islink(os.path.join(current, d))
|
||||
)
|
||||
@@ -888,8 +883,6 @@ class AdminRoutesMixin:
|
||||
extensions = IMAGE_EXTENSIONS + VIDEO_EXTENSIONS
|
||||
media_files = []
|
||||
seen = set()
|
||||
# #480: a file removed from the library is never a copy or a keeper.
|
||||
marks = self._removal_marks()
|
||||
for slot, root in enumerate(roots):
|
||||
for path in _collect_media_files([root], extensions):
|
||||
try:
|
||||
@@ -898,9 +891,6 @@ class AdminRoutesMixin:
|
||||
stat = os.stat(path)
|
||||
except OSError:
|
||||
continue
|
||||
locator = os.path.normcase(os.path.realpath(str(path)))
|
||||
if locator in marks and mark_hides(marks, locator, path):
|
||||
continue
|
||||
key = _physical_file_key(path, stat)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
@@ -1561,13 +1551,8 @@ class AdminRoutesMixin:
|
||||
if job.cancelled:
|
||||
return
|
||||
batch = media_files[batch_start : batch_start + SCAN_BATCH_SIZE]
|
||||
# #480: a file removed from the library is not read or linked
|
||||
# again while its bytes are unchanged.
|
||||
removed = await self._run_in_executor(self._removed_paths, batch)
|
||||
skipped = [f for f in batch if str(f) in known or str(f) in removed]
|
||||
to_read = [
|
||||
f for f in batch if str(f) not in known and str(f) not in removed
|
||||
]
|
||||
skipped = [f for f in batch if str(f) in known]
|
||||
to_read = [f for f in batch if str(f) not in known]
|
||||
outcomes = [_scan_outcome(f, roots, "skipped") for f in skipped]
|
||||
counts["processed"] += len(skipped)
|
||||
counts["unchanged"] += len(skipped)
|
||||
|
||||
@@ -23,11 +23,6 @@ from .prompts import thumbnail_url_for
|
||||
from . import thumbnail_settings
|
||||
from .. import model_gate
|
||||
|
||||
try:
|
||||
from ...utils.media_folders import is_skipped_folder
|
||||
except ImportError:
|
||||
from utils.media_folders import is_skipped_folder
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
_INTERNAL_ERROR = "An internal error occurred. Check server logs for details."
|
||||
@@ -1324,9 +1319,8 @@ class AutotagRoutesMixin:
|
||||
def _collect_scan_files(output_path: Path):
|
||||
"""Media files under ``output_path`` from a bounded, symlink-free walk.
|
||||
|
||||
Descends at most ``_SCAN_MAX_DEPTH`` levels, skips ``thumbnails`` and
|
||||
trash folders (``utils.media_folders``) and symlinked files, and stops
|
||||
after ``_SCAN_MAX_FILES`` files.
|
||||
Descends at most ``_SCAN_MAX_DEPTH`` levels, skips ``thumbnails``
|
||||
trees and symlinked files, and stops after ``_SCAN_MAX_FILES`` files.
|
||||
Returns ``(paths, truncated)``.
|
||||
"""
|
||||
found = []
|
||||
@@ -1336,7 +1330,7 @@ class AutotagRoutesMixin:
|
||||
dirnames[:] = sorted(
|
||||
name
|
||||
for name in dirnames
|
||||
if not is_skipped_folder(name) and depth + 1 <= _SCAN_MAX_DEPTH
|
||||
if name != "thumbnails" and depth + 1 <= _SCAN_MAX_DEPTH
|
||||
)
|
||||
for name in sorted(filenames):
|
||||
if Path(name).suffix.lower() not in _MEDIA_EXTENSIONS:
|
||||
@@ -1363,10 +1357,6 @@ class AutotagRoutesMixin:
|
||||
files, truncated = await self._run_in_executor(
|
||||
self._collect_scan_files, output_path
|
||||
)
|
||||
# #501: a file removed from the library is not offered for tagging.
|
||||
files = await self._run_in_executor(
|
||||
self._without_removed, output_path, files
|
||||
)
|
||||
has_thumbnails = (output_path / "thumbnails").is_dir()
|
||||
|
||||
images = []
|
||||
|
||||
@@ -213,8 +213,6 @@ def public_config():
|
||||
"max_image_age_days": GalleryConfig.MAX_IMAGE_AGE_DAYS,
|
||||
"max_concurrent_processing": GalleryConfig.MAX_CONCURRENT_PROCESSING,
|
||||
"metadata_extraction_timeout": GalleryConfig.METADATA_EXTRACTION_TIMEOUT,
|
||||
# config.json gallery.monitoring.save_foreign_runs (#247)
|
||||
"save_foreign_runs": GalleryConfig.SAVE_FOREIGN_RUNS,
|
||||
},
|
||||
"integrations": {
|
||||
"lora_manager_enabled": bool(IntegrationConfig.LORA_MANAGER_ENABLED),
|
||||
|
||||
+1
-13
@@ -27,11 +27,7 @@ from .history_current import (
|
||||
resolve_current_candidate,
|
||||
snapshot_current_output_scope,
|
||||
)
|
||||
from .inspection_service import (
|
||||
CurrentFileChanged,
|
||||
image_tags_reader,
|
||||
inspect_current_file,
|
||||
)
|
||||
from .inspection_service import CurrentFileChanged, inspect_current_file
|
||||
from .images import image_category_key
|
||||
|
||||
try:
|
||||
@@ -782,14 +778,6 @@ class HistoryRoutesMixin:
|
||||
),
|
||||
own,
|
||||
),
|
||||
# The file's own image tags, keyed as the Tag dialog
|
||||
# keys them: the normcased realpath (#457).
|
||||
image_tags=image_tags_reader(
|
||||
db,
|
||||
str(resolved.canonical_target),
|
||||
self.logger,
|
||||
close=True,
|
||||
),
|
||||
recorded=recorded_output_lookup(
|
||||
db, close=True, reference=receipt["reference"]
|
||||
),
|
||||
|
||||
@@ -1,521 +0,0 @@
|
||||
"""Delete selected images: remove them from the library, or trash them (#480).
|
||||
|
||||
``POST /prompt_manager/images/delete/preview {items | filter}`` reviews the
|
||||
selection for both modes at once and answers a reviewed plan (the System ›
|
||||
Maintenance contract, ``py/api/maintenance.py``): the token, the counts of
|
||||
the records each file loses, the refused files with a coded reason (both
|
||||
modes, and the trash-only ones), the prompts that lose their last image and
|
||||
the safety backup. It writes nothing. ``items`` are Images listing identities
|
||||
(at most 500); ``filter`` is "Select all N matching". Each target is matched
|
||||
to its entry in the current listing (``not-listed`` otherwise), so every
|
||||
later step uses the walked spelling, never the request's text.
|
||||
|
||||
``POST /prompt_manager/maintenance/apply {token, mode}`` runs the plan as the
|
||||
``image_delete`` job (admission ``records``, not cancellable): the sources
|
||||
are checked, the promised safety backup is taken, then each file is checked
|
||||
again (containment and fingerprint) and
|
||||
|
||||
- ``mode="library"``: its records are deleted and its removal mark written
|
||||
in one transaction; the file stays on disk, hidden;
|
||||
- ``mode="trash"``: it is moved to the trash first (``utils/trash.py``); a
|
||||
refused move leaves the file and its records alone; after a move its
|
||||
records are deleted.
|
||||
|
||||
No path here unlinks a media file. A failed records write stops the job; the
|
||||
files after it are not attempted.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat as stat_module
|
||||
from pathlib import Path
|
||||
|
||||
from .admin import DELETABLE_MEDIA_EXTENSIONS, _remove_thumbnail
|
||||
from .duplicate_review import (
|
||||
CHANGED_CONTENT,
|
||||
GONE,
|
||||
LINK,
|
||||
OUTSIDE_SOURCE,
|
||||
UNREADABLE,
|
||||
file_problem,
|
||||
)
|
||||
from .images import _favorite_relative_key, _favorite_root_key
|
||||
from .jobs import IMAGE_DELETE_GROUP_NAMES, IMAGE_DELETE_JOB_KIND, JobBusy
|
||||
from .job_routes import busy_response
|
||||
from .maintenance import MAINTENANCE_PREVIEW_LEASE, _fail_job, _failure, backup_note
|
||||
|
||||
try:
|
||||
from ...database import image_removal_schema
|
||||
from ...database.image_removal import mark_hides
|
||||
from ...database.annotation_schema import MAX_LOCATOR_LENGTH
|
||||
from ...utils import trash
|
||||
from ...utils.media_folders import is_trash_folder
|
||||
except ImportError:
|
||||
from database import image_removal_schema
|
||||
from database.image_removal import mark_hides
|
||||
from database.annotation_schema import MAX_LOCATOR_LENGTH
|
||||
from utils import trash
|
||||
from utils.media_folders import is_trash_folder
|
||||
|
||||
IMAGE_DELETE = "delete_images"
|
||||
MODES = ("library", "trash")
|
||||
TITLES = {
|
||||
"library": "Delete images (remove from the library)",
|
||||
"trash": "Delete images (move to the trash)",
|
||||
}
|
||||
MAX_DETAIL_ROWS = 50
|
||||
# The system moved the file but its place in a trash could not be shown (#499).
|
||||
TRASH_UNCONFIRMED = "trash-unconfirmed"
|
||||
RECORDS_FAILED_CODE = "records-failed"
|
||||
|
||||
# Reason codes (design §4.4) and the sentence each shows.
|
||||
REASONS = {
|
||||
"not-listed": "No longer listed in Images.",
|
||||
"outside": "Outside the output folders.",
|
||||
"link": "A link, not a file.",
|
||||
"gone": "The file no longer exists.",
|
||||
"unreadable": "The file can't be read.",
|
||||
"thumbnail": "A generated preview, not an image.",
|
||||
"not-media": "Not a media file.",
|
||||
"trash-folder": "Inside a trash folder.",
|
||||
"path-too-long": "The path is too long.",
|
||||
"changed": "The file changed after the review.",
|
||||
trash.NO_TRASH: "This drive has no trash.",
|
||||
trash.NETWORK: "Network drives can't use the trash.",
|
||||
trash.TRASH_IN_ROOT: "This drive's trash is inside an output folder.",
|
||||
trash.LOCKED: "The file is open in another program. Close it and try again.",
|
||||
trash.FAILED: "The file could not be moved to the trash.",
|
||||
trash.NO_PERMISSION: (
|
||||
"PromptManager isn't allowed to move this file (no permission, or it is "
|
||||
"locked in Finder)."
|
||||
),
|
||||
trash.CLOUD_SYNC: (
|
||||
"Files in iCloud Drive or a cloud-synced folder can't be moved to the "
|
||||
"trash here."
|
||||
),
|
||||
TRASH_UNCONFIRMED: (
|
||||
"Moved by macOS, but its place in the trash could not be confirmed."
|
||||
),
|
||||
trash.WINDOWS_RECYCLE: (
|
||||
"Moving files to the Recycle Bin isn't available on Windows: files larger "
|
||||
"than the bin's capacity would be deleted permanently."
|
||||
),
|
||||
"not-attempted": "Not attempted: the job stopped after an error.",
|
||||
}
|
||||
_PROBLEM_CODES = {
|
||||
OUTSIDE_SOURCE: "outside",
|
||||
LINK: "link",
|
||||
GONE: "gone",
|
||||
UNREADABLE: "unreadable",
|
||||
CHANGED_CONTENT: "changed",
|
||||
}
|
||||
SOURCES_CHANGED = "The sources changed since the review; nothing was changed."
|
||||
RECORDS_FAILED = "The image records could not be removed; the job stopped."
|
||||
|
||||
|
||||
def lexical_problem(relative: str) -> str | None:
|
||||
"""The refusal code a root-relative POSIX path earns before any file access."""
|
||||
parts = relative.split("/")
|
||||
if "thumbnails" in parts[:-1]:
|
||||
return "thumbnail"
|
||||
if any(is_trash_folder(part) for part in parts[:-1]):
|
||||
return "trash-folder"
|
||||
if os.path.splitext(parts[-1])[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||||
return "not-media"
|
||||
return None
|
||||
|
||||
|
||||
def file_check(path, root, relative, *, expected=None):
|
||||
"""``(code, lstat)``: why *path* may not be deleted, or ``(None, lstat)``.
|
||||
|
||||
Text first (inside the root, not a preview or trash folder, a media
|
||||
file), then ``duplicate_review.file_problem``: a regular file, not a
|
||||
link, whose resolved path is still inside the resolved root. With
|
||||
*expected* (``{size, mtime_ns, inode}`` from the review) the file must
|
||||
still be those bytes. Every handle is closed when this returns.
|
||||
"""
|
||||
code = lexical_problem(relative)
|
||||
if code:
|
||||
return code, None
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
return "gone", None
|
||||
except OSError:
|
||||
return "unreadable", None
|
||||
if getattr(info, "st_file_attributes", 0) & getattr(
|
||||
stat_module, "FILE_ATTRIBUTE_REPARSE_POINT", 0
|
||||
):
|
||||
return "link", info
|
||||
if expected is None:
|
||||
problem = file_problem(path, root, None, info.st_size, hash_check=False)
|
||||
else:
|
||||
problem = file_problem(
|
||||
path,
|
||||
root,
|
||||
None,
|
||||
expected["size"],
|
||||
hash_check=False,
|
||||
mtime_ns=expected["mtime_ns"],
|
||||
inode=expected["inode"],
|
||||
)
|
||||
if problem:
|
||||
return _PROBLEM_CODES.get(problem, "outside"), info
|
||||
if len(os.path.normcase(os.path.realpath(path))) > MAX_LOCATOR_LENGTH:
|
||||
return "path-too-long", info
|
||||
return None, info
|
||||
|
||||
|
||||
def _row(target, code):
|
||||
return {
|
||||
"path": target["rel"],
|
||||
"root_label": target["root_label"],
|
||||
"code": code,
|
||||
"detail": REASONS[code],
|
||||
}
|
||||
|
||||
|
||||
def _summary(rows):
|
||||
counts = {}
|
||||
for row in rows:
|
||||
counts[row["code"]] = counts.get(row["code"], 0) + 1
|
||||
return {"count": len(rows), "counts": counts, "rows": rows[:MAX_DETAIL_ROWS]}
|
||||
|
||||
|
||||
def _fingerprint(targets) -> str:
|
||||
material = [
|
||||
(t["root_index"], t["rel"], t["size"], t["mtime_ns"], t["inode"])
|
||||
for t in targets
|
||||
]
|
||||
return hashlib.sha256(json.dumps(material).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class ImageDeleteMixin:
|
||||
"""The Images Delete… review and its apply job (#480)."""
|
||||
|
||||
def _register_image_delete_routes(self, routes):
|
||||
# A fixed path beside the dynamic /images/{image_id} routes: aiohttp
|
||||
# (3.10+) looks fixed paths up first (see image_categories.py).
|
||||
@routes.post("/prompt_manager/images/delete/preview")
|
||||
async def preview_image_delete_route(request):
|
||||
return await self.preview_image_delete(request)
|
||||
|
||||
# ── review ────────────────────────────────────────────────────────
|
||||
|
||||
async def preview_image_delete(self, request):
|
||||
"""Review the selection under the maintenance preview lease."""
|
||||
try:
|
||||
async with self._jobs.lease(MAINTENANCE_PREVIEW_LEASE):
|
||||
return await self._preview_image_delete(request)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def _preview_image_delete(self, request):
|
||||
_data, targets, roots, error = await self._category_request(
|
||||
request, frozenset()
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
db = self.db
|
||||
identity = self._database_identity(db)
|
||||
try:
|
||||
files = {}
|
||||
for index in dict.fromkeys(t[0] for t in targets):
|
||||
files[index] = await self._get_gallery_files(roots[index])
|
||||
review = await self._run_in_executor(
|
||||
self._review_image_delete, db, roots, targets, files
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Image delete review failed")
|
||||
return _failure("The review failed. Check server logs for details.", 500)
|
||||
return await self._plan_response(
|
||||
db,
|
||||
identity,
|
||||
IMAGE_DELETE,
|
||||
_fingerprint(review["targets"]),
|
||||
review["public"],
|
||||
{"targets": review["targets"]},
|
||||
)
|
||||
|
||||
def _review_image_delete(self, db, roots, targets, files):
|
||||
"""Every target checked now (blocking); ``{targets, public}``."""
|
||||
listings, seen = {}, set()
|
||||
acted, refused, trash_refused = [], [], []
|
||||
spellings = self._source_spellings()
|
||||
root_texts = [str(root) for root in roots]
|
||||
for index, _scope, relative in targets:
|
||||
root = roots[index]
|
||||
if index not in listings:
|
||||
listings[index] = self._favorite_listing(root, files[index])
|
||||
spelled = listings[index].get(_favorite_relative_key(relative))
|
||||
target = {
|
||||
"root_index": index,
|
||||
"root_scope": _scope,
|
||||
"root": str(root),
|
||||
"root_label": self._public_path(root),
|
||||
"rel": spelled or relative,
|
||||
}
|
||||
if spelled is None:
|
||||
refused.append(_row(target, "not-listed"))
|
||||
continue
|
||||
absolute = os.path.join(str(root), *spelled.split("/"))
|
||||
code, info = file_check(absolute, str(root), spelled)
|
||||
if code:
|
||||
refused.append(_row(target, code))
|
||||
continue
|
||||
identity = (info.st_dev, info.st_ino)
|
||||
if identity in seen:
|
||||
continue # one physical file named twice (nested roots, hardlinks)
|
||||
seen.add(identity)
|
||||
target.update(self._removal_of(absolute, info, spellings))
|
||||
target["trash"] = trash.trash_available(absolute, root_texts)
|
||||
if target["trash"]:
|
||||
trash_refused.append(_row(target, target["trash"]))
|
||||
acted.append(target)
|
||||
counted = db.count_removal_records([t["removal"] for t in acted])
|
||||
library_count = len(acted)
|
||||
public = {
|
||||
"operation": IMAGE_DELETE,
|
||||
"scope": {"count": library_count, "label": "images"},
|
||||
"library": {"count": library_count},
|
||||
"trash": {"count": library_count - len(trash_refused)},
|
||||
"protected": {"count": len(refused), "reasons": []},
|
||||
"refused": _summary(refused),
|
||||
"trash_refused": _summary(trash_refused),
|
||||
"records": counted["records"],
|
||||
"orphaned_prompts": counted["orphaned_prompts"],
|
||||
"media": {"deletes_files": False, "label": "Chosen in the review"},
|
||||
"details": [],
|
||||
}
|
||||
return {"targets": acted, "public": public}
|
||||
|
||||
@staticmethod
|
||||
def _removal_of(absolute, info, spellings):
|
||||
favorites = [
|
||||
(_favorite_root_key(root), _favorite_relative_key(relative))
|
||||
for root, relative in spellings.favorites(absolute).items()
|
||||
]
|
||||
return {
|
||||
"abs": absolute,
|
||||
"size": info.st_size,
|
||||
"mtime_ns": info.st_mtime_ns,
|
||||
"inode": [info.st_dev, info.st_ino],
|
||||
"removal": {
|
||||
"spellings": list(spellings.of(absolute).values()),
|
||||
"locator": os.path.normcase(os.path.realpath(absolute)),
|
||||
"favorites": favorites,
|
||||
"category": spellings.category_key(absolute),
|
||||
},
|
||||
}
|
||||
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
def _start_image_delete(self, db, plan, mode, backup_on):
|
||||
"""Register the delete job for a reviewed plan (on the loop)."""
|
||||
return self._jobs.start(
|
||||
IMAGE_DELETE_JOB_KIND,
|
||||
TITLES[mode],
|
||||
admission="records",
|
||||
cancellable=False,
|
||||
scope={"operation": IMAGE_DELETE, "mode": mode},
|
||||
groups=IMAGE_DELETE_GROUP_NAMES,
|
||||
runner=lambda job: self._run_image_delete(job, db, plan, mode, backup_on),
|
||||
)
|
||||
|
||||
async def _run_image_delete(self, job, db, plan, mode, backup_on):
|
||||
targets = plan.payload["targets"]
|
||||
unchanged = await self._run_in_executor(self._sources_unchanged, targets)
|
||||
if not unchanged:
|
||||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return
|
||||
try:
|
||||
await self._delete_images(job, db, targets, mode, backup)
|
||||
finally:
|
||||
if backup:
|
||||
await self._prune_backup_pool(job, db)
|
||||
|
||||
async def _delete_images(self, job, db, targets, mode, backup):
|
||||
job.update(total=len(targets), status_line=TITLES[mode])
|
||||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||||
done_group = "trashed" if mode == "trash" else "removed"
|
||||
changed, failed = 0, None
|
||||
for done, target in enumerate(targets, start=1):
|
||||
if failed:
|
||||
self._record_delete(job, "not_changed", target, "not-attempted")
|
||||
continue
|
||||
group, code, label = await self._run_in_executor(
|
||||
self._delete_one_image, db, target, mode, roots
|
||||
)
|
||||
if group == "errors":
|
||||
failed = code or RECORDS_FAILED_CODE
|
||||
text = REASONS[code] if code else RECORDS_FAILED
|
||||
job.record(
|
||||
"errors",
|
||||
{
|
||||
"path": target["rel"],
|
||||
"root": target["root_label"],
|
||||
"root_index": target["root_index"],
|
||||
"root_scope": target["root_scope"],
|
||||
"detail": text,
|
||||
},
|
||||
f"! {target['root_label']}/{target['rel']} · {text}",
|
||||
)
|
||||
elif group == done_group:
|
||||
changed += 1
|
||||
detail = (
|
||||
f"Moved to the {label}" if label else "Removed from the library"
|
||||
)
|
||||
self._record_delete(job, group, target, None, detail)
|
||||
else:
|
||||
self._record_delete(job, group, target, code)
|
||||
job.update(processed=done, progress=int(done / len(targets) * 100))
|
||||
if changed:
|
||||
self.invalidate_gallery_cache()
|
||||
verb = "Moved" if mode == "trash" else "Removed"
|
||||
where = "to the trash" if mode == "trash" else "from the library"
|
||||
line = f"{verb} {changed:,} of {len(targets):,} images {where}"
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
if failed:
|
||||
# "records-failed" names every stop at an errors row (the page's
|
||||
# "Stopped after N of M" toast); the row says which error it was.
|
||||
job.scope["failure"] = RECORDS_FAILED_CODE
|
||||
job.publish({"type": "error", "code": failed, "message": line})
|
||||
job.finish("failed")
|
||||
|
||||
@staticmethod
|
||||
def _record_delete(job, group, target, code, detail=None):
|
||||
text = detail or REASONS[code]
|
||||
row = {
|
||||
"path": target["rel"],
|
||||
"root": target["root_label"],
|
||||
"root_index": target["root_index"],
|
||||
"root_scope": target["root_scope"],
|
||||
"code": code,
|
||||
"detail": text,
|
||||
}
|
||||
mark = {"removed": "-", "trashed": "-"}.get(group, "=")
|
||||
job.record(
|
||||
group, row, f"{mark} {target['root_label']}/{target['rel']} · {text}"
|
||||
)
|
||||
|
||||
def _delete_one_image(self, db, target, mode, roots):
|
||||
"""``(group, code, trash label)`` for one reviewed file (blocking)."""
|
||||
code, _info = file_check(
|
||||
target["abs"],
|
||||
target["root"],
|
||||
target["rel"],
|
||||
expected={
|
||||
"size": target["size"],
|
||||
"mtime_ns": target["mtime_ns"],
|
||||
"inode": target["inode"],
|
||||
},
|
||||
)
|
||||
if code:
|
||||
return "not_changed", code, None
|
||||
label = None
|
||||
if mode == "trash":
|
||||
try:
|
||||
label = trash.move_to_trash(target["abs"], roots)
|
||||
except trash.TrashRefused as refused:
|
||||
code = refused.code if refused.code in REASONS else trash.FAILED
|
||||
self.logger.info(
|
||||
"Not moved to the trash (%s): %s %s",
|
||||
code,
|
||||
target["rel"],
|
||||
refused.detail,
|
||||
)
|
||||
return "not_changed", code, None
|
||||
except trash.TrashUnconfirmed as unconfirmed:
|
||||
# The file left its folder: keep its records, stop the job.
|
||||
self.logger.warning(
|
||||
"Trash move not confirmed: %s %s", target["rel"], unconfirmed.detail
|
||||
)
|
||||
return "errors", TRASH_UNCONFIRMED, None
|
||||
mark = None
|
||||
else:
|
||||
mark = (target["size"], target["mtime_ns"])
|
||||
try:
|
||||
db.remove_image_records(target["removal"], mark=mark)
|
||||
except Exception:
|
||||
self.logger.exception("Removing an image's records failed")
|
||||
return "errors", None, label
|
||||
_remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
|
||||
return ("trashed" if mode == "trash" else "removed"), None, label
|
||||
|
||||
# ── the workspace listing ─────────────────────────────────────────
|
||||
|
||||
def _removal_marks(self) -> dict:
|
||||
"""The removal marks, cached until a mark changes or the database does.
|
||||
|
||||
A listing page reads no table while nothing changed: the cache follows
|
||||
this process's marks generation and the database identity (a restore).
|
||||
"""
|
||||
key = (self._database_identity(self.db), image_removal_schema.generation())
|
||||
cached = getattr(self, "_marks_cache", None)
|
||||
if cached is None or cached[0] != key:
|
||||
cached = (key, self.db.removal_marks())
|
||||
self._marks_cache = cached
|
||||
return cached[1]
|
||||
|
||||
def _hidden_entries(self, listings) -> set:
|
||||
"""Normalized paths of listed files a removal mark hides (blocking).
|
||||
|
||||
Only marked locators are looked at, and only those files are
|
||||
``lstat``'ed: a mark hides a file while its size and ``mtime_ns`` are
|
||||
still the ones it was removed with.
|
||||
"""
|
||||
marks = self._removal_marks()
|
||||
if not marks:
|
||||
return set()
|
||||
located = self._workspace_locators(listings)
|
||||
return {
|
||||
entry.normalized
|
||||
for locator, entry in ((locator, located.get(locator)) for locator in marks)
|
||||
if entry is not None and mark_hides(marks, locator, entry.path)
|
||||
}
|
||||
|
||||
def _removed_paths(self, paths) -> set:
|
||||
"""The absolute paths among *paths* a removal mark hides (blocking)."""
|
||||
marks = self._removal_marks()
|
||||
if not marks:
|
||||
return set()
|
||||
return {
|
||||
str(path)
|
||||
for path in paths
|
||||
if mark_hides(marks, os.path.normcase(os.path.realpath(str(path))), path)
|
||||
}
|
||||
|
||||
def _removal_check_under(self, root):
|
||||
"""``hidden(path) -> bool`` for files a walk of *root* found (blocking).
|
||||
|
||||
A walk never follows a link below *root*, so a file's locator is
|
||||
``normcase(join(realpath(root), relative))``: no ``realpath`` per
|
||||
file, and an ``lstat`` only for a marked one. Without marks it
|
||||
answers False at once.
|
||||
"""
|
||||
marks = self._removal_marks()
|
||||
if not marks:
|
||||
return lambda _path: False
|
||||
root = os.fspath(root)
|
||||
real = os.path.realpath(root)
|
||||
|
||||
def hidden(path):
|
||||
relative = os.path.relpath(os.fspath(path), root)
|
||||
locator = os.path.normcase(os.path.join(real, relative))
|
||||
return mark_hides(marks, locator, path)
|
||||
|
||||
return hidden
|
||||
|
||||
def _without_removed(self, root, paths) -> list:
|
||||
"""*paths* found by a walk of *root*, less the hidden ones (blocking)."""
|
||||
removed = self._removal_check_under(root)
|
||||
return [path for path in paths if not removed(path)]
|
||||
|
||||
|
||||
def parse_mode(value):
|
||||
"""The apply's ``mode``, or None when it is not one of MODES."""
|
||||
return value if isinstance(value, str) and value in MODES else None
|
||||
@@ -488,11 +488,7 @@ class ImageTagJobsMixin:
|
||||
_still_targets, self, index, root, files, seen
|
||||
)
|
||||
)
|
||||
# #480: a file removed from the library is never tagged by a batch.
|
||||
hidden = await self._run_in_executor(
|
||||
self._removed_paths, [target.path for target in found]
|
||||
)
|
||||
return [target for target in found if target.path not in hidden]
|
||||
return found
|
||||
|
||||
async def _start_autotag(
|
||||
self, *, model_type, existing, options, keep_in_memory, title, owner_kind
|
||||
|
||||
+27
-19
@@ -55,7 +55,6 @@ from .inspection_service import (
|
||||
IMAGE_EXTENSIONS as _IMAGE_EXTENSIONS,
|
||||
SERVABLE_EXTENSIONS,
|
||||
VIDEO_EXTENSIONS,
|
||||
image_tags_reader,
|
||||
inspect_current_file,
|
||||
)
|
||||
|
||||
@@ -66,7 +65,6 @@ try:
|
||||
from ...database.models import normalize_image_path
|
||||
from ...database.run_attribution import recorded_output_lookup
|
||||
from ...utils.card_seed import card_seed, header_params, parsed_positive
|
||||
from ...utils.media_folders import in_skipped_folder, is_skipped_folder
|
||||
from ...utils.parallel import map_parallel
|
||||
from ...utils.json_strict import dumps_strict
|
||||
from ...utils.inspection_response import (
|
||||
@@ -92,7 +90,6 @@ except ImportError:
|
||||
from database.models import normalize_image_path
|
||||
from database.run_attribution import recorded_output_lookup
|
||||
from utils.card_seed import card_seed, header_params, parsed_positive
|
||||
from utils.media_folders import in_skipped_folder, is_skipped_folder
|
||||
from utils.parallel import map_parallel
|
||||
from utils.json_strict import dumps_strict
|
||||
from utils.inspection_response import (
|
||||
@@ -917,8 +914,7 @@ def _name_suffix(name):
|
||||
def _walk_media(output_path, max_files):
|
||||
"""Yield the ``os.DirEntry`` of each media file under *output_path*.
|
||||
|
||||
Symlinks are never followed, ``thumbnails/`` and trash folders are
|
||||
skipped (``utils.media_folders``, #501), descent stops
|
||||
Symlinks are never followed, ``thumbnails/`` is skipped, descent stops
|
||||
at MAX_SCAN_DEPTH and at most *max_files* entries are produced. The order
|
||||
is ``os.walk``'s (top-down, subfolders in ``os.scandir`` order, files by
|
||||
name), which the gallery's ties in mtime keep. Each entry's type comes
|
||||
@@ -930,6 +926,8 @@ def _walk_media(output_path, max_files):
|
||||
produced = 0
|
||||
while stack:
|
||||
top, rel_parts = stack.pop()
|
||||
if "thumbnails" in rel_parts:
|
||||
continue
|
||||
dirs, files = [], []
|
||||
try:
|
||||
with os.scandir(top) as entries:
|
||||
@@ -953,7 +951,7 @@ def _walk_media(output_path, max_files):
|
||||
if len(rel_parts) >= MAX_SCAN_DEPTH:
|
||||
continue
|
||||
for entry in reversed(dirs):
|
||||
if not entry.is_symlink() and not is_skipped_folder(entry.name):
|
||||
if not entry.is_symlink():
|
||||
stack.append((entry.path, rel_parts + (entry.name,)))
|
||||
|
||||
|
||||
@@ -977,7 +975,7 @@ def _settled_entry(root, relative):
|
||||
*folders, name = relative.split(os.sep)
|
||||
if _name_suffix(name).lower() not in MEDIA_EXTENSIONS:
|
||||
return None
|
||||
if in_skipped_folder(folders) or len(folders) > MAX_SCAN_DEPTH:
|
||||
if "thumbnails" in folders or len(folders) > MAX_SCAN_DEPTH:
|
||||
return None
|
||||
folder = os.fspath(root)
|
||||
for part in ("", *folders):
|
||||
@@ -2451,10 +2449,6 @@ class ImageRoutesMixin:
|
||||
def key(entry):
|
||||
return _workspace_key(entry, order)
|
||||
|
||||
# #480: files removed from the library stay on disk, hidden here.
|
||||
hidden = self._hidden_entries(listings)
|
||||
if hidden:
|
||||
entries = [entry for entry in entries if entry.normalized not in hidden]
|
||||
return (
|
||||
entries,
|
||||
key,
|
||||
@@ -3076,9 +3070,7 @@ class ImageRoutesMixin:
|
||||
no symlinks), each preview folder is listed once, and at most
|
||||
MAX_THUMBNAIL_FILES targets are queued per batch (a job plans again
|
||||
after each batch); ``remaining`` counts the targets beyond that cap.
|
||||
Sources in *exclude* (worked on by an earlier batch) are left out, and
|
||||
so are files removed from the library (#501: a removal mark hides
|
||||
them, and their previews went with their records).
|
||||
Sources in *exclude* (worked on by an earlier batch) are left out.
|
||||
``stop()`` true ends the walk before the next file.
|
||||
"""
|
||||
settings = writer_settings(settings)
|
||||
@@ -3087,14 +3079,11 @@ class ImageRoutesMixin:
|
||||
plan = {key: 0 for key in _PLAN_COUNTS}
|
||||
plan.update(targets=[], fresh_files=[])
|
||||
folders = {}
|
||||
removed = self._removal_check_under(output_path)
|
||||
for src in _iter_media_files(output_path, MAX_GALLERY_FILES):
|
||||
if stop is not None and stop():
|
||||
break
|
||||
if exclude and src in exclude:
|
||||
continue
|
||||
if removed(src):
|
||||
continue
|
||||
rel_no_ext = src.relative_to(output_path).with_suffix("")
|
||||
stem = f"{rel_no_ext.name}{thumbnail_settings.THUMB_SUFFIX}"
|
||||
dst = thumbnails_dir / rel_no_ext.parent / (stem + settings.extension)
|
||||
@@ -3973,9 +3962,28 @@ class ImageRoutesMixin:
|
||||
)
|
||||
|
||||
def _inspection_image_tags(self, root: Path, parts: tuple) -> Callable:
|
||||
"""The inspector's ``image_tags`` reader for one rooted file (S1)."""
|
||||
"""The inspector's ``image_tags`` reader for one rooted file (S1).
|
||||
|
||||
The returned function takes the inspection's own stable stat and
|
||||
answers ``current`` only for a byte record verified at exactly that
|
||||
``(size, mtime_ns)``; a failed read is ``unavailable``, never an error.
|
||||
"""
|
||||
locator = os.path.normcase(os.path.join(os.path.realpath(root), *parts))
|
||||
return image_tags_reader(self.db, locator, self.logger)
|
||||
|
||||
def read(size: int, mtime_ns: int) -> dict:
|
||||
try:
|
||||
found = self.db.current_image_tags({locator: (size, mtime_ns)})
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"Inspection image tags could not be read", exc_info=True
|
||||
)
|
||||
return {"state": "unavailable", "tags": []}
|
||||
tags = found.get(locator)
|
||||
if tags is None:
|
||||
return {"state": "none", "tags": []}
|
||||
return {"state": "current", "tags": list(tags)}
|
||||
|
||||
return read
|
||||
|
||||
def _read_own_category(self, sources, path):
|
||||
"""One file's own category, None, or OWN_CATEGORY_UNREAD (blocking).
|
||||
|
||||
@@ -93,35 +93,6 @@ def _in_host_temp(path: Path) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def image_tags_reader(
|
||||
db: Any, locator: str, logger: Any, *, close: bool = False
|
||||
) -> Callable[[int, int], dict]:
|
||||
"""The inspector's ``image_tags`` reader for one file *locator* (S1).
|
||||
|
||||
The returned function takes the inspection's own stable stat and answers
|
||||
``current`` only for a byte record verified at exactly that
|
||||
``(size, mtime_ns)``; a failed read is ``unavailable``, never an error.
|
||||
``close`` closes the calling thread's connection afterwards, for routes
|
||||
that close theirs after reading (History).
|
||||
"""
|
||||
|
||||
def read(size: int, mtime_ns: int) -> dict:
|
||||
try:
|
||||
found = db.current_image_tags({locator: (size, mtime_ns)})
|
||||
except Exception:
|
||||
logger.warning("Inspection image tags could not be read", exc_info=True)
|
||||
return {"state": "unavailable", "tags": []}
|
||||
finally:
|
||||
if close:
|
||||
db.model.close()
|
||||
tags = found.get(locator)
|
||||
if tags is None:
|
||||
return {"state": "none", "tags": []}
|
||||
return {"state": "current", "tags": list(tags)}
|
||||
|
||||
return read
|
||||
|
||||
|
||||
def inspect_current_file(
|
||||
path: Path,
|
||||
*,
|
||||
|
||||
@@ -123,11 +123,6 @@ MODEL_DOWNLOAD_JOB_KIND = "model_download"
|
||||
DUPLICATES_JOB_KIND = "duplicates"
|
||||
DUPLICATE_GROUP_NAMES = ("groups_found",)
|
||||
DUPLICATE_CLEANUP_GROUP_NAMES = ("removed", "kept", "errors")
|
||||
# The workspace Library's Delete… (#443, py/api/library_delete.py): a
|
||||
# ``records`` hold around its safety backup and write.
|
||||
PROMPT_DELETE_JOB_KIND = "prompt_delete"
|
||||
IMAGE_DELETE_JOB_KIND = "image_delete" # #480
|
||||
IMAGE_DELETE_GROUP_NAMES = ("trashed", "removed", "not_changed", "errors")
|
||||
# The System health check (py/api/system.py): a read-only ``read`` job with
|
||||
# no file outcomes, so it may run beside a scan.
|
||||
HEALTH_JOB_KIND = "health"
|
||||
|
||||
@@ -157,13 +157,6 @@ def _archive_scope(request: web.Request) -> str:
|
||||
return value
|
||||
|
||||
|
||||
def _shelf(request: web.Request) -> str:
|
||||
value = request.query.get("shelf", "active")
|
||||
if value not in ("active", "archived"):
|
||||
raise ValueError("shelf must be active or archived")
|
||||
return value
|
||||
|
||||
|
||||
def _missing() -> web.Response:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Positive prompt not found"}, status=404
|
||||
@@ -339,9 +332,6 @@ class LibraryRoutesMixin:
|
||||
"tags": sorted(set(request.query.getall("tag", []))),
|
||||
"tag_mode": request.query.get("tag_mode", "all"),
|
||||
"sort": request.query.get("sort", "newest"),
|
||||
# #443: which shelf; part of the cursor scope, so a cursor never
|
||||
# crosses shelves. Not the variant ``archive`` scope of /{id}.
|
||||
"shelf": _shelf(request),
|
||||
}
|
||||
limit = _page_limit(request)
|
||||
cursor = request.query.get("cursor")
|
||||
|
||||
@@ -1,263 +0,0 @@
|
||||
"""Delete or archive selected Library prompts, and restore archived ones (#443).
|
||||
|
||||
The workspace Prompt library's Delete… (D152). The legacy ``POST
|
||||
/prompt_manager/bulk/delete`` and ``DELETE /prompt_manager/delete/{id}`` are
|
||||
separate and unchanged.
|
||||
|
||||
- ``POST /prompt_manager/library/delete/preview {prompt_ids, expected_epoch}``
|
||||
classifies the selection (reads only, under a lease): the ids to ``delete``
|
||||
(in no saved pair), to ``archive`` (paired), the ``protected`` ones (tagged
|
||||
``__protected__``, kept), counts of ``already_archived`` and ``not_found``,
|
||||
the ``image_links`` a delete removes, the safety ``backup`` policy and,
|
||||
when a backup would be taken, the ``disk`` it needs.
|
||||
- ``POST /prompt_manager/library/delete {archive_ids, delete_ids,
|
||||
expected_epoch, backup}`` executes that review under a ``records`` hold
|
||||
(refused 409 ``busy`` beside a scan, restore, maintenance, move or
|
||||
Optimize). When a delete remains and a backup was promised (the reviewed
|
||||
``backup`` was "on" or the policy is on now) it checks the disk (507
|
||||
``insufficient-space``) and takes a verified safety backup (500
|
||||
``backup-failed``); then one transaction deletes the reviewed deletes whose
|
||||
fingerprint is unchanged since the backup and archives the paired ones.
|
||||
- ``POST /prompt_manager/library/restore {prompt_ids, expected_epoch}``
|
||||
removes the archive mark (under a lease).
|
||||
|
||||
A changed storage epoch answers 409 ``storage-changed``; nothing is written.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
try:
|
||||
from ...database.library_commands import LibraryCommandError
|
||||
from ...database.safety_backup import SafetyBackupFailed
|
||||
except ImportError:
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.safety_backup import SafetyBackupFailed
|
||||
|
||||
from . import maintenance as _maintenance
|
||||
from .admin import _prune_safety_backups
|
||||
from .job_routes import busy_response
|
||||
from .jobs import PROMPT_DELETE_JOB_KIND, JobBusy
|
||||
from .prompts import parse_bulk_prompt_ids
|
||||
|
||||
PREVIEW_URL = "/prompt_manager/library/delete/preview"
|
||||
DELETE_URL = "/prompt_manager/library/delete"
|
||||
RESTORE_URL = "/prompt_manager/library/restore"
|
||||
DELETE_TITLE = "Delete prompts (Library)"
|
||||
_SELECTION_KEYS = frozenset({"prompt_ids", "expected_epoch"})
|
||||
_DELETE_KEYS = frozenset({"archive_ids", "delete_ids", "expected_epoch", "backup"})
|
||||
STORAGE_CHANGED = "Prompt storage was restored; refresh the library"
|
||||
|
||||
|
||||
def _failure(status, code, error, **extra):
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": error, **extra}, status=status
|
||||
)
|
||||
|
||||
|
||||
def _invalid(message):
|
||||
return _failure(400, "invalid-request", message)
|
||||
|
||||
|
||||
def _storage_changed(exc):
|
||||
return _failure(409, exc.code, STORAGE_CHANGED, library_epoch=exc.epoch)
|
||||
|
||||
|
||||
async def _body(request, keys):
|
||||
"""The JSON object with exactly ``keys``, or raise ValueError."""
|
||||
try:
|
||||
data = await request.json()
|
||||
except ValueError:
|
||||
raise ValueError("Expected a JSON object") from None
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("Expected a JSON object")
|
||||
if set(data) != keys:
|
||||
raise ValueError("Request must contain exactly " + ", ".join(sorted(keys)))
|
||||
return data
|
||||
|
||||
|
||||
def _reviewed_ids(value, name):
|
||||
"""A reviewed id list: possibly empty, else as ``parse_bulk_prompt_ids``."""
|
||||
if value == []:
|
||||
return []
|
||||
try:
|
||||
return parse_bulk_prompt_ids(value)
|
||||
except ValueError as exc:
|
||||
raise ValueError(str(exc).replace("prompt_ids", name)) from None
|
||||
|
||||
|
||||
class LibraryDeleteRoutesMixin:
|
||||
"""Mixin providing the workspace Library's delete, archive and restore."""
|
||||
|
||||
def _register_library_delete_routes(self, routes):
|
||||
routes.post(PREVIEW_URL)(self.preview_library_delete)
|
||||
routes.post(DELETE_URL)(self.delete_library_prompts)
|
||||
routes.post(RESTORE_URL)(self.restore_library_prompts)
|
||||
|
||||
async def preview_library_delete(self, request):
|
||||
try:
|
||||
data = await _body(request, _SELECTION_KEYS)
|
||||
ids = parse_bulk_prompt_ids(data["prompt_ids"])
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
db = self.db
|
||||
policy = self._safety_backup_policy()
|
||||
try:
|
||||
async with self._jobs.lease("Library delete review"):
|
||||
review = await self._run_in_executor(
|
||||
db.classify_library_delete,
|
||||
ids,
|
||||
expected_epoch=data["expected_epoch"],
|
||||
)
|
||||
disk = None
|
||||
if review["delete"] and policy == "on":
|
||||
public, short, _size = await self._run_in_executor(
|
||||
_maintenance.disk_plan,
|
||||
db.model.db_path,
|
||||
backup=True,
|
||||
vacuum=False,
|
||||
)
|
||||
disk = {
|
||||
**public,
|
||||
"message": (
|
||||
None
|
||||
if short is None
|
||||
else _maintenance.short_space_message(short)
|
||||
),
|
||||
}
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
except LibraryCommandError as exc:
|
||||
return _storage_changed(exc)
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Library delete review failed")
|
||||
return _failure(500, "review-failed", "The delete could not be reviewed")
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"library_epoch": review["library_epoch"],
|
||||
"delete": review["delete"],
|
||||
"archive": review["archive"],
|
||||
"protected": review["protected"],
|
||||
"already_archived": len(review["already_archived"]),
|
||||
"not_found": len(review["not_found"]),
|
||||
"image_links": review["image_links"],
|
||||
"backup": policy,
|
||||
"disk": disk,
|
||||
}
|
||||
)
|
||||
|
||||
async def delete_library_prompts(self, request):
|
||||
try:
|
||||
data = await _body(request, _DELETE_KEYS)
|
||||
archive_ids = _reviewed_ids(data["archive_ids"], "archive_ids")
|
||||
delete_ids = _reviewed_ids(data["delete_ids"], "delete_ids")
|
||||
if not archive_ids and not delete_ids:
|
||||
raise ValueError("Nothing to delete or archive")
|
||||
if len(archive_ids) + len(delete_ids) > 1000:
|
||||
raise ValueError("At most 1000 prompts can be changed at once")
|
||||
if set(archive_ids) & set(delete_ids):
|
||||
raise ValueError("A prompt cannot be both archived and deleted")
|
||||
if data["backup"] not in ("on", "off"):
|
||||
raise ValueError("backup must be on or off")
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
busy = self._jobs.blocker(PROMPT_DELETE_JOB_KIND, admission="records")
|
||||
if busy is not None:
|
||||
return busy_response(JobBusy(busy))
|
||||
try:
|
||||
async with self._jobs.hold(
|
||||
PROMPT_DELETE_JOB_KIND, DELETE_TITLE, admission="records"
|
||||
) as job:
|
||||
response = await self._held_library_delete(
|
||||
self.db, data, archive_ids, delete_ids
|
||||
)
|
||||
if response.status >= 400:
|
||||
job.finish("failed")
|
||||
return response
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def _held_library_delete(self, db, data, archive_ids, delete_ids):
|
||||
epoch = data["expected_epoch"]
|
||||
promised = data["backup"] == "on" or self._safety_backup_policy() == "on"
|
||||
backup = None
|
||||
try:
|
||||
candidates = None
|
||||
if delete_ids:
|
||||
review = await self._run_in_executor(
|
||||
db.classify_library_delete, delete_ids, expected_epoch=epoch
|
||||
)
|
||||
candidates = review["fingerprints"]
|
||||
if candidates and promised:
|
||||
_disk, short, _size = await self._run_in_executor(
|
||||
_maintenance.disk_plan,
|
||||
db.model.db_path,
|
||||
backup=True,
|
||||
vacuum=False,
|
||||
)
|
||||
if short is not None:
|
||||
return _failure(
|
||||
507,
|
||||
"insufficient-space",
|
||||
_maintenance.short_space_message(short),
|
||||
)
|
||||
backup = await self._run_in_executor(
|
||||
_maintenance.take_safety_backup, db.model
|
||||
)
|
||||
result = await self._run_in_executor(
|
||||
db.archive_or_delete_library_prompts,
|
||||
archive_ids,
|
||||
delete_ids,
|
||||
expected_epoch=epoch,
|
||||
candidates=candidates if candidates is not None else {},
|
||||
)
|
||||
except LibraryCommandError as exc:
|
||||
return _storage_changed(exc)
|
||||
except SafetyBackupFailed as error:
|
||||
self.logger.error("Safety backup before a Library delete failed")
|
||||
return _failure(
|
||||
500, "backup-failed", _maintenance.backup_failed_message(error)
|
||||
)
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Library delete failed")
|
||||
return _failure(500, "delete-failed", "The prompts could not be changed")
|
||||
finally:
|
||||
if backup is not None:
|
||||
await self._prune_after_delete(db)
|
||||
return web.json_response({"success": True, **result, "backup": backup})
|
||||
|
||||
async def _prune_after_delete(self, db):
|
||||
try:
|
||||
await self._run_in_executor(_prune_safety_backups, db.model.db_path)
|
||||
except Exception: # pruning old copies never fails the delete
|
||||
self.logger.exception("Could not prune old safety backups")
|
||||
|
||||
async def restore_library_prompts(self, request):
|
||||
try:
|
||||
data = await _body(request, _SELECTION_KEYS)
|
||||
ids = parse_bulk_prompt_ids(data["prompt_ids"])
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
try:
|
||||
async with self._jobs.lease("Library restore"):
|
||||
result = await self._run_in_executor(
|
||||
self.db.restore_library_prompts,
|
||||
ids,
|
||||
expected_epoch=data["expected_epoch"],
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
except LibraryCommandError as exc:
|
||||
return _storage_changed(exc)
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Library restore failed")
|
||||
return _failure(500, "restore-failed", "The prompts could not be restored")
|
||||
return web.json_response({"success": True, **result})
|
||||
+4
-21
@@ -281,7 +281,7 @@ def short_space_message(short):
|
||||
)
|
||||
|
||||
|
||||
def _changed_line(operation, count, marks=0):
|
||||
def _changed_line(operation, count):
|
||||
if operation == "optimize":
|
||||
return "Optimized the database"
|
||||
if operation == "duplicate_prompts":
|
||||
@@ -291,11 +291,7 @@ def _changed_line(operation, count, marks=0):
|
||||
noun = "unused prompt component" if count == 1 else "unused prompt components"
|
||||
return f"Removed {count:,} {noun}"
|
||||
noun = "link to a missing file" if count == 1 else "links to missing files"
|
||||
line = f"Removed {count:,} {noun}"
|
||||
if marks:
|
||||
gone = "removed image" if marks == 1 else "removed images"
|
||||
line += f"; forgot {marks:,} {gone} whose files are gone or changed"
|
||||
return line
|
||||
return f"Removed {count:,} {noun}"
|
||||
|
||||
|
||||
def take_safety_backup(model, label="backup"):
|
||||
@@ -423,7 +419,6 @@ class MaintenanceRoutesMixin:
|
||||
draft.public(),
|
||||
{
|
||||
"targets": draft.targets,
|
||||
"marks": draft.marks,
|
||||
"roots": roots,
|
||||
"protected": draft.protected["count"],
|
||||
"rows": draft.scope["count"],
|
||||
@@ -493,9 +488,7 @@ class MaintenanceRoutesMixin:
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
async def apply_maintenance_plan(self, request):
|
||||
from .image_delete import IMAGE_DELETE, parse_mode
|
||||
|
||||
data, error = await self._read_maintenance_body(request, ("token", "mode"))
|
||||
data, error = await self._read_maintenance_body(request, ("token",))
|
||||
if error is not None:
|
||||
return error
|
||||
token = data.get("token")
|
||||
@@ -504,13 +497,6 @@ class MaintenanceRoutesMixin:
|
||||
plan, refused = self._peek_plan(token)
|
||||
if refused is not None:
|
||||
return refused
|
||||
mode = None
|
||||
if plan.operation == IMAGE_DELETE:
|
||||
mode = parse_mode(data.get("mode"))
|
||||
if mode is None:
|
||||
return _invalid('mode must be "library" or "trash"')
|
||||
elif "mode" in data:
|
||||
return _invalid("Unknown field: mode")
|
||||
store = self._maintenance_plans
|
||||
if plan.operation == ROTATED_LOGS:
|
||||
# Log files only: no database identity, backup or disk check.
|
||||
@@ -558,8 +544,6 @@ class MaintenanceRoutesMixin:
|
||||
try:
|
||||
if plan.operation == DUPLICATE_FILES:
|
||||
job = self._start_duplicate_cleanup(db, plan, backup)
|
||||
elif plan.operation == IMAGE_DELETE:
|
||||
job = self._start_image_delete(db, plan, mode, backup)
|
||||
elif plan.operation == WORKFLOW_STORAGE:
|
||||
job = self._start_workflow_storage(db, backup)
|
||||
else:
|
||||
@@ -658,7 +642,6 @@ class MaintenanceRoutesMixin:
|
||||
plan.fingerprint,
|
||||
payload["targets"],
|
||||
payload["roots"],
|
||||
payload.get("marks", ()),
|
||||
)
|
||||
except PlanChanged:
|
||||
return _fail_job(job, "plan-changed", PLAN_CHANGED)
|
||||
@@ -670,7 +653,7 @@ class MaintenanceRoutesMixin:
|
||||
)
|
||||
job.record_counted("changed", result["changed"], rows)
|
||||
job.record_counted("protected", payload["protected"])
|
||||
line = _changed_line(plan.operation, result["changed"], result.get("marks", 0))
|
||||
line = _changed_line(plan.operation, result["changed"])
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
"""Revalidated responses for the pages and static files PromptManager serves.
|
||||
|
||||
``Cache-Control: no-cache`` lets the browser keep a copy but makes it ask the
|
||||
server before every reuse, so a PromptManager update reaches the browser on a
|
||||
normal reload (#478). The ETag lets an unchanged file answer 304 without a
|
||||
body, which keeps that reload cheap.
|
||||
|
||||
The tag hashes the bytes rather than the file's mtime and size: an update can
|
||||
keep both (an archive extract, a same-length edit within one clock tick), and
|
||||
the handlers read the whole file anyway. It is weak because the gzip
|
||||
middleware may send the same content in a different encoding.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
REVALIDATE = "no-cache"
|
||||
|
||||
|
||||
def entity_tag(body: bytes) -> str:
|
||||
"""Weak ETag for ``body``."""
|
||||
return f'W/"{hashlib.sha256(body).hexdigest()[:32]}"'
|
||||
|
||||
|
||||
def etag_matches(if_none_match, etag: str) -> bool:
|
||||
"""True when an If-None-Match header names ``etag`` (weak comparison)."""
|
||||
if not if_none_match:
|
||||
return False
|
||||
if if_none_match.strip() == "*":
|
||||
return True
|
||||
opaque = etag.removeprefix("W/")
|
||||
return any(
|
||||
candidate.strip().removeprefix("W/") == opaque
|
||||
for candidate in if_none_match.split(",")
|
||||
)
|
||||
|
||||
|
||||
def revalidated_response(request, body: bytes, content_type: str, charset=None):
|
||||
"""200 with ``body``, or 304 when the request already holds this version."""
|
||||
etag = entity_tag(body)
|
||||
headers = {"Cache-Control": REVALIDATE, "ETag": etag}
|
||||
if etag_matches(request.headers.get("If-None-Match"), etag):
|
||||
return web.Response(status=304, headers=headers)
|
||||
return web.Response(
|
||||
body=body, content_type=content_type, charset=charset, headers=headers
|
||||
)
|
||||
@@ -8,8 +8,7 @@ is shared with the LoRA integration's save.
|
||||
GET also reports the image sources and the monitor (``sources.py``). A save
|
||||
that changes the sources runs as a ``source_change`` registry hold, so it is
|
||||
refused while a record-changing job (a scan) runs. ``monitoring_enabled``
|
||||
takes effect at the next restart; ``save_foreign_runs`` (#247) at once, since
|
||||
the monitor reads it for every file. ``POST /prompt_manager/settings/validate-source``
|
||||
takes effect at the next restart. ``POST /prompt_manager/settings/validate-source``
|
||||
checks one path and writes nothing.
|
||||
"""
|
||||
|
||||
@@ -44,7 +43,6 @@ _SETTINGS_FILE_KEYS = {
|
||||
"INFINITE_SCROLL": ("web_ui", "infinite_scroll"),
|
||||
"WORKER_THREADS": ("performance", "worker_threads"),
|
||||
"MONITORING_ENABLED": ("gallery", "monitoring", "enabled"),
|
||||
"SAVE_FOREIGN_RUNS": ("gallery", "monitoring", "save_foreign_runs"),
|
||||
"SAFETY_BACKUP": ("maintenance", "safety_backup"),
|
||||
"THUMBNAIL_FORMAT": ("thumbnails", "format"),
|
||||
"THUMBNAIL_QUALITY": ("thumbnails", "quality"),
|
||||
@@ -60,7 +58,7 @@ _SETTINGS_FILE_KEYS = {
|
||||
# The thumbnail settings a running Convert outdated job fixes (#317).
|
||||
_THUMBNAIL_SETTINGS = ("THUMBNAIL_FORMAT", "THUMBNAIL_QUALITY", "THUMBNAIL_EDGE")
|
||||
# Attributes that live on GalleryConfig; the others on PromptManagerConfig.
|
||||
_GALLERY_SETTINGS = frozenset({"MONITORING_ENABLED", "SAVE_FOREIGN_RUNS"})
|
||||
_GALLERY_SETTINGS = frozenset({"MONITORING_ENABLED"})
|
||||
# Attributes that live on AutotagConfig, named without the AUTOTAG_ prefix.
|
||||
_AUTOTAG_SETTINGS = frozenset(
|
||||
name for name in _SETTINGS_FILE_KEYS if name.startswith("AUTOTAG_")
|
||||
@@ -266,7 +264,6 @@ class SettingsRoutesMixin:
|
||||
"thumbnail_encoders": encoders,
|
||||
"settings": {
|
||||
"monitoring_enabled": GalleryConfig.MONITORING_ENABLED,
|
||||
"save_foreign_runs": GalleryConfig.SAVE_FOREIGN_RUNS,
|
||||
"result_timeout": PromptManagerConfig.RESULT_TIMEOUT,
|
||||
"webui_display_mode": PromptManagerConfig.WEBUI_DISPLAY_MODE,
|
||||
"infinite_scroll": PromptManagerConfig.INFINITE_SCROLL,
|
||||
@@ -629,10 +626,6 @@ class SettingsRoutesMixin:
|
||||
if not isinstance(data["monitoring_enabled"], bool):
|
||||
raise ValueError("monitoring_enabled must be true or false")
|
||||
updates["MONITORING_ENABLED"] = data["monitoring_enabled"]
|
||||
if "save_foreign_runs" in data:
|
||||
if not isinstance(data["save_foreign_runs"], bool):
|
||||
raise ValueError("save_foreign_runs must be true or false")
|
||||
updates["SAVE_FOREIGN_RUNS"] = data["save_foreign_runs"]
|
||||
if "safety_backup" in data:
|
||||
if not isinstance(data["safety_backup"], bool):
|
||||
raise ValueError("safety_backup must be true or false")
|
||||
|
||||
@@ -194,8 +194,6 @@ class GalleryConfig:
|
||||
|
||||
Attributes:
|
||||
MONITORING_ENABLED (bool): Enable/disable automatic image monitoring
|
||||
SAVE_FOREIGN_RUNS (bool): The monitor stores the certain prompt of an
|
||||
output whose workflow has no PromptManager node (#247)
|
||||
MONITORING_DIRECTORIES (List[str]): Directories to monitor for new images
|
||||
SUPPORTED_EXTENSIONS (List[str]): Image file extensions to process
|
||||
PROCESSING_DELAY (float): Delay in seconds before processing new files
|
||||
@@ -212,8 +210,6 @@ class GalleryConfig:
|
||||
|
||||
# Image monitoring settings
|
||||
MONITORING_ENABLED = True
|
||||
# Read by the monitor for every file, so a save applies without a restart
|
||||
SAVE_FOREIGN_RUNS = True
|
||||
MONITORING_DIRECTORIES = [] # Auto-detect if empty
|
||||
SUPPORTED_EXTENSIONS = [".png", ".jpg", ".jpeg", ".webp", ".gif"]
|
||||
PROCESSING_DELAY = 2.0 # Seconds to wait before processing new files
|
||||
@@ -255,7 +251,6 @@ class GalleryConfig:
|
||||
return {
|
||||
"monitoring": {
|
||||
"enabled": cls.MONITORING_ENABLED,
|
||||
"save_foreign_runs": cls.SAVE_FOREIGN_RUNS,
|
||||
"directories": cls.MONITORING_DIRECTORIES,
|
||||
"extensions": cls.SUPPORTED_EXTENSIONS,
|
||||
"processing_delay": cls.PROCESSING_DELAY,
|
||||
@@ -308,14 +303,6 @@ class GalleryConfig:
|
||||
config_logger.warning(
|
||||
"Ignoring gallery monitoring enabled: expected true or false"
|
||||
)
|
||||
if "save_foreign_runs" in monitoring:
|
||||
if isinstance(monitoring["save_foreign_runs"], bool):
|
||||
cls.SAVE_FOREIGN_RUNS = monitoring["save_foreign_runs"]
|
||||
else:
|
||||
config_logger.warning(
|
||||
"Ignoring gallery monitoring save_foreign_runs: expected true "
|
||||
"or false"
|
||||
)
|
||||
if "directories" in monitoring:
|
||||
directories = cls._validated_directories(monitoring["directories"])
|
||||
if directories is not None:
|
||||
|
||||
@@ -149,8 +149,8 @@ test('panel matches the concept order and copy',async({page,request})=>{
|
||||
await expect(panel.locator('.meta-grid dt')).toHaveText(['Seed','Steps','CFG','Sampler','Scheduler','Model']);
|
||||
await expect(panel.locator('.meta-grid dd')).toHaveText(['9007199254740993','28','6.5','dpmpp_2m','karras','fixture-model.safetensors']);
|
||||
const footer=panel.locator('.detail-footer');
|
||||
// #331 D139: every listed file's footer has Collections… (folder icon); #480 D159: Delete… ends it.
|
||||
await expect(footer.locator('button')).toHaveText(['Send to ComfyUI','Prompt history','Collections…','Delete…']);
|
||||
// #331 D139: every listed file's footer ends with Collections… (folder icon).
|
||||
await expect(footer.locator('button')).toHaveText(['Send to ComfyUI','Prompt history','Collections…']);
|
||||
await expect(footer.locator('button.send')).toHaveAccessibleName(`Send prompt pair to ComfyUI: ${POSITIVE_TITLE}`);
|
||||
await expect(footer.locator('button.send svg.icon')).toHaveCount(1);await expect(footer.locator('button:not(.send) svg.icon')).toHaveCount(2);
|
||||
await expect(panel).not.toContainText(/prompt evidence|Positive prompt tags/);
|
||||
@@ -162,10 +162,10 @@ test('panel matches the concept order and copy',async({page,request})=>{
|
||||
await selectFile(page,'ambiguous.png');await expect(panel.locator('.prompt-section.positive .prompt-text')).toHaveText('embedded positive');
|
||||
await expect(panel.locator('span.chip.blue')).toHaveText('Embedded parameters');
|
||||
await expect(panel.locator('.panel-heading .chip')).toHaveText('File extension PNG');await expect(panel.locator('.panel-heading .chip .ca-visually-hidden')).toHaveText('File extension ');
|
||||
await expect(panel.locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Collections…','Delete…']);
|
||||
// Unlinked: no badge, category, tags, pair or history; the footer holds Collections… (D139) and Delete… (D159).
|
||||
await expect(panel.locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Collections…']);
|
||||
// Unlinked: no badge, category, tags, pair or history; the footer holds only Collections… (D139).
|
||||
reads.patch=null;await selectFile(page,'empty.png');await expect(panel.locator('.panel-heading h2')).toHaveText('empty.png');
|
||||
await expect(panel.locator('.detail-footer button')).toHaveText(['Collections…','Delete…']);
|
||||
await expect(panel.locator('.detail-footer button')).toHaveText(['Collections…']);
|
||||
expect(await panelStructure(panel)).toEqual({outer:['img.detail-cover','div.detail-inner','div.detail-footer'],
|
||||
inner:['div.panel-heading','p.small','section.prompt-section.positive','section.prompt-section.negative','dl.meta-grid']});
|
||||
await expect(panel.locator('.meta-grid dd')).toHaveText(Array(6).fill('Not recorded'));
|
||||
@@ -266,7 +266,7 @@ test('pair send and history appear only with real roles and link',async({page,re
|
||||
await expect(panel.getByRole('button',{name:'Send prompt pair to ComfyUI: explicit-empty.png',exact:true})).toBeVisible();
|
||||
await expect(panel.getByRole('button',{name:'Prompt history',exact:true})).toHaveCount(0);
|
||||
await selectFile(page,'empty.png');await expect(panel.locator('.panel-heading h2')).toHaveText('empty.png');
|
||||
await expect(panel.locator('.detail-footer button')).toHaveText(['Collections…','Delete…']);await expect(panel.getByRole('button',{name:/^Send prompt pair/})).toHaveCount(0);
|
||||
await expect(panel.locator('.detail-footer button')).toHaveText(['Collections…']);await expect(panel.getByRole('button',{name:/^Send prompt pair/})).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('no provenance, raw or download controls in the side panel',async({page,request})=>{
|
||||
@@ -627,10 +627,6 @@ test('viewer matches the concept shell and toolbar order',async({page,request})=
|
||||
expect(modePaths.map(path=>path.split(' > ').slice(2).join(' > '))).toEqual(['button#viewer-mode.icon-btn','button#viewer-mode.icon-btn > svg.icon','button#viewer-mode.icon-btn > svg.icon > path']);
|
||||
expect(production.paths[production.paths.indexOf(modePaths[2])+1]).toBe('dialog#viewer-dialog > div.dialog-header > button#viewer-close');
|
||||
production.paths=production.paths.filter(path=>!modePaths.includes(path));
|
||||
// #480 D159: the Images viewer's Delete… (`danger`) follows Download; the concept has none.
|
||||
expect(production.toolbar[8].text).toBe('Delete…');
|
||||
production.toolbar.splice(8,1);
|
||||
production.paths=production.paths.filter(path=>!path.includes('button.danger'));
|
||||
// #249 D34: after Next → production appends its own status lines (position, then the transform and slideshow
|
||||
// reasons), and Download stays the original-file link; every other control keeps the concept's text and order.
|
||||
const appended=production.toolbar.slice(9);
|
||||
|
||||
@@ -31,15 +31,13 @@ function captionDay(seconds,now=new Date()) {
|
||||
}
|
||||
|
||||
/**
|
||||
* D8 (amended by #249 PR D and #331): the selection bar's real actions only. Add to collection compares in place
|
||||
* (D138). #480 D159: Remove links… returns as Delete…; OMITTED_BULK names the concept button it replaces.
|
||||
* D8 (amended by #249 PR D and #331): the selection bar's real actions only; Remove links… is left out. Add to
|
||||
* collection compares in place (D138).
|
||||
*/
|
||||
const OMITTED_BULK=Object.freeze(['Remove links…']);
|
||||
const OMITTED_BULK_SELECTORS=Object.freeze(['bulk-remove'].map(action=>`#bulkbar [data-action="${action}"]`));
|
||||
/** D85: Auto-tag selected follows the concept's Add tags, a concept button of its own text. */
|
||||
const AUTO_TAG_CLONE=Object.freeze([{selector:'#bulkbar [data-action="bulk-tags"]',text:'Auto-tag selected'}]);
|
||||
/** #480 D159: Delete… takes the place of Remove links… (the concept's `danger` button, under another text). */
|
||||
const DELETE_CLONE=Object.freeze([{selector:'#bulkbar [data-action="bulk-remove"]',text:'Delete…'}]);
|
||||
|
||||
/**
|
||||
* `withoutItems` for a row with padding and a border (the `.bulkbar`): its items flow again inside the content box,
|
||||
@@ -454,12 +452,10 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`images selecti
|
||||
const restore=await mutate(page,{remove:[...OMITTED_BULK_SELECTORS]});
|
||||
const removed=(await inventoryOf(page,'#main')).bulkbar;
|
||||
await restore();
|
||||
// The oracle for production's bar: the concept page itself with D85's button and D159's Delete… in place of
|
||||
// Remove links….
|
||||
const restoreClones=await mutate(page,{clone:[...AUTO_TAG_CLONE,...DELETE_CLONE]});
|
||||
const restoreRemoved=await mutate(page,{remove:[...OMITTED_BULK_SELECTORS]});
|
||||
// The oracle for production's bar: the concept page itself without the omitted actions and with D85's button.
|
||||
const restoreProduction=await mutate(page,{remove:[...OMITTED_BULK_SELECTORS],clone:AUTO_TAG_CLONE});
|
||||
const oracle=await inventoryOf(page,'#main');
|
||||
await restoreRemoved();await restoreClones();
|
||||
await restoreProduction();
|
||||
await captureCanonicalState(page,testInfo,`concept-images-selected-${width}-${theme}`);
|
||||
await openProductionLoaded(page,theme,'no-preference');
|
||||
const emptyActual=await inventoryOf(page,'#workspace-main');
|
||||
@@ -472,7 +468,7 @@ for(const [width,height] of SIZES)for(const theme of THEMES)test(`images selecti
|
||||
'Add to collection','Export metadata','Remove links…','Clear']);
|
||||
expect.soft(diffInventory(stripExact(mapBulkbar(reference.bulkbar,0).bulkbar.bar),stripExact(removed.bar)),'bar without the omitted actions').toEqual([]);
|
||||
expect.soft(actual.bulkbar.bar.items.map(item=>item.text),'production bar').toEqual(['1 selected','Add tags','Auto-tag selected',
|
||||
'Set category','Add to collection','Export metadata','Delete…','Clear']);
|
||||
'Set category','Add to collection','Export metadata','Clear']);
|
||||
expect.soft(compare(empty,emptyActual,manifest,['bulkbar','cards','panel']),'nothing checked').toEqual([]);
|
||||
expect.soft(compare(oracle,actual,manifest,['bulkbar','cards','panel']),'one card checked').toEqual([]);
|
||||
// The bar comparison notices a change to the production bar.
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {SIZES,THEMES,seedConceptAt,openConcept,openProduction,diffInventory,mutate} from './concept-inventory.mjs';
|
||||
|
||||
// #480 (D159): the Images selection bar's Delete… against the concept's "Remove links…" (app.js a432, `danger`) and
|
||||
// its review against the concept's `maintenancePlan` (a953–957): the bar button's place and style, the review's
|
||||
// `dl.meta-grid` rows and its actions, on the concept-shaped fixture, at 1600x1100 and 390x844 in both themes.
|
||||
// Paired screenshots are artifacts only; there are no PNG baselines.
|
||||
test.use({locale:'en-US',timezoneId:'UTC'});
|
||||
|
||||
const CARDS='.gallery-body > .image-grid > .media-card';
|
||||
|
||||
/** Runs in the page: the selection bar's danger button and where it sits among the bar's buttons. */
|
||||
function measureBar() {
|
||||
const bar=document.querySelector('#bulkbar > .bulkbar');
|
||||
const buttons=[...bar.querySelectorAll(':scope > button')];
|
||||
const danger=bar.querySelector(':scope > button.danger');
|
||||
const css=(node,...keys)=>Object.fromEntries(keys.map(key=>[key,getComputedStyle(node)[key]]));
|
||||
return {
|
||||
// The danger action sits right before Clear, the last button.
|
||||
dangerBeforeClear:buttons.indexOf(danger)===buttons.length-2,
|
||||
lastText:buttons.at(-1).textContent.trim(),
|
||||
danger:css(danger,'color','backgroundColor','borderTopColor','fontSize','fontWeight','paddingTop','paddingLeft',
|
||||
'borderTopLeftRadius','height'),
|
||||
};
|
||||
}
|
||||
|
||||
/** Runs in the page: the open review's grid and actions (the parts production shares with the concept). */
|
||||
function measureReview() {
|
||||
const dialog=document.querySelector('dialog#modal[open]');
|
||||
const css=(node,...keys)=>node?Object.fromEntries(keys.map(key=>[key,getComputedStyle(node)[key]])):null;
|
||||
const grid=dialog.querySelector('#modal-body dl.meta-grid');
|
||||
const entry=grid.querySelector(':scope > div');
|
||||
const actions=dialog.querySelector('#modal-body > .dialog-actions');
|
||||
const [cancel]=actions.children;
|
||||
return {
|
||||
grid:css(grid,'display','gridTemplateColumns','columnGap','rowGap'),
|
||||
dt:css(entry.querySelector('dt'),'fontSize','color','textTransform','letterSpacing'),
|
||||
dd:css(entry.querySelector('dd'),'fontSize','color','marginLeft'),
|
||||
terms:[...grid.querySelectorAll('dt')].map(dt=>dt.textContent.trim()),
|
||||
actions:css(actions,'display','justifyContent','columnGap','marginTop'),
|
||||
cancel:{text:cancel.textContent.trim(),...css(cancel,'backgroundColor','color','fontSize','paddingTop','paddingLeft')},
|
||||
};
|
||||
}
|
||||
|
||||
async function checkFirst(page) {
|
||||
await page.locator(CARDS).first().locator('.image-overlays > label > input[type="checkbox"]').check();
|
||||
}
|
||||
|
||||
async function concept(page,theme) {
|
||||
await openConcept(page,'images',theme);
|
||||
await checkFirst(page);
|
||||
const bar=await page.evaluate(measureBar);
|
||||
await page.locator('#main #bulkbar').getByRole('button',{name:'Remove links…',exact:true}).click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review maintenance plan');
|
||||
await page.mouse.move(0,0);
|
||||
return {bar,review:await page.evaluate(measureReview)};
|
||||
}
|
||||
|
||||
async function production(page,theme) {
|
||||
await openProduction(page,'images',theme);
|
||||
await expect(page.locator(CARDS).first()).toBeVisible();
|
||||
await checkFirst(page);
|
||||
const bar=await page.evaluate(measureBar);
|
||||
await page.locator('#workspace-main #bulkbar').getByRole('button',{name:'Delete…',exact:true}).click();
|
||||
const dialog=page.locator('dialog#modal[open]');
|
||||
await expect(dialog.locator('#modal-title')).toHaveText('Delete 1 image');
|
||||
await dialog.getByRole('radio',{name:/^Remove from the library only/}).check();
|
||||
await page.mouse.move(0,0);
|
||||
return {bar,review:await page.evaluate(measureReview)};
|
||||
}
|
||||
|
||||
/**
|
||||
* D159: production's rows follow the chosen mode (Scope, then what the files lose, Backup and Media on disk; the
|
||||
* concept's sample has Scope, Protected, Backup and Media on disk). The geometry, tokens and the Cancel button match.
|
||||
*/
|
||||
function mapConcept(reference,actual) {
|
||||
return {...reference,review:{...reference.review,terms:actual.review.terms}};
|
||||
}
|
||||
|
||||
test.beforeEach(async({page,request},testInfo)=>{testInfo.manifest=await seedConceptAt(page,request);});
|
||||
|
||||
for(const theme of THEMES)for(const [width,height] of SIZES) {
|
||||
test(`delete matches the concept's remove links ${theme} ${width}x${height}`,async({page},testInfo)=>{
|
||||
await page.setViewportSize({width,height});
|
||||
const reference=await concept(page,theme);
|
||||
await page.screenshot({path:testInfo.outputPath(`concept-remove-links-${theme}-${width}.png`)});
|
||||
const actual=await production(page,theme);
|
||||
await page.screenshot({path:testInfo.outputPath(`production-delete-${theme}-${width}.png`)});
|
||||
expect(reference.review.terms).toEqual(['Scope','Protected','Backup','Media on disk']);
|
||||
expect(actual.review.terms[0]).toBe('Scope');
|
||||
expect(actual.review.terms.slice(-2)).toEqual(['Backup','Media on disk']);
|
||||
expect(diffInventory(mapConcept(reference,actual),actual)).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
test('delete comparison detects mutations',async({page})=>{
|
||||
await page.setViewportSize({width:1600,height:1100});
|
||||
const reference=await concept(page,'dark');
|
||||
const actual=await production(page,'dark');
|
||||
const expected=mapConcept(reference,actual);
|
||||
expect(diffInventory(expected,actual)).toEqual([]);
|
||||
const undo=await mutate(page,{css:'dialog#modal dl.meta-grid { column-gap: 31px; } dialog#modal .dialog-actions { gap: 29px; }'});
|
||||
const paths=diffInventory(expected,{...actual,review:await page.evaluate(measureReview)}).map(item=>item.path);
|
||||
expect(paths).toEqual(expect.arrayContaining(['review.grid.columnGap','review.actions.columnGap']));
|
||||
await undo();
|
||||
});
|
||||
@@ -56,8 +56,7 @@ function withCategoryScope(inventory) {
|
||||
* equal production. `reference.viewerPlain` is the concept's own full details
|
||||
* with its tag chips made plain (D20), D117's summary button added and, under a coarse pointer, D34's targets, all at
|
||||
* runtime: the browser lays out what follows them, as the Images spec's removed toolbar items do. D139's
|
||||
* "Collections…" is drawn after Prompt history in both the concept's panel footer and its full details; #480's
|
||||
* D159 "Delete…" follows it in the panel footer.
|
||||
* "Collections…" is drawn after Prompt history in both the concept's panel footer and its full details.
|
||||
*/
|
||||
export function mapInspectorConcept(reference,{viewport}) {
|
||||
const {panel,viewerPlain,fileControls}=reference;
|
||||
@@ -126,13 +125,6 @@ async function withCollectionsButton(page,selector) {
|
||||
for(const key of Object.keys(button.dataset))delete button.dataset[key];
|
||||
button.append(document.querySelector('.filter-row .chip > svg.icon').cloneNode(true),' Collections…');
|
||||
history.after(button);(window.__collectionsButtons??=[]).push(button);
|
||||
// #480 D159: Delete… ends the panel's actions, drawn as the concept's own `danger` button (its Remove links…).
|
||||
if(selector.startsWith('#image-inspector')) {
|
||||
const remove=history.cloneNode(false);
|
||||
for(const key of Object.keys(remove.dataset))delete remove.dataset[key];
|
||||
remove.className='danger';remove.textContent='Delete…';
|
||||
button.after(remove);window.__collectionsButtons.push(remove);
|
||||
}
|
||||
},selector);
|
||||
await settle(page);
|
||||
return async()=>{await page.evaluate(()=>{for(const button of window.__collectionsButtons??[])button.remove();delete window.__collectionsButtons;});};
|
||||
@@ -381,7 +373,7 @@ test('a recipe-linked image without recipe metadata shows its positive prompt me
|
||||
await expect(tags.locator('.role-label')).toHaveText('Prompt tags · 3');
|
||||
await expect(tags.locator('.tags > button.chip')).toHaveText(['minimal','warm_light','editorial']);
|
||||
await expect(tags.locator('span.chip')).toHaveCount(0);
|
||||
await expect(panelOf(page).locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Prompt history','Collections…','Delete…']);
|
||||
await expect(panelOf(page).locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Prompt history','Collections…']);
|
||||
});
|
||||
|
||||
// #391: a tag chip replaces the collection; until its listing answers the caption says it is loading, never that the
|
||||
|
||||
@@ -44,12 +44,7 @@ export function mapLibraryConcept(reference) {
|
||||
// "4 runs" and "Used today". The two-line clamp is
|
||||
// applied below through `clampedCards`, measured on the concept with `CLAMP_CSS` (its samples wrap at 390px).
|
||||
footer:card.footer}));
|
||||
// D152 (#443): Show archived prompts (N) comes first in the heading actions, a plain button like Export (the concept
|
||||
// fixture has nothing archived, so N is 0).
|
||||
const actions=reference.heading.actions;
|
||||
const plain=actions.buttons.find(button=>!button.primary);
|
||||
const heading={...reference.heading,actions:{...actions,buttons:[{...plain,text:'Show archived prompts (0)'},...actions.buttons]}};
|
||||
return {...reference,heading,library:{...library,toolbar,
|
||||
return {...reference,library:{...library,toolbar,
|
||||
// D35: the caption uses the singular for one.
|
||||
caption:{...library.caption,top:library.caption.top-rise,spans:library.caption.spans.map(span=>({...span,text:countCopy(span.text)}))},
|
||||
...(({grid,cards:clamped})=>({grid:{...grid,top:grid.top-rise},cards:clamped}))(clampedCards({grid:library.grid,cards})),
|
||||
|
||||
@@ -1,147 +0,0 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
|
||||
// #443 D152: the Library's Delete… review against the concept's own reviewed archive: the recipe editor's "Archive…"
|
||||
// (app.js 976), which opens `maintenancePlan("archive <title>")` (953–958, 1309–1310). The concept's shell, grid tokens,
|
||||
// `p.small` and actions are kept; its title, demo note, rows, note text and `ca-primary` start are replaced by D152's,
|
||||
// and the start takes the concept's `danger` class, measured on the editor's own "Archive…". The bar's Delete… is
|
||||
// compared with the concept Images bar's danger action, "Remove links…" (app.js 432).
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
page.conceptErrors = [];
|
||||
page.on('pageerror', error => page.conceptErrors.push(error.message));
|
||||
});
|
||||
test.afterEach(async ({ page, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.conceptErrors).toEqual([]);
|
||||
});
|
||||
|
||||
/** Runs in the page: the open review dialog's title, body tree, grid, notes and actions with the concept's tokens. */
|
||||
function measureReview() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const body = dialog.querySelector('#modal-body');
|
||||
const grid = body.querySelector(':scope > dl.meta-grid');
|
||||
const notes = [...body.querySelectorAll(':scope > p.small')];
|
||||
const actions = body.querySelector(':scope > .dialog-actions');
|
||||
return {
|
||||
title: text(dialog.querySelector('#modal-title')),
|
||||
tree: [...body.children].map(name).filter(item => item !== 'p.demo-note' && item !== 'h3'),
|
||||
grid: { rows: [...grid.children].map(row => [text(row.querySelector('dt')), text(row.querySelector('dd'))]),
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
dt: css(grid.querySelector('dt'), 'fontSize', 'color', 'textTransform'), dd: css(grid.querySelector('dd'), 'fontSize', 'color') },
|
||||
notes: notes.map(note => ({ text: text(note), ...css(note, 'fontSize', 'color', 'marginTop') })),
|
||||
actions: { tree: [...actions.children].map(name), ...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(item => ({ text: text(item), className: item.className,
|
||||
...css(item, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor') })) },
|
||||
};
|
||||
}
|
||||
|
||||
/** Runs in the page: a button's class and tokens (the concept's danger button, or production's). */
|
||||
function measureButton(selector) {
|
||||
const node = document.querySelector(selector);
|
||||
const style = getComputedStyle(node);
|
||||
return { className: node.className, ...Object.fromEntries(['backgroundColor', 'color', 'fontSize', 'fontWeight', 'paddingTop',
|
||||
'paddingLeft', 'borderTopColor', 'borderTopLeftRadius', 'height'].map(key => [key, key === 'height'
|
||||
? `${Math.round(node.getBoundingClientRect().height * 100) / 100}px` : style[key]])) };
|
||||
}
|
||||
|
||||
async function conceptReview(page, theme) {
|
||||
await openConcept(page, 'prompt/alpine', theme);
|
||||
await page.locator('#main .heading-actions').getByRole('button', { name: 'Edit metadata', exact: true }).click();
|
||||
const archive = page.locator('dialog#modal[open] .dialog-actions button', { hasText: 'Archive…' });
|
||||
await expect(archive).toBeVisible();await page.mouse.move(0, 0);
|
||||
const danger = await page.evaluate(measureButton, 'dialog#modal[open] .dialog-actions button.danger');
|
||||
await archive.click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review maintenance plan');
|
||||
await page.mouse.move(0, 0);
|
||||
return { danger, review: await page.evaluate(measureReview) };
|
||||
}
|
||||
|
||||
async function seedMixed(request) {
|
||||
// One unpaired prompt beside the concept's paired ones, so the review deletes one and archives one.
|
||||
await expect(await request.post('/prompt_manager/save', { data: { text: 'Unpaired study', category: 'Studies' } })).toBeOK();
|
||||
const items = (await (await request.get('/prompt_manager/library?limit=100')).json()).items;
|
||||
return { unpaired: items.find(item => item.text === 'Unpaired study'), paired: items.find(item => item.variant_count > 0) };
|
||||
}
|
||||
|
||||
async function productionReview(page, request, theme) {
|
||||
const { unpaired, paired } = await seedMixed(request);
|
||||
await openProduction(page, 'library', theme);
|
||||
for (const item of [unpaired, paired]) await page.locator(`article[data-prompt-id="${item.id}"] .image-overlays input`).check();
|
||||
await page.locator('#library-bulkbar .bulkbar').getByRole('button', { name: 'Delete…', exact: true }).click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Delete or archive selected prompts');
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureReview);
|
||||
}
|
||||
|
||||
/** D152's changes to the concept's review; everything else must equal the concept. */
|
||||
function mapDeleteConcept({ danger, review }) {
|
||||
const [cancel] = review.actions.buttons;
|
||||
const note = review.notes.at(-1);
|
||||
return {
|
||||
...review,
|
||||
title: 'Delete or archive selected prompts',
|
||||
tree: ['dl.meta-grid', 'p.small', 'p.small', 'div.dialog-actions'],
|
||||
grid: { ...review.grid, rows: [['Delete permanently', '1 prompt not in a saved pair'],
|
||||
['Archive', '1 prompt in a saved pair: hidden, restorable'],
|
||||
['Image links removed', '0 images will no longer show these prompts (files stay on disk)'],
|
||||
['Backup', 'Create and verify before deleting']] },
|
||||
notes: [
|
||||
{ ...note, text: 'Prompts in a saved pair, as its positive or its negative, are archived instead of deleted, so the '
|
||||
+ 'pair and its runs stay intact. Archived prompts cannot be deleted. To bring them back, press Show archived prompts, '
|
||||
+ 'select them and press Restore.' },
|
||||
// A second paragraph takes the concept's own `p + p` margin (style.css 93-95).
|
||||
{ ...note, marginTop: '10px', text: "Deleting removes each prompt's text, title, category, tags, rating, notes, run count, last use and "
|
||||
+ 'image links. Deleted prompts cannot be brought back one by one: the only way back is to restore the safety '
|
||||
+ 'backup taken first (System › Backup & export), which returns the whole library to that moment. It is kept '
|
||||
+ 'among the last 10 safety backups. Image files, favorites, collections and recorded runs stay. A later scan, a '
|
||||
+ 'new image or a node run with the same text adds it again as a new prompt.' },
|
||||
],
|
||||
actions: { ...review.actions, tree: [review.actions.tree[0], 'button.danger'], buttons: [cancel, { text: 'Delete 1 and archive 1 prompt', className: danger.className,
|
||||
backgroundColor: danger.backgroundColor, color: danger.color, fontSize: danger.fontSize, paddingTop: danger.paddingTop,
|
||||
paddingLeft: danger.paddingLeft, borderTopColor: danger.borderTopColor }] },
|
||||
};
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`the Delete… review matches the concept's reviewed archive ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const concept = await conceptReview(page, theme);
|
||||
expect(concept.review.tree).toEqual(['dl.meta-grid', 'p.small', 'div.dialog-actions']);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-archive-review-${theme}-${width}.png`) });
|
||||
const production = await productionReview(page, request, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-delete-review-${theme}-${width}.png`) });
|
||||
expect(diffInventory(mapDeleteConcept(concept), production)).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
test('the Delete… review comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const concept = await conceptReview(page, 'dark');
|
||||
await productionReview(page, request, 'dark');
|
||||
const expected = mapDeleteConcept(concept);
|
||||
const paths = async () => diffInventory(expected, await page.evaluate(measureReview)).map(item => item.path);
|
||||
expect(await paths()).toEqual([]);
|
||||
await mutate(page, { css: '#modal-body .dialog-actions .danger { padding-left: 2px }' });
|
||||
expect(await paths()).toContain('actions.buttons.1.paddingLeft');
|
||||
});
|
||||
|
||||
test('the bar\'s Delete… matches the concept Images bar\'s danger action', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await openConcept(page, 'images', 'dark');
|
||||
await page.locator('#image-grid .media-card').first().locator('input[data-select]').check();
|
||||
await expect(page.locator('#bulkbar .bulkbar')).toBeVisible();await page.mouse.move(0, 0);
|
||||
const reference = await page.evaluate(measureButton, '#bulkbar .bulkbar button.danger');
|
||||
const { paired } = await seedMixed(request);
|
||||
await openProduction(page, 'library', 'dark');
|
||||
await page.locator(`article[data-prompt-id="${paired.id}"] .image-overlays input`).check();
|
||||
await expect(page.locator('#library-bulkbar .bulkbar')).toBeVisible();await page.mouse.move(0, 0);
|
||||
expect(await page.evaluate(measureButton, '#library-bulkbar .bulkbar button.danger')).toEqual(reference);
|
||||
});
|
||||
@@ -220,12 +220,6 @@ def create_app(root: Path) -> web.Application:
|
||||
PromptManagerConfig.DEFAULT_DB_PATH = str(root / "browser.db")
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [str(root / "output"), str(root / "input")]
|
||||
|
||||
# #480: Delete… never reaches a real system trash from a page test; the fake
|
||||
# renames into this server's root, which goes when the server exits.
|
||||
from tests import fake_trash
|
||||
|
||||
fake_trash.install(root / "fake-trash")
|
||||
|
||||
from py.api import PromptManagerAPI
|
||||
from py import autotag
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
@@ -87,8 +87,8 @@ function measureBackups(mainSelector) {
|
||||
}
|
||||
|
||||
const SCHEDULE_REASON = 'Daily and Weekly are not available yet: back up with Back up whenever you choose.';
|
||||
const SAFETY_DESCRIPTION = 'Keep a verified copy of the database on disk before maintenance, cleanups, LoRA imports and ' +
|
||||
'prompt deletes change records. A restore always keeps one.';
|
||||
const SAFETY_DESCRIPTION = 'Keep a verified copy of the database on disk before maintenance, cleanups and LoRA imports change ' +
|
||||
'records. A restore always keeps one.';
|
||||
|
||||
/**
|
||||
* D103: "Back up" (the concept's "Simulate backup"), the status line from /backup/status, "Export JSON" (its "Export
|
||||
|
||||
@@ -618,21 +618,10 @@ const WATCHED = new Set(['watching', 'stopping']);
|
||||
const REFRESH_FILES = { title: 'Refresh files', description: 'Read every source again to find files added or removed outside ComfyUI.',
|
||||
label: 'Refresh' };
|
||||
|
||||
// D158 (#247): the row production adds directly below "Watch for new outputs", in that row's markup.
|
||||
const FOREIGN_RUNS = { id: 'foreign-runs', label: 'Save prompts from other workflows',
|
||||
description: 'New outputs of workflows without a PromptManager node add their prompt to the library when it is certain.' };
|
||||
|
||||
/** D158: the concept's "Watch for new outputs" row as the inserted "Save prompts from other workflows" row. */
|
||||
function foreignRunsRow(monitoringRow, settings) {
|
||||
return { ...monitoringRow, text: [['label', FOREIGN_RUNS.label, FOREIGN_RUNS.id], ['p', FOREIGN_RUNS.description, null]],
|
||||
control: { ...monitoringRow.control, id: FOREIGN_RUNS.id, checked: settings.save_foreign_runs } };
|
||||
}
|
||||
|
||||
/** The concept's Sources with the recorded deviations applied, given the server's real settings and sources. */
|
||||
function mapSourcesConcept(reference, { settings, sources }) {
|
||||
const [sample] = reference.sources;
|
||||
const scan = sample.actions[0];
|
||||
const rowAt = reference.surface.tree.indexOf('div.setting-row');
|
||||
return {
|
||||
...reference,
|
||||
// The real sources replace the two samples (without the "sample path" note). D98: each configured one has Remove,
|
||||
@@ -640,15 +629,13 @@ function mapSourcesConcept(reference, { settings, sources }) {
|
||||
sources: sources.effective.map(source => ({ ...sample, title: source.label,
|
||||
description: `${source.path} · ${WATCH_TEXT[source.watch]}`,
|
||||
actions: [{ ...scan, text: WATCHED.has(source.watch) ? 'Scan changes' : 'Scan folder' }, { ...scan, text: 'Remove' }] })),
|
||||
rows: reference.rows.flatMap(row => {
|
||||
if (row.control.id === 'monitoring') return [{ ...row, control: { ...row.control, checked: settings.monitoring_enabled } },
|
||||
foreignRunsRow(row, settings)];
|
||||
rows: reference.rows.map(row => {
|
||||
if (row.control.id === 'monitoring') return { ...row, control: { ...row.control, checked: settings.monitoring_enabled } };
|
||||
// D113: the range ends at the server's core count (the concept's fixed 16) and shows the real worker count.
|
||||
return [{ ...row, control: { ...row.control, max: String(settings.cpu_count), value: String(settings.worker_threads) } }];
|
||||
return { ...row, control: { ...row.control, max: String(settings.cpu_count), value: String(settings.worker_threads) } };
|
||||
}),
|
||||
// D158: the inserted row after the first setting row; D98: "Refresh files" closes the surface.
|
||||
surface: { ...reference.surface, tree: [...reference.surface.tree.slice(0, rowAt + 1), 'div.setting-row',
|
||||
...reference.surface.tree.slice(rowAt + 1), 'div.operation'] },
|
||||
// D98: "Refresh files" closes the surface.
|
||||
surface: { ...reference.surface, tree: [...reference.surface.tree, 'div.operation'] },
|
||||
operations: [...reference.operations, { ...reference.operations[0], title: REFRESH_FILES.title,
|
||||
description: REFRESH_FILES.description, actions: [{ ...reference.operations[0].actions[0], text: REFRESH_FILES.label }] }],
|
||||
};
|
||||
@@ -702,12 +689,10 @@ test('sources comparison detects mutations', async ({ page, request }) => {
|
||||
const expected = mapSourcesConcept(concept, server);
|
||||
const production = await productionSources(page, 'dark', { notice: true });
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Unmapped, the concept differs only in the real sources, the D98 operation, the D158 row (which moves the D113
|
||||
// range to the third row) and the D113 range.
|
||||
// Unmapped, the concept differs only in the real sources, the D98 operation and the D113 range.
|
||||
expect(diffInventory(concept, production).map(item => item.path.split('.').slice(0, 2).join('.'))
|
||||
.filter((path, index, all) => all.indexOf(path) === index).sort())
|
||||
.toEqual(['operations.2', 'operations.length', 'rows.1', 'rows.2', 'rows.length', 'sources.0', 'sources.1',
|
||||
'surface.tree'].sort());
|
||||
.toEqual(['operations.2', 'operations.length', 'rows.1', 'sources.0', 'sources.1', 'surface.tree'].sort());
|
||||
const measure = () => page.evaluate(measureSources, '#workspace-main');
|
||||
const paths = async () => diffInventory(expected, await measure()).map(item => item.path);
|
||||
// A removed notice and a removed source row.
|
||||
@@ -721,13 +706,6 @@ test('sources comparison detects mutations', async ({ page, request }) => {
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main label[for="monitoring"]').textContent = 'Watch for outputs'; });
|
||||
expect(await paths()).toEqual(['rows.0.text.0.1']);
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main label[for="monitoring"]').textContent = 'Watch for new outputs'; });
|
||||
// D158: the inserted row is compared too: a missing one and a changed label are caught.
|
||||
const undoForeign = await mutate(page, { remove: ['#workspace-main .setting-row:has(#foreign-runs)'] });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['rows.length', 'surface.tree.length']));
|
||||
await undoForeign();
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main label[for="foreign-runs"]').textContent = 'Save other prompts'; });
|
||||
expect(await paths()).toEqual(['rows.1.text.0.1']);
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main label[for="foreign-runs"]').textContent = 'Save prompts from other workflows'; });
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main .panel-heading button').textContent = 'Add source'; });
|
||||
expect(await paths()).toEqual(['heading.action.text']);
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main .panel-heading button').textContent = '+ Add source'; });
|
||||
|
||||
@@ -527,7 +527,7 @@ test('Add to collection creates a collection and adds to it ignoring case (CUJ 1
|
||||
await expect(imageCards(page)).toHaveCount(21);
|
||||
for (const file of ['loose.png', 'landscape_01.png', 'experiment_01.png']) await check(page, file);
|
||||
await expect(bar(page).locator('button')).toHaveText(['Add tags', 'Auto-tag selected', 'Set category', 'Add to collection',
|
||||
'Export metadata', 'Delete…', 'Clear']);
|
||||
'Export metadata', 'Clear']);
|
||||
await barButton(page, 'Add to collection').click();
|
||||
// The concept's simpleInput with Add (D138) and the existing names as suggestions.
|
||||
await expect(addDialog(page).locator('label')).toHaveText('Collection name');
|
||||
@@ -578,7 +578,7 @@ test('Remove from collection takes the checked images off the open collection (C
|
||||
await check(page, 'landscape_01.png');
|
||||
await check(page, 'landscape_02.png');
|
||||
await expect(bar(page).locator('button')).toHaveText(['Add tags', 'Auto-tag selected', 'Set category', 'Add to collection',
|
||||
'Remove from collection', 'Export metadata', 'Delete…', 'Clear']);
|
||||
'Remove from collection', 'Export metadata', 'Clear']);
|
||||
await barButton(page, 'Remove from collection').click();
|
||||
await expect(toast(page)).toHaveText('Removed 2 images from “Landscapes”.');
|
||||
await expect(imageCards(page)).toHaveCount(10);
|
||||
@@ -715,8 +715,7 @@ test('Collections… in the side panel adds, removes and creates for one image (
|
||||
// The newest image, landscape_01, is the panel's.
|
||||
await expect(panel(page).getByRole('heading', { level: 2 })).toHaveText('landscape_01.png');
|
||||
const open = panel(page).getByRole('button', { name: 'Collections…', exact: true });
|
||||
// #480 D159: Delete… follows Collections… at the end of the panel's actions.
|
||||
await expect(panel(page).locator('.detail-footer > button').nth(-2)).toHaveText('Collections…');
|
||||
await expect(panel(page).locator('.detail-footer > button').last()).toHaveText('Collections…');
|
||||
await open.click();
|
||||
await expect(collectionsDialog(page)).toBeVisible();
|
||||
await expect(choice(page, 'Landscapes')).toBeChecked();
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
|
||||
import {saveImageTags} from './concept-inventory.mjs';
|
||||
import {readFile,writeFile} from 'node:fs/promises';
|
||||
import {createHash} from 'node:crypto';
|
||||
|
||||
@@ -496,38 +495,23 @@ test('the inline current file names its prompts apart from the frozen run fields
|
||||
expect(await exact(dialog(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'))).toBe(before);
|
||||
});
|
||||
|
||||
// #449 (D154), #457: History's inline current file draws the file's own image tags in the shared labelled section, named
|
||||
// with its context so they stay apart from the frozen run's fields. The tags are saved on the real file through the Tag
|
||||
// routes and come from History's own current-file read (its `image_tags` facet), as Images reads them.
|
||||
test('the inline current file labels its image tags after its negative prompt',async({page,request})=>{
|
||||
const reads=[];
|
||||
// #449 (D154): History's inline current file draws tags in the shared labelled sections, named with its context so they
|
||||
// stay apart from the frozen run's fields. The server sends History no image_tags facet today (D88), so the read is
|
||||
// given one here to prove the shared render in History.
|
||||
test('the inline current file labels its image tags after its negative prompt',async({page})=>{
|
||||
await page.route(url=>url.pathname.endsWith('/inspection'),async route=>{const response=await route.fetch();const body=await response.json();
|
||||
reads.push(body.image_tags);
|
||||
body.prompts={positive:{status:'resolved',text:'current file positive',source:'graph-inferred'},negative:{status:'resolved',text:'current file negative',source:'graph-inferred'}};
|
||||
body.image_tags={state:'current',tags:['history_tag']};
|
||||
await route.fulfill({response,json:body});});
|
||||
await open(page,page.fixture.detail_id);await marker(page,page.fixture.runs.shared).click();await expect(dialog(page).locator('[data-receipt]')).toHaveCount(5);
|
||||
await dialog(page).getByRole('button',{name:'Raw',exact:true}).click();
|
||||
await dialog(page).locator('[data-receipt]').first().getByRole('button',{name:'View current file',exact:true}).click();
|
||||
await expect(page.locator('dialog#viewer-dialog')).toBeVisible();await page.keyboard.press('Escape');await expect(dialog(page)).toBeVisible();
|
||||
const current=dialog(page).getByRole('region',{name:'Current reported file',exact:true});
|
||||
const tags=current.getByRole('region',{name:'Current file image tags',exact:true});
|
||||
const showCurrent=async()=>{
|
||||
await open(page,page.fixture.detail_id);await marker(page,page.fixture.runs.shared).click();await expect(dialog(page).locator('[data-receipt]')).toHaveCount(5);
|
||||
await dialog(page).getByRole('button',{name:'Raw',exact:true}).click();
|
||||
await dialog(page).locator('[data-receipt]').first().getByRole('button',{name:'View current file',exact:true}).click();
|
||||
await expect(page.locator('dialog#viewer-dialog')).toBeVisible();await page.keyboard.press('Escape');await expect(dialog(page)).toBeVisible();
|
||||
await expect(current.getByRole('region',{name:'Current file negative prompt',exact:true})).toBeVisible();
|
||||
};
|
||||
// An untagged file draws no image tags section.
|
||||
await showCurrent();await expect(tags).toHaveCount(0);expect(reads.at(-1)).toEqual({state:'none',tags:[]});
|
||||
await dialog(page).getByRole('button',{name:'Close recorded run',exact:true}).click();
|
||||
expect(await saveImageTags(request,'history/current.png',['history_tag','dusk'])).toEqual(['history_tag','dusk']);
|
||||
await showCurrent();expect(reads.at(-1)).toEqual({state:'current',tags:['history_tag','dusk']});
|
||||
await expect(tags.locator('.role-label')).toHaveText('Image tags · 2');await expect(tags.locator('.tags > .chip')).toHaveText(['history_tag','dusk']);
|
||||
await expect(tags.locator('.role-label')).toHaveText('Image tags · 1');await expect(tags.locator('.tags > .chip')).toHaveText(['history_tag']);
|
||||
const negative=current.getByRole('region',{name:'Current file negative prompt',exact:true});
|
||||
expect(await negative.evaluate((node,section)=>({after:Boolean(node.compareDocumentPosition(section)&Node.DOCUMENT_POSITION_FOLLOWING),inside:node.contains(section)}),await tags.elementHandle())).toEqual({after:true,inside:false});
|
||||
// The viewer opened from History shares the inspection: the same section, plain chips (no Tags filter there, D88).
|
||||
await dialog(page).locator('[data-receipt]').first().getByRole('button',{name:'View current file',exact:true}).click();
|
||||
const viewerTags=page.locator('dialog#viewer-dialog #viewer-details section.tag-section.image-tags');
|
||||
await expect(viewerTags.locator('.role-label')).toHaveText('Image tags · 2');
|
||||
await expect(viewerTags.locator('span.chip')).toHaveText(['history_tag','dusk']);await expect(viewerTags.locator('button')).toHaveCount(0);
|
||||
await page.keyboard.press('Escape');await expect(dialog(page)).toBeVisible();
|
||||
});
|
||||
|
||||
// #332 CUJ 5 (MC8, §5.3): History's current-file inspector reads the file's own category with the Images key rule, so
|
||||
|
||||
@@ -1,184 +0,0 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConceptAt, openProduction } from './concept-inventory.mjs';
|
||||
|
||||
// #480 (D159): Images › Delete…: the review asks every time (nothing preselected), "Remove from the library only"
|
||||
// hides the file and keeps it on disk, "Also move the files to the trash" moves it (the server's fake trash here).
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
const CARDS = '#workspace-main article[data-image-key]';
|
||||
const bar = page => page.locator('#workspace-main #bulkbar');
|
||||
const review = page => page.getByRole('dialog', { name: /^Delete \d+ images?$/ });
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConceptAt(page, request);
|
||||
page.deleteErrors = [];
|
||||
page.on('pageerror', error => page.deleteErrors.push(error.message));
|
||||
page.on('dialog', dialog => { page.deleteErrors.push(`browser ${dialog.type()}`); dialog.dismiss().catch(() => {}); });
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
expect(page.deleteErrors).toEqual([]);
|
||||
});
|
||||
|
||||
async function openImages(page) {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
await openProduction(page, 'images', 'dark');
|
||||
await expect(page.locator(CARDS).first()).toBeVisible();
|
||||
return page.locator(CARDS).count();
|
||||
}
|
||||
|
||||
async function check(page, index) {
|
||||
const card = page.locator(CARDS).nth(index);
|
||||
const key = await card.getAttribute('data-image-key');
|
||||
await card.locator('.image-overlays input[type="checkbox"]').check();
|
||||
return key;
|
||||
}
|
||||
|
||||
async function deleteChecked(page, mode) {
|
||||
await bar(page).getByRole('button', { name: 'Delete…', exact: true }).click();
|
||||
const dialog = review(page);
|
||||
await expect(dialog).toBeVisible();
|
||||
const primary = dialog.locator('.dialog-actions button.danger');
|
||||
// Ask every time: no mode is preselected and the primary waits for one.
|
||||
await expect(dialog.getByRole('radio')).toHaveCount(2);
|
||||
for (const radio of await dialog.getByRole('radio').all()) await expect(radio).not.toBeChecked();
|
||||
await expect(primary).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(dialog.locator('dl.meta-grid')).toBeHidden();
|
||||
const label = mode === 'trash' ? 'Also move the files to the trash' : 'Remove from the library only';
|
||||
await dialog.getByRole('radio', { name: new RegExp(`^${label}`) }).check();
|
||||
await expect(dialog.locator('dl.meta-grid')).toBeVisible();
|
||||
await expect(primary).not.toHaveAttribute('aria-disabled', 'true');
|
||||
return { dialog, primary };
|
||||
}
|
||||
|
||||
for (const mode of ['library', 'trash']) {
|
||||
test(`${mode}: the deleted image leaves Images and stays gone after a reload`, async ({ page }) => {
|
||||
const before = await openImages(page);
|
||||
const key = await check(page, 0);
|
||||
const { dialog, primary } = await deleteChecked(page, mode);
|
||||
await expect(dialog.locator('dt', { hasText: 'Media on disk' })).toBeVisible();
|
||||
await expect(primary).toHaveText(mode === 'trash' ? 'Move 1 image to the trash' : 'Remove 1 image from the library');
|
||||
await primary.click();
|
||||
await expect(toast(page)).toContainText(mode === 'trash' ? 'Moved 1 image to the trash.' : 'Removed 1 image from the library.');
|
||||
await expect(page.locator(`${CARDS}[data-image-key="${key}"]`)).toHaveCount(0);
|
||||
await expect(page.locator(CARDS)).toHaveCount(before - 1);
|
||||
await expect(bar(page)).toBeEmpty();
|
||||
await page.reload();
|
||||
await expect(page.locator(CARDS).first()).toBeVisible();
|
||||
await expect(page.locator(`${CARDS}[data-image-key="${key}"]`)).toHaveCount(0);
|
||||
});
|
||||
}
|
||||
|
||||
// #501: the sidebar's Images count (read once at mount) follows a finished delete without a reload.
|
||||
test('a finished delete re-reads the sidebar Images count', async ({ page }) => {
|
||||
await openImages(page);
|
||||
const badge = page.locator('#primary-nav a[data-route="images"] .nav-count');
|
||||
await expect(badge).toHaveText(/^\d[\d,]*$/);
|
||||
const before = Number((await badge.textContent()).replace(/,/g, ''));
|
||||
await check(page, 0);
|
||||
const { primary } = await deleteChecked(page, 'library');
|
||||
await primary.click();
|
||||
await expect(toast(page)).toContainText('Removed 1 image from the library.');
|
||||
await expect(badge).toHaveText((before - 1).toLocaleString('en-US'));
|
||||
});
|
||||
|
||||
test('Cancel changes nothing', async ({ page }) => {
|
||||
const before = await openImages(page);
|
||||
await check(page, 0);
|
||||
const { dialog } = await deleteChecked(page, 'library');
|
||||
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog).toBeHidden();
|
||||
await expect(page.locator(CARDS)).toHaveCount(before);
|
||||
});
|
||||
|
||||
test('switching modes redraws the rows from one review', async ({ page }) => {
|
||||
await openImages(page);
|
||||
await check(page, 0);
|
||||
let previews = 0;
|
||||
page.on('request', request => { if (request.url().endsWith('/images/delete/preview')) previews += 1; });
|
||||
const { dialog, primary } = await deleteChecked(page, 'library');
|
||||
await expect(dialog.locator('dd', { hasText: 'Files stay on disk, hidden from Images.' })).toBeVisible();
|
||||
await dialog.getByRole('radio', { name: /^Also move the files to the trash/ }).check();
|
||||
await expect(dialog.locator('dd', { hasText: 'Nothing is deleted permanently' })).toBeVisible();
|
||||
await expect(primary).toHaveText('Move 1 image to the trash');
|
||||
expect(previews).toBe(1);
|
||||
});
|
||||
|
||||
test('Delete… in the inspector reviews that one image', async ({ page }) => {
|
||||
const before = await openImages(page);
|
||||
const card = page.locator(CARDS).first();
|
||||
await card.getByRole('button', { name: /^Inspect / }).click();
|
||||
const button = page.locator('#workspace-main .file-inspector button.danger', { hasText: 'Delete…' });
|
||||
await expect(button).toBeVisible();
|
||||
await button.click();
|
||||
const dialog = review(page);
|
||||
await expect(dialog).toBeVisible();
|
||||
await dialog.getByRole('radio', { name: /^Remove from the library only/ }).check();
|
||||
await dialog.locator('.dialog-actions button.danger').click();
|
||||
await expect(page.locator(CARDS)).toHaveCount(before - 1);
|
||||
});
|
||||
|
||||
test('Delete… in the viewer closes it after the delete', async ({ page }) => {
|
||||
const before = await openImages(page);
|
||||
await page.locator(CARDS).first().getByRole('button', { name: /^Open / }).click();
|
||||
const viewer = page.locator('dialog#viewer-dialog[open]');
|
||||
await expect(viewer).toBeVisible();
|
||||
await viewer.getByRole('button', { name: 'Delete…', exact: true }).click();
|
||||
const dialog = review(page);
|
||||
await expect(dialog).toBeVisible();
|
||||
await dialog.getByRole('radio', { name: /^Remove from the library only/ }).check();
|
||||
await dialog.locator('.dialog-actions button.danger').click();
|
||||
await expect(viewer).toBeHidden();
|
||||
await expect(page.locator(CARDS)).toHaveCount(before - 1);
|
||||
});
|
||||
|
||||
// The review never starts a delete without an explicit choice (live test 2026-10-29): every activation of the
|
||||
// aria-disabled primary is ignored, the dialog has no form to submit, and a chosen primary starts once.
|
||||
test('without a chosen mode nothing can start a delete', async ({ page }) => {
|
||||
const before = await openImages(page);
|
||||
await check(page, 0);
|
||||
const applies = [];
|
||||
page.on('request', request => { if (request.url().includes('/maintenance/apply')) applies.push(request.postDataJSON()); });
|
||||
await bar(page).getByRole('button', { name: 'Delete…', exact: true }).click();
|
||||
const dialog = review(page);
|
||||
await expect(dialog).toBeVisible();
|
||||
const primary = dialog.locator('.dialog-actions button.danger');
|
||||
await expect(primary).toHaveAttribute('aria-disabled', 'true');
|
||||
await expect(primary).toHaveText('Choose what happens to the files');
|
||||
// Visibly dimmed while no mode is chosen.
|
||||
await expect(primary).toHaveCSS('opacity', '0.55');
|
||||
await expect(primary).toHaveCSS('cursor', 'not-allowed');
|
||||
await primary.click({ force: true });
|
||||
await primary.dblclick({ force: true });
|
||||
await primary.focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await page.keyboard.press('Space');
|
||||
await dialog.locator('#modal-title').click();
|
||||
await page.keyboard.press('Enter');
|
||||
expect(await dialog.locator('form').count()).toBe(0);
|
||||
await page.evaluate(() => document.querySelector('dialog#modal[open]')?.dispatchEvent(new Event('submit', { bubbles: true })));
|
||||
await expect(dialog).toBeVisible();
|
||||
expect(applies).toEqual([]);
|
||||
for (const radio of await dialog.getByRole('radio').all()) await expect(radio).not.toBeChecked();
|
||||
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(page.locator(CARDS)).toHaveCount(before);
|
||||
expect(applies).toEqual([]);
|
||||
});
|
||||
|
||||
test('a chosen mode starts exactly one delete, with that mode, however often it is pressed', async ({ page }) => {
|
||||
const before = await openImages(page);
|
||||
await check(page, 0);
|
||||
const applies = [];
|
||||
page.on('request', request => { if (request.url().includes('/maintenance/apply')) applies.push(request.postDataJSON()); });
|
||||
const { primary } = await deleteChecked(page, 'library');
|
||||
await primary.dblclick();
|
||||
await expect(toast(page)).toContainText('Removed 1 image from the library.');
|
||||
await expect(page.locator(CARDS)).toHaveCount(before - 1);
|
||||
expect(applies.length).toBe(1);
|
||||
expect(applies[0].mode).toBe('library');
|
||||
expect(Object.keys(applies[0]).sort()).toEqual(['mode', 'token']);
|
||||
});
|
||||
@@ -1376,61 +1376,6 @@ async function expectStar(button, favorite) {
|
||||
await expect(button).toHaveAttribute('aria-pressed', String(favorite));
|
||||
}
|
||||
|
||||
// #494: the Favorites count is re-read 300 ms after a star changes. A page that has begun to leave starts no count read
|
||||
// (WebKit fails it with "Fetch API cannot load … due to access control checks.", reported as a page error; queue run
|
||||
// 40934368716). A leave that is called off counts again on the next input.
|
||||
for (const leave of ['beforeunload', 'pagehide']) {
|
||||
test(`a star changed just before ${leave} starts no Favorites count read while the page leaves`, async ({ page }) => {
|
||||
await page.clock.install();
|
||||
const countReads = [];
|
||||
page.on('request', item => {
|
||||
const url = new URL(item.url());
|
||||
if (url.pathname === '/prompt_manager/images/output' && url.searchParams.get('favorites') === '1' && url.searchParams.get('limit') === '1') countReads.push(url.search);
|
||||
});
|
||||
await page.goto(workspace);
|
||||
const card = page.locator('article[data-image-key="0:shared/same.png"]');
|
||||
await expectStar(star(card, 'same.png'), false);
|
||||
await expect.poll(() => countReads.length).toBe(1);
|
||||
await star(card, 'same.png').click();
|
||||
await expectStar(star(card, 'same.png'), true);
|
||||
await expect(toast(page)).toHaveText('Added to favorites.');
|
||||
await page.evaluate(type => window.dispatchEvent(new Event(type)), leave);
|
||||
await page.clock.runFor(1000);
|
||||
// A read made after the clock ran is listed, so the listener saw every count read.
|
||||
await page.evaluate(() => fetch('/prompt_manager/images/output?mode=workspace&favorites=1&limit=1&probe=1'));
|
||||
await expect.poll(() => countReads.length).toBe(2);
|
||||
expect(countReads.at(-1)).toContain('probe=1');
|
||||
if (leave === 'beforeunload') {
|
||||
// The leave was called off (the page is still here): the next input counts again, once, and the badge shows it.
|
||||
await page.keyboard.press('Shift');
|
||||
await expect.poll(() => countReads.length).toBe(3);
|
||||
expect(countReads.at(-1)).not.toContain('probe=1');
|
||||
await expect(page.locator('nav [data-route="favorites"] .nav-count')).toHaveText('1');
|
||||
await page.clock.runFor(1000);
|
||||
expect(countReads).toHaveLength(3);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('a count read in flight when the page starts to leave is dropped, then read again if the page stays (#494)', async ({ page }) => {
|
||||
const held = [];
|
||||
// The page's first Library count read is held at the server boundary while the page starts to leave.
|
||||
await page.route('**/prompt_manager/library?limit=1', route => { held.push(route); });
|
||||
await page.goto(workspace);
|
||||
await expect.poll(() => held.length).toBe(1);
|
||||
const failed = page.waitForEvent('requestfailed', item => item.url().includes('/prompt_manager/library?limit=1'));
|
||||
await page.evaluate(() => window.dispatchEvent(new Event('beforeunload')));
|
||||
await failed;
|
||||
await page.unroute('**/prompt_manager/library?limit=1');
|
||||
await held[0].continue().catch(() => {});
|
||||
await expect(page.locator('nav [data-route="library"] .nav-count')).toHaveCount(0);
|
||||
// The page stayed: the next input reads the dropped count again.
|
||||
const reread = page.waitForRequest(item => item.url().includes('/prompt_manager/library?limit=1'));
|
||||
await page.keyboard.press('Shift');
|
||||
await reread;
|
||||
await expect(page.locator('nav [data-route="library"] .nav-count')).toHaveText(/^\d+$/);
|
||||
});
|
||||
|
||||
test('favorite persists across reload', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/reset', { data: { include_video: true } })).toBeOK();
|
||||
const sent = [];
|
||||
@@ -1849,7 +1794,7 @@ test('card checkboxes select images into the concept bar and Clear empties it',
|
||||
await checkbox(first, 'same.png').check();
|
||||
await expect(bulkbar(page).locator('.bulkbar > strong')).toHaveText('1 selected');
|
||||
// D85: Add tags (the concept's) and Auto-tag selected lead the real actions.
|
||||
await expect(barButtons(page)).toHaveText(['Add tags', 'Auto-tag selected', 'Set category', 'Add to collection', 'Export metadata', 'Delete…', 'Clear']);
|
||||
await expect(barButtons(page)).toHaveText(['Add tags', 'Auto-tag selected', 'Set category', 'Add to collection', 'Export metadata', 'Clear']);
|
||||
// Toggling neither opens the viewer, nor inspects, nor moves focus.
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
await expect(checkbox(first, 'same.png')).toBeFocused();
|
||||
|
||||
@@ -704,147 +704,6 @@ test('a video replaced while still starting is reset once it settles', async ({
|
||||
}
|
||||
});
|
||||
|
||||
test('a new selection while the viewer\'s own video is still starting leaves it until it settles (#483)', async ({ context, browserName }) => {
|
||||
test.setTimeout(180000);
|
||||
// D155 opens the viewer once the inline preview has settled; the viewer then starts a video of its own. A new
|
||||
// selection from that video's own `loadstart` (readyState HAVE_NOTHING), or a few milliseconds after it, must not
|
||||
// reset it or take it off the page while it starts (WebKit, GStreamer: TESTING.md D62; merge-queue run
|
||||
// 40903238387). The viewer stays open until it reports metadata or an error, then closes; the URL goes last.
|
||||
for (let attempt = 0; attempt < 10; attempt++) {
|
||||
const page = await context.newPage();
|
||||
try {
|
||||
if (attempt === 0) await startCoverage(page, browserName);
|
||||
await page.addInitScript(() => {
|
||||
const load = HTMLMediaElement.prototype.load;
|
||||
HTMLMediaElement.prototype.load = function () { if (!this.hasAttribute('src')) { this.__resets = (this.__resets ?? 0) + 1; this.__resetReadyState ??= this.readyState; } return load.call(this); };
|
||||
const revoke = URL.revokeObjectURL;
|
||||
URL.revokeObjectURL = url => {
|
||||
const starting = window.__starting;
|
||||
if (url === starting?.url) starting.revoked = { src: starting.node.hasAttribute('src'), connected: starting.node.isConnected, resets: starting.node.__resets ?? 0 };
|
||||
return revoke(url);
|
||||
};
|
||||
});
|
||||
await openInspect(page);
|
||||
await page.evaluate(() => {
|
||||
const input = document.querySelector('#local-file');
|
||||
new MutationObserver((records, observer) => {
|
||||
const video = document.querySelector('#viewer-dialog video');
|
||||
if (!video) return;
|
||||
observer.disconnect();
|
||||
const replace = () => {
|
||||
window.__starting = { node: video, url: video.getAttribute('src'), readyState: video.readyState };
|
||||
input.files = window.__pngFiles; input.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
window.__starting.replaced = { connected: video.isConnected, src: video.hasAttribute('src'), resets: video.__resets ?? 0 };
|
||||
};
|
||||
video.addEventListener('loadstart', () => { if (window.__delay) setTimeout(replace, window.__delay); else replace(); }, { once: true });
|
||||
}).observe(document.body, { childList: true, subtree: true });
|
||||
});
|
||||
const png = parityPng('f02.png');
|
||||
await page.evaluate(bytes => {
|
||||
const transfer = new DataTransfer();
|
||||
transfer.items.add(new File([new Uint8Array(bytes)], 'f02.png', { type: 'image/png' }));
|
||||
window.__pngFiles = transfer.files;
|
||||
}, [...png.buffer]);
|
||||
await page.evaluate(delay => { window.__delay = delay; }, attempt % 5);
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible({ timeout: 8000 });
|
||||
await expect.poll(() => page.evaluate(() => window.__starting?.revoked ?? null)).toEqual({ src: false, connected: false, resets: 1 });
|
||||
const { readyState, replaced } = await page.evaluate(() => ({ readyState: window.__starting.readyState, replaced: window.__starting.replaced }));
|
||||
if (attempt % 5 === 0) expect(readyState).toBe(0);
|
||||
// Starting when replaced: still on the page with its URL, never reset; settled: released at once.
|
||||
expect(replaced).toEqual(readyState === 0 ? { connected: true, src: true, resets: 0 } : { connected: false, src: false, resets: 1 });
|
||||
expect(await page.evaluate(() => window.__starting.node.__resets)).toBe(1);
|
||||
// Its one reset came after it had metadata, never while it was starting.
|
||||
expect(await page.evaluate(() => window.__starting.node.__resetReadyState)).toBeGreaterThanOrEqual(1);
|
||||
// The new file's viewer opens after the old one closed, and the page still answers.
|
||||
await expect(viewer(page).getByRole('img', { name: 'f02.png', exact: true })).toBeVisible({ timeout: 8000 });
|
||||
await expect(viewer(page).locator('video')).toHaveCount(0);
|
||||
if (attempt === 0) await saveCoverage(page, browserName);
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
/** #483: holds the viewer's video in HAVE_NOTHING (its `loadedmetadata` never fires) so a hold can be driven by hand. */
|
||||
async function stallViewerVideo(page) {
|
||||
await page.addInitScript(() => {
|
||||
const describe = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'readyState');
|
||||
Object.defineProperty(HTMLMediaElement.prototype, 'readyState', { configurable: true, get() {
|
||||
return this.closest?.('#viewer-dialog') && window.__stallViewer ? 0 : describe.get.call(this);
|
||||
} });
|
||||
window.addEventListener('loadedmetadata', event => {
|
||||
if (window.__stallViewer && event.target.closest?.('#viewer-dialog')) event.stopImmediatePropagation();
|
||||
}, true);
|
||||
});
|
||||
}
|
||||
|
||||
test('a held viewer closed by hand, or replaced twice, opens only the latest file\'s viewer at once (#483)', async ({ page }) => {
|
||||
await stallViewerVideo(page);
|
||||
await openInspect(page);
|
||||
await page.evaluate(() => { window.__stallViewer = true; });
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
// Two selections while the viewer's video is (made to look) still starting: the viewer stays, untouched.
|
||||
await chooser(page).setInputFiles(parityPng('f02.png', 'first.png'));
|
||||
await chooser(page).setInputFiles(parityPng('f02.png', 'second.png'));
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
// Open file metadata pressed meanwhile (the page behind the viewer is inert, so by script): the latest request.
|
||||
await action(page).evaluate(node => node.click());
|
||||
// Closing it by hand ends the hold at once: the latest file's viewer opens, once, well before D62's 15 s.
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(viewer(page).getByRole('img', { name: 'second.png', exact: true })).toBeVisible({ timeout: 5000 });
|
||||
await expect(viewer(page).getByRole('img', { name: 'first.png', exact: true })).toHaveCount(0);
|
||||
await expect(viewer(page).locator('video')).toHaveCount(0);
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(1);
|
||||
});
|
||||
|
||||
test('a held viewer revokes its URL only once it is gone, drops a stale open, and ends on page leave (#483)', async ({ page }) => {
|
||||
await stallViewerVideo(page);
|
||||
await openInspect(page);
|
||||
await trackReleases(page);
|
||||
await page.evaluate(() => { window.__stallViewer = true; });
|
||||
const videoUrl = () => page.evaluate(() => document.querySelector('#viewer-dialog video')?.getAttribute('src') ?? null);
|
||||
const released = url => page.evaluate(url => window.__released.filter(item => item.url === url), url);
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
const first = await videoUrl();
|
||||
await chooser(page).setInputFiles(parityPng('f02.png', 'held.png'));
|
||||
await expect(preview(page).locator('img.local-file-preview')).toBeVisible();
|
||||
// The session asked to revoke the video's URL; it waits while the held viewer still plays it.
|
||||
expect(await released(first)).toEqual([]);
|
||||
// An Open asked for during the hold, then an AVI (which never opens the viewer by itself): the open for the earlier
|
||||
// file is stale and is dropped when the hold ends.
|
||||
await action(page).evaluate(node => node.click());
|
||||
const avi = { name: 'clip.avi', mimeType: 'video/x-msvideo', buffer: Buffer.concat([Buffer.from('RIFF'), Buffer.alloc(4), Buffer.from('AVI LIST'), Buffer.alloc(64)]) };
|
||||
await chooser(page).setInputFiles(avi);
|
||||
await expect(page.locator('#workspace-toast')).toHaveText(TOAST);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
await expect.poll(() => released(first)).toEqual([{ url: first, live: 0 }]);
|
||||
// Page leave while held: the viewer goes and the deferred URL is revoked with nothing holding it.
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
const second = await videoUrl();
|
||||
await chooser(page).setInputFiles(parityPng('f02.png', 'leave.png'));
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
expect(await released(second)).toEqual([]);
|
||||
await page.evaluate(() => window.dispatchEvent(new PageTransitionEvent('pagehide', { persisted: true })));
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
await expect.poll(() => released(second)).toEqual([{ url: second, live: 0 }]);
|
||||
// Another route while held: the page's dispose ends the hold and revokes the URL with nothing holding it.
|
||||
await page.evaluate(() => window.dispatchEvent(new PageTransitionEvent('pageshow', { persisted: true })));
|
||||
await choose(page, localFixture('core-tags.webm', 'video/webm'));
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
const third = await videoUrl();
|
||||
await chooser(page).setInputFiles(parityPng('f02.png', 'route.png'));
|
||||
await expect(viewer(page).locator('video')).toBeAttached();
|
||||
await page.evaluate(() => { window.location.hash = '#/library'; });
|
||||
await expect(main(page).getByRole('heading', { level: 1, name: 'Prompt library', exact: true })).toBeVisible();
|
||||
await expect.poll(() => released(third)).toEqual([{ url: third, live: 0 }]);
|
||||
});
|
||||
|
||||
test('leaving for another route while a video starts waits for it to settle', async ({ context, browserName }) => {
|
||||
test.setTimeout(180000);
|
||||
// A WebKit (GStreamer) video detached before its pipeline had started deadlocked the page (TESTING.md D62); a
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
|
||||
|
||||
// #443 D152: Delete… on the Library bar deletes unpaired prompts, archives paired ones (restorable), keeps protected ones.
|
||||
const route='/prompt_manager/workspace#/library';
|
||||
const PREVIEW='/prompt_manager/library/delete/preview';
|
||||
const DELETE='/prompt_manager/library/delete';
|
||||
const cards=page=>page.locator('article[data-prompt-id]');
|
||||
const card=(page,id)=>page.locator(`article[data-prompt-id="${id}"]`);
|
||||
const box=(page,id)=>card(page,id).locator('.image-overlays input[type="checkbox"]');
|
||||
const bar=page=>page.locator('#library-bulkbar > .bulkbar');
|
||||
const toast=page=>page.locator('.toast');
|
||||
const review=page=>page.getByRole('dialog');
|
||||
const showArchived=page=>page.locator('#workspace-main .heading-actions').getByRole('button',{name:/^Show archived prompts/});
|
||||
|
||||
async function items(request,shelf='active') {
|
||||
const query=shelf==='archived'?'&shelf=archived':'';
|
||||
return (await(await request.get(`/prompt_manager/library?limit=100${query}`)).json()).items;
|
||||
}
|
||||
const byText=(list,text)=>list.find(item=>item.text===text);
|
||||
function recordPosts(page,path) {
|
||||
const posts=[];
|
||||
page.on('request',item=>{if(item.method()==='POST'&&new URL(item.url()).pathname===path)posts.push(item.postDataJSON());});
|
||||
return posts;
|
||||
}
|
||||
async function openReview(page) {
|
||||
await bar(page).getByRole('button',{name:'Delete…',exact:true}).click();
|
||||
await expect(review(page)).toBeVisible();
|
||||
}
|
||||
|
||||
test.beforeEach(async({page,request,browserName})=>{
|
||||
await startCoverage(page,browserName);await expect(await request.post('/__test__/reset',{data:{native_library:true,extra_prompts:2}})).toBeOK();
|
||||
page.libraryErrors=[];page.on('pageerror',error=>page.libraryErrors.push(error.message));
|
||||
});
|
||||
test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.libraryErrors).toEqual([]);});
|
||||
|
||||
test('Delete… reviews the selection, deletes the unpaired, archives the paired and keeps the protected',async({page,request})=>{
|
||||
const before=await items(request);
|
||||
const dog=byText(before,'Dog landscape'),cat=byText(before,'Cat portrait'),guarded=byText(before,'Library extra 0000');
|
||||
await expect(await request.post('/prompt_manager/bulk/tags',{data:{prompt_ids:[guarded.id],tags:['__protected__']}})).toBeOK();
|
||||
const previews=recordPosts(page,PREVIEW),writes=recordPosts(page,DELETE);
|
||||
await page.goto(route);await expect(cards(page)).toHaveCount(before.length);
|
||||
for(const item of [dog,cat,guarded]) await box(page,item.id).check();
|
||||
await expect(bar(page).getByRole('button')).toHaveText([`Select all ${before.length} loaded`,'Add tags','Set category','Delete…','Clear']);
|
||||
await expect(bar(page).getByRole('button',{name:'Delete…',exact:true})).toHaveClass(/danger/);
|
||||
await openReview(page);
|
||||
await expect(review(page)).toHaveAccessibleName('Delete or archive selected prompts');
|
||||
const grid=review(page).locator('dl.meta-grid > div');
|
||||
await expect(grid.locator('dt')).toHaveText(['Delete permanently','Archive','Protected (kept)','Image links removed','Backup']);
|
||||
await expect(grid.locator('dd')).toHaveText(['1 prompt not in a saved pair','1 prompt in a saved pair: hidden, restorable',
|
||||
'1: Library extra 0000','0 images will no longer show these prompts (files stay on disk)','Create and verify before deleting']);
|
||||
await expect(review(page).locator('p.small')).toHaveCount(3);
|
||||
await expect(review(page).getByRole('button',{name:'Delete 1 and archive 1 prompt',exact:true})).toHaveClass(/danger/);
|
||||
// Cancel writes nothing and releases the bar.
|
||||
await review(page).getByRole('button',{name:'Cancel',exact:true}).click();
|
||||
await expect(review(page)).toHaveCount(0);expect(writes).toEqual([]);
|
||||
await expect(bar(page).getByRole('button',{name:'Delete…',exact:true})).not.toHaveAttribute('aria-disabled','true');
|
||||
await openReview(page);
|
||||
await review(page).getByRole('button',{name:'Delete 1 and archive 1 prompt',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText(/^Deleted 1 prompt\. Archived 1 prompt\. 1 protected prompt was kept\. Safety backup saved: \S+\.backup_\S+ \(System › Backup & export\)\.$/);
|
||||
await expect(page.locator('#library-bulkbar')).toBeEmpty();
|
||||
await expect(card(page,dog.id)).toHaveCount(0);await expect(card(page,cat.id)).toHaveCount(0);await expect(card(page,guarded.id)).toHaveCount(1);
|
||||
const epoch=(await(await request.get('/prompt_manager/library/options')).json()).library_epoch;
|
||||
// One preview per review opened (Cancel, then Delete…).
|
||||
const reviewed={prompt_ids:[dog.id,cat.id,guarded.id],expected_epoch:epoch};
|
||||
expect(previews).toEqual([reviewed,reviewed]);
|
||||
expect(writes).toEqual([{archive_ids:[cat.id],delete_ids:[dog.id],expected_epoch:epoch,backup:'on'}]);
|
||||
const after=await items(request);
|
||||
expect(byText(after,'Dog landscape')).toBeUndefined();expect(byText(after,'Cat portrait')).toBeUndefined();
|
||||
expect((await items(request,'archived')).map(item=>item.id)).toEqual([cat.id]);
|
||||
await expect(showArchived(page)).toHaveText('Show archived prompts (1)');
|
||||
});
|
||||
|
||||
test('Show archived prompts lists archived prompts, and Restore on its bar returns them',async({page,request})=>{
|
||||
const before=await items(request);const cat=byText(before,'Cat portrait'),dual=byText(before,'Dual role portrait');
|
||||
await page.goto(route);await expect(cards(page)).toHaveCount(before.length);
|
||||
await box(page,cat.id).check();await box(page,dual.id).check();
|
||||
await openReview(page);
|
||||
await expect(review(page)).toHaveAccessibleName('Archive selected prompts');
|
||||
// The archive reason is the saved pair, for one or many prompts (live note on #470): never negative variants.
|
||||
await expect(review(page).locator('dl.meta-grid > div').filter({hasText:'Archive'}).locator('dd')).toHaveText('2 prompts in a saved pair: hidden, restorable');
|
||||
await review(page).getByRole('button',{name:'Archive 2 prompts',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('Archived 2 prompts.');
|
||||
await expect(cards(page)).toHaveCount(before.length-2);
|
||||
await expect(showArchived(page)).toHaveAttribute('aria-pressed','false');
|
||||
await showArchived(page).click();
|
||||
await expect(showArchived(page)).toHaveAttribute('aria-pressed','true');
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
await expect(page.locator('.section-caption > span').first()).toHaveText(/^2 archived prompts · \d+ negative variants? hidden with them$/);
|
||||
await expect(card(page,cat.id).locator('.card-title .chip',{hasText:'Archived'})).toHaveCount(1);
|
||||
await box(page,cat.id).check();
|
||||
await expect(bar(page).getByRole('button')).toHaveText(['Select all 2 loaded','Restore','Clear']);
|
||||
await bar(page).getByRole('button',{name:'Restore',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('Restored 1 prompt to the Prompt library.');
|
||||
await expect(cards(page)).toHaveCount(1);
|
||||
await showArchived(page).click();
|
||||
await expect(cards(page)).toHaveCount(before.length-1);
|
||||
await expect(card(page,cat.id)).toHaveCount(1);
|
||||
expect((await items(request,'archived')).map(item=>item.id)).toEqual([dual.id]);
|
||||
// An empty archived view says so.
|
||||
await showArchived(page).click();await box(page,dual.id).check();
|
||||
await bar(page).getByRole('button',{name:'Restore',exact:true}).click();
|
||||
await expect(page.locator('.library-status')).toHaveText('No archived prompts.');
|
||||
});
|
||||
|
||||
test('an archived prompt\'s page says so and restores it from its notice',async({page,request})=>{
|
||||
const cat=byText(await items(request),'Cat portrait');
|
||||
const epoch=(await(await request.get('/prompt_manager/library/options')).json()).library_epoch;
|
||||
await expect(await request.post(DELETE,{data:{archive_ids:[cat.id],delete_ids:[],expected_epoch:epoch,backup:'off'}})).toBeOK();
|
||||
await page.goto(`/prompt_manager/workspace#/library/${cat.id}`);
|
||||
const notice=page.locator('.library-archived-notice');
|
||||
await expect(notice.locator('h2')).toHaveText('Archived');
|
||||
// The notice says why (the saved pair) and counts the prompt's negative variants (live note on #470).
|
||||
expect(cat.variant_count).toBeGreaterThan(1);
|
||||
await expect(notice.locator('p').first()).toHaveText(`This prompt is archived because it is in a saved pair. It and its ${cat.variant_count} negative variants are hidden from the Prompt library; nothing was removed.`);
|
||||
await notice.getByRole('button',{name:'Restore this prompt to the Prompt library'}).click();
|
||||
await expect(toast(page)).toHaveText('Restored 1 prompt to the Prompt library.');
|
||||
await expect(notice).toHaveCount(0);
|
||||
expect(byText(await items(request),'Cat portrait')?.id).toBe(cat.id);
|
||||
});
|
||||
|
||||
// #443: a starved renderer (headless WebKit under CPU load, a background tab) can hold animation frames for
|
||||
// seconds; a loaded page of prompts is shown without waiting for one, on first load and after a delete.
|
||||
async function holdAnimationFrames(page) {
|
||||
await page.addInitScript(()=>{
|
||||
const request=window.requestAnimationFrame.bind(window),cancel=window.cancelAnimationFrame.bind(window);
|
||||
const held=new Map(),real=new Map();let holding=true,next=0;
|
||||
window.requestAnimationFrame=callback=>{const id=--next;if(holding)held.set(id,callback);else real.set(id,request(callback));return id;};
|
||||
window.cancelAnimationFrame=id=>{held.delete(id);if(real.has(id))cancel(real.get(id));real.delete(id);};
|
||||
window.__releaseAnimationFrames=()=>{holding=false;for(const [id,callback] of held)real.set(id,request(callback));held.clear();};
|
||||
});
|
||||
}
|
||||
test('a held animation frame does not hide the loaded prompts, on first load or after a delete',async({page,request})=>{
|
||||
const before=await items(request);const dog=byText(before,'Dog landscape');
|
||||
await holdAnimationFrames(page);
|
||||
await page.goto(route);await expect(cards(page)).toHaveCount(before.length);
|
||||
await box(page,dog.id).check();await openReview(page);
|
||||
await review(page).getByRole('button',{name:'Delete 1 prompt',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText(/^Deleted 1 prompt\./);
|
||||
await expect(cards(page)).toHaveCount(before.length-1);await expect(card(page,dog.id)).toHaveCount(0);
|
||||
await page.evaluate(()=>window.__releaseAnimationFrames());
|
||||
await expect(cards(page)).toHaveCount(before.length-1);
|
||||
});
|
||||
|
||||
test('a double press sends one preview, and the bar holds while it is reviewed',async({page,request})=>{
|
||||
const dog=byText(await items(request),'Dog landscape');
|
||||
const previews=recordPosts(page,PREVIEW);
|
||||
let release;const held=new Promise(resolve=>{release=resolve;});
|
||||
await page.route(`**${PREVIEW}`,async item=>{await held;await item.fallback();});
|
||||
await page.goto(route);await box(page,dog.id).check();
|
||||
const remove=bar(page).getByRole('button',{name:/^(Delete…|Reviewing…)$/});
|
||||
await remove.click();
|
||||
await expect(remove).toHaveText('Reviewing…');await expect(remove).toHaveAttribute('aria-disabled','true');
|
||||
await expect(bar(page).getByRole('button',{name:'Clear',exact:true})).toHaveAttribute('aria-disabled','true');
|
||||
await remove.click({force:true});
|
||||
release();
|
||||
await expect(review(page)).toHaveAccessibleName('Delete selected prompts');
|
||||
expect(previews).toHaveLength(1);
|
||||
await review(page).getByRole('button',{name:'Cancel',exact:true}).click();
|
||||
await expect(bar(page).getByRole('button',{name:'Delete…',exact:true})).not.toHaveAttribute('aria-disabled','true');
|
||||
});
|
||||
|
||||
test('a restore since the page loaded, a failed backup, a busy job and an unconfirmed answer each say what happened',async({page,request})=>{
|
||||
const before=await items(request);const dog=byText(before,'Dog landscape');
|
||||
await armUnseenRestores(page);
|
||||
await page.goto(route);await expect(cards(page)).toHaveCount(before.length);
|
||||
// Failed backup: nothing changed, the selection stays.
|
||||
await page.route(`**${DELETE}`,item=>item.fulfill({status:500,contentType:'application/json',
|
||||
body:JSON.stringify({success:false,code:'backup-failed',error:'The safety backup could not be taken and verified; nothing was changed.'})}));
|
||||
await box(page,dog.id).check();await openReview(page);
|
||||
await review(page).getByRole('button',{name:'Delete 1 prompt',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('The safety backup failed. The safety backup could not be taken and verified; nothing was changed.');
|
||||
await expect(box(page,dog.id)).toBeChecked();
|
||||
// Busy: names the job, nothing changed.
|
||||
await page.unroute(`**${DELETE}`);
|
||||
await page.route(`**${DELETE}`,item=>item.fulfill({status:409,contentType:'application/json',
|
||||
body:JSON.stringify({success:false,code:'busy',error:'busy',running:{id:'abc',title:'Scan'}})}));
|
||||
await openReview(page);await review(page).getByRole('button',{name:'Delete 1 prompt',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('Scan is running. Nothing was changed; try again when it finishes.');
|
||||
// Unconfirmed: the shelf reloads.
|
||||
await page.unroute(`**${DELETE}`);
|
||||
await page.route(`**${DELETE}`,item=>item.abort());
|
||||
await openReview(page);await review(page).getByRole('button',{name:'Delete 1 prompt',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('The server did not confirm the change. The Library was reloaded: check the shelf and Show archived prompts.');
|
||||
await expect(page.locator('#library-bulkbar')).toBeEmpty();
|
||||
await page.unroute(`**${DELETE}`);
|
||||
// A restore the page has not seen: the review opened before it is refused on the server.
|
||||
await box(page,dog.id).check();await openReview(page);
|
||||
const backup=await request.get('/prompt_manager/backup');await expect(backup).toBeOK();const buffer=await backup.body();
|
||||
const restore=await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'library.db',mimeType:'application/octet-stream',buffer}}}));
|
||||
await expect(restore).toBeOK();
|
||||
await review(page).getByRole('button',{name:'Delete 1 prompt',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('The library was restored or moved since this page loaded. Nothing was deleted or archived.');
|
||||
expect(byText(await items(request),'Dog landscape')).toBeDefined();
|
||||
});
|
||||
|
||||
test('a failed review keeps the selection, and a selection nothing can change opens the close-only review',async({page,request})=>{
|
||||
const before=await items(request);const guarded=byText(before,'Library extra 0000');
|
||||
await expect(await request.post('/prompt_manager/bulk/tags',{data:{prompt_ids:[guarded.id],tags:['__protected__']}})).toBeOK();
|
||||
await page.goto(route);await expect(cards(page)).toHaveCount(before.length);
|
||||
await box(page,guarded.id).check();
|
||||
await page.route(`**${PREVIEW}`,item=>item.fulfill({status:500,contentType:'application/json',body:JSON.stringify({success:false,code:'review-failed',error:'x'})}),{times:1});
|
||||
await bar(page).getByRole('button',{name:'Delete…',exact:true}).click();
|
||||
await expect(toast(page)).toHaveText('The selected prompts could not be reviewed. Try again.');
|
||||
await expect(box(page,guarded.id)).toBeChecked();
|
||||
await expect(bar(page).getByRole('button',{name:'Delete…',exact:true})).not.toHaveAttribute('aria-disabled','true');
|
||||
await openReview(page);
|
||||
await expect(review(page)).toHaveAccessibleName('Delete selected prompts');
|
||||
await expect(review(page).locator('p.small').last()).toHaveText('None of the selected prompts can be deleted or archived.');
|
||||
await expect(review(page).locator('.dialog-actions > button')).toHaveText(['Close']);
|
||||
await review(page).getByRole('button',{name:'Close',exact:true}).click();
|
||||
await expect(bar(page).getByRole('button',{name:'Delete…',exact:true})).not.toHaveAttribute('aria-disabled','true');
|
||||
expect(byText(await items(request),'Library extra 0000')).toBeDefined();
|
||||
});
|
||||
@@ -493,15 +493,13 @@ test('a restore seen finishing keeps New prompt open and reloads the Library aft
|
||||
test('library heading offers Export then + New prompt, and New opens the saved prompt history',async({page,request})=>{
|
||||
await page.goto(`${workspace}#/library`);await expect(page.locator('article[data-prompt-id]').first()).toBeVisible();
|
||||
const actions=headingActions(page).locator('button');
|
||||
// D152 (#443): Show archived prompts (N) comes first, a plain toggle.
|
||||
expect(await visibleNames(actions)).toEqual(['Show archived prompts (0)','Export','+ New prompt']);
|
||||
await expect(actions.nth(0)).toHaveAttribute('aria-pressed','false');await expect(actions.nth(0)).not.toHaveClass(/ca-primary/);
|
||||
expect(await visibleNames(actions)).toEqual(['Export','+ New prompt']);
|
||||
// One image is counted in the singular.
|
||||
const f=page.fixture;await expect(page.locator(`article[data-prompt-id="${f.positive_id}"]`).getByText('1 image',{exact:true})).toHaveCount(1);
|
||||
await expect(actions.nth(1)).toHaveAccessibleName('Export full Library JSON');await expect(actions.nth(1)).not.toHaveClass(/ca-primary/);
|
||||
await expect(actions.nth(2)).toHaveClass(/\bca-primary\b/);
|
||||
for(const index of [0,1,2])expect((await actions.nth(index).boundingBox()).height).toBeGreaterThanOrEqual(32);
|
||||
await actions.nth(2).click();const dialog=editorDialog(page);await expect(dialog.getByRole('heading',{name:'New prompt',exact:true})).toBeVisible();await ready(dialog);
|
||||
await expect(actions.nth(0)).toHaveAccessibleName('Export full Library JSON');await expect(actions.nth(0)).not.toHaveClass(/ca-primary/);
|
||||
await expect(actions.nth(1)).toHaveClass(/\bca-primary\b/);
|
||||
for(const index of [0,1])expect((await actions.nth(index).boundingBox()).height).toBeGreaterThanOrEqual(32);
|
||||
await actions.nth(1).click();const dialog=editorDialog(page);await expect(dialog.getByRole('heading',{name:'New prompt',exact:true})).toBeVisible();await ready(dialog);
|
||||
await field(dialog,'Title').fill('Harbor at dawn');await field(dialog,'Positive prompt').fill('A quiet harbor at dawn');await field(dialog,'Negative prompt (optional)').fill('fog, noise');
|
||||
await button(dialog,'Save').click();await expect(dialog).toHaveCount(0);
|
||||
const created=(await(await request.get('/prompt_manager/library?text=quiet%20harbor')).json()).items[0];
|
||||
@@ -635,11 +633,7 @@ test('restored storage after a confirmed save or before an archive offers an exp
|
||||
const response=await route.fetch();await expect(await restore()).toBeOK();return route.fulfill({response});
|
||||
});
|
||||
await page.getByRole('button',{name:'Edit metadata',exact:true}).click();let dialog=editorDialog(page);await ready(dialog);
|
||||
await field(dialog,'Notes').fill('Lost to a restore');
|
||||
// The save's answer waits for a whole restore in the route above; wait for that answer, not the 5 s expect
|
||||
// default (#501 merge queue: under load the restore outlasted it on WebKit).
|
||||
const saved=page.waitForResponse(item=>item.request().method()==='PATCH'&&new URL(item.url()).pathname===`/prompt_manager/library/${f.positive_id}`);
|
||||
await button(dialog,'Save').click();await saved;
|
||||
await field(dialog,'Notes').fill('Lost to a restore');await button(dialog,'Save').click();
|
||||
await expect(note(dialog)).toHaveText('Saved; Library refresh failed because prompt storage was restored. Reload the Library to see current data.');
|
||||
expect(await actionNames(dialog)).toEqual(['Reload Library','Create text revision','Archive…','Cancel','Save']);
|
||||
for(const name of ['Create text revision','Archive…'])await expect(button(dialog,name)).toBeDisabled();
|
||||
|
||||
@@ -62,8 +62,7 @@ test('Set category writes the selected prompts on the server, reloads the shelf
|
||||
await box(page,first.id).check();await expect(bar(page).locator('strong')).toHaveText('1 selected');
|
||||
await box(page,second.id).check();await expect(bar(page).locator('strong')).toHaveText('2 selected');
|
||||
// Two of three loaded are checked, so Select all N loaded follows the count (D151, #444 ruling).
|
||||
// D152 (#443) adds Delete… before Clear.
|
||||
await expect(bar(page).getByRole('button')).toHaveText([`Select all ${items.length} loaded`,'Add tags','Set category','Delete…','Clear']);
|
||||
await expect(bar(page).getByRole('button')).toHaveText([`Select all ${items.length} loaded`,'Add tags','Set category','Clear']);
|
||||
await setCategory(page,' Studio set ');
|
||||
await expect(toast(page)).toHaveText('Category “Studio set” set on 2 prompts.');
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
@@ -288,7 +287,7 @@ test('Ctrl/Cmd+A and Select all N loaded select every loaded prompt, windowed on
|
||||
let release;const held=new Promise(resolve=>{release=resolve;});
|
||||
await page.route('**/prompt_manager/bulk/tags',async route=>{await held;await route.continue();},{times:1});
|
||||
await addTags(page,'everyone');
|
||||
for(const name of ['Add tags','Set category','Delete…','Clear',`Select all ${all} loaded`]) await expect(bar(page).getByRole('button',{name,exact:true})).toHaveAttribute('aria-disabled','true');
|
||||
for(const name of ['Add tags','Set category',`Select all ${all} loaded`]) await expect(bar(page).getByRole('button',{name,exact:true})).toHaveAttribute('aria-disabled','true');
|
||||
release();
|
||||
await expect(toast(page)).toHaveText(`Added “everyone” to ${all-1} prompts.`);
|
||||
expect(writes).toHaveLength(1);expect(new Set(writes[0].prompt_ids).size).toBe(all-1);
|
||||
|
||||
@@ -37,15 +37,6 @@ test.afterEach(async ({ page, request, browserName }) => {
|
||||
await expect(await request.post('/__test__/tagging/reset', { data: {} })).toBeOK();
|
||||
});
|
||||
|
||||
async function openSettled(page, url) {
|
||||
// The page reads its selected file's details as soon as it lists. A source change before that read is answered
|
||||
// makes the page find the file again on its own, so the test's old-numbering locators are gone before they are
|
||||
// used (#484): every test changes sources only after the read was answered.
|
||||
const inspected = page.waitForResponse(item => new URL(item.url()).pathname.startsWith('/prompt_manager/images/inspect/') &&
|
||||
item.request().method() === 'GET');
|
||||
await page.goto(url);
|
||||
expect((await inspected).status()).toBe(200);
|
||||
}
|
||||
async function openTag(page, path = FIRST, root = 0) {
|
||||
const observed = answered(page, '/image-annotations/observe');
|
||||
await tagButton(page, path, root).click();
|
||||
@@ -62,7 +53,7 @@ async function savedTags(request, path, root) {
|
||||
}
|
||||
|
||||
test('renumbered sources under an open Tag dialog: it finds the file again, keeps the draft and Generate works', async ({ page, request }) => {
|
||||
await openSettled(page, images);
|
||||
await page.goto(images);
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
expect((await openTag(page)).status()).toBe(200);
|
||||
await manual(page).fill('kept_draft');
|
||||
@@ -100,7 +91,7 @@ test('renumbered sources under an open Tag dialog: it finds the file again, keep
|
||||
});
|
||||
|
||||
test('the inspector finds a renumbered file again; checked files from the old numbering are cleared', async ({ page, request }) => {
|
||||
await openSettled(page, images);
|
||||
await page.goto(images);
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
await card(page, FIRST).getByRole('checkbox', { name: /^Select / }).check();
|
||||
await expect(page.locator('.bulkbar')).toBeVisible();
|
||||
@@ -122,7 +113,7 @@ test('the inspector finds a renumbered file again; checked files from the old nu
|
||||
});
|
||||
|
||||
test('a scope that cannot be mapped: Generate and Save are visibly disabled, the reason and Refresh sit beside them', async ({ page }) => {
|
||||
await openSettled(page, images);
|
||||
await page.goto(images);
|
||||
await expect(card(page, FIRST)).toBeVisible();
|
||||
await expect(await page.request.post('/__test__/scope-key/rotate')).toBeOK();
|
||||
const rescoped = answered(page, '/images/rescope');
|
||||
@@ -166,7 +157,7 @@ test('a scope that cannot be mapped: Generate and Save are visibly disabled, the
|
||||
});
|
||||
|
||||
test('an unmappable inspector read keeps its visible Refresh, and Refresh recovers the panel', async ({ page }) => {
|
||||
await openSettled(page, images);
|
||||
await page.goto(images);
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
await expect(await page.request.post('/__test__/scope-key/rotate')).toBeOK();
|
||||
const rescoped = answered(page, '/images/rescope');
|
||||
@@ -182,7 +173,7 @@ test('an unmappable inspector read keeps its visible Refresh, and Refresh recove
|
||||
|
||||
test('a Load more cursor the server cannot verify offers Refresh instead of a dead Retry', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/tagging/reset', { data: { extra_images: 60 } })).toBeOK();
|
||||
await openSettled(page, images);
|
||||
await page.goto(images);
|
||||
await expect(cards(page).first()).toBeVisible();
|
||||
await expect(page.locator('#gallery-continuation')).toContainText('more load as you scroll');
|
||||
await expect(await request.post('/__test__/scope-key/rotate')).toBeOK();
|
||||
@@ -200,7 +191,7 @@ test('a Load more cursor the server cannot verify offers Refresh instead of a de
|
||||
});
|
||||
|
||||
test('a viewer whose media source was renumbered closes onto the file, kept selected', async ({ page, request }) => {
|
||||
await openSettled(page, images);
|
||||
await page.goto(images);
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
await expect(await request.post('/__test__/sources/reverse')).toBeOK();
|
||||
const rescoped = answered(page, '/images/rescope');
|
||||
@@ -226,7 +217,7 @@ test('a collection page finds a renumbered member again and stays on its members
|
||||
await expect(added).toBeOK();
|
||||
const collections = (await (await request.get('/prompt_manager/collections')).json()).collections;
|
||||
const { id } = collections.find(item => item.name === name);
|
||||
await openSettled(page, `/prompt_manager/workspace#/collection/${id}`);
|
||||
await page.goto(`/prompt_manager/workspace#/collection/${id}`);
|
||||
await expect(cards(page)).toHaveCount(1);
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
await expect(await request.post('/__test__/sources/reverse')).toBeOK();
|
||||
|
||||
@@ -11,8 +11,8 @@ const STATUS_URL = '/prompt_manager/backup/status';
|
||||
const PREVIEW_URL = '/prompt_manager/restore/preview';
|
||||
const APPLY_URL = '/prompt_manager/restore/apply';
|
||||
const WHEN = { dateStyle: 'medium', timeStyle: 'short' };
|
||||
const SAFETY_DESCRIPTION = 'Keep a verified copy of the database on disk before maintenance, cleanups, LoRA imports and ' +
|
||||
'prompt deletes change records. A restore always keeps one.';
|
||||
const SAFETY_DESCRIPTION = 'Keep a verified copy of the database on disk before maintenance, cleanups and LoRA imports change ' +
|
||||
'records. A restore always keeps one.';
|
||||
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
|
||||
@@ -274,18 +274,11 @@ test('level filter is minimum-level over all six choices', async ({ page, reques
|
||||
for (const line of (await lines(page).textContent()).split('\n')) expect(line).toMatch(/^\d\d:\d\d:\d\d [A-Z]+ +\S/);
|
||||
expect(urls.at(-1)).toBe('?limit=100');
|
||||
for (const minimum of LEVELS) {
|
||||
// Wait for this level's answer: the previous level's lines also hold the CRITICAL line, so seeing it proved
|
||||
// nothing, and under load the filtered answer painted after the check (#501 merge queue, firefox).
|
||||
const answered = page.waitForResponse(item => new URL(item.url()).pathname === LOGS_URL
|
||||
&& new URL(item.url()).search === `?limit=100&level=${minimum}`);
|
||||
await level.selectOption(minimum);
|
||||
await answered;
|
||||
await expect.poll(() => urls.at(-1)).toBe(`?limit=100&level=${minimum}`);
|
||||
await expect(lines(page)).toContainText('CRITICAL The database could not be opened');
|
||||
await expect.poll(async () => {
|
||||
const shown = await shownLevels(page);
|
||||
return shown.every(name => LEVELS.indexOf(name) >= LEVELS.indexOf(minimum)) || `${minimum}: ${shown}`;
|
||||
}).toBe(true);
|
||||
const shown = await shownLevels(page);
|
||||
expect(shown.every(name => LEVELS.indexOf(name) >= LEVELS.indexOf(minimum)), `${minimum}: ${shown}`).toBe(true);
|
||||
if (minimum === 'DEBUG') await expect(lines(page)).toContainText('DEBUG Thumbnail cache lookup');
|
||||
if (minimum === 'WARNING') {
|
||||
await expect(lines(page)).not.toContainText('Database backup verified');
|
||||
|
||||
@@ -268,71 +268,6 @@ test('Watch for new outputs saves monitoring_enabled; a restart applies it', asy
|
||||
await expect(sourceList(page).locator('.operation p')).toHaveText(['output · watching', 'input · watching']);
|
||||
});
|
||||
|
||||
test('D158: Save prompts from other workflows saves save_foreign_runs live; off while monitoring is off (#247)', async ({ page, request }, testInfo) => {
|
||||
try {
|
||||
const layout = await openSources(page);
|
||||
const rows = layout.locator('section.surface > .setting-row');
|
||||
await expect(rows.locator(':scope > div > label')).toHaveText(['Watch for new outputs', 'Save prompts from other workflows', 'Worker threads']);
|
||||
const foreign = layout.getByRole('switch', { name: 'Save prompts from other workflows', exact: true });
|
||||
const foreignRow = rows.filter({ has: page.locator('#foreign-runs') });
|
||||
await expect(foreignRow.locator(':scope > div > p')).toHaveText('New outputs of workflows without a PromptManager node add their prompt to the library when it is certain.');
|
||||
await expect(foreign).toBeChecked();
|
||||
await expect(foreign).toBeEnabled();
|
||||
// Keyboard on desktop, a tap on the touch project.
|
||||
if (testInfo.project.name === 'touch') await foreign.tap();
|
||||
else {
|
||||
await foreign.focus();
|
||||
await page.keyboard.press('Space');
|
||||
}
|
||||
await expect(foreign).not.toBeChecked();
|
||||
await expect(statusOf(page)).toHaveText('Unsaved changes.');
|
||||
const saved = page.waitForResponse(response => response.request().method() === 'POST' && response.url().endsWith(SETTINGS_URL));
|
||||
await saveButton(page).click();
|
||||
const response = await saved;
|
||||
expect(response.request().postDataJSON()).toEqual(expect.objectContaining({ save_foreign_runs: false }));
|
||||
expect((await response.json()).restart_required_fields).toEqual([]);
|
||||
// Applied live: the monitor reads it for every file, so no restart is named.
|
||||
await expect(statusOf(page)).toHaveText('Saved to the server configuration.');
|
||||
expect((await serverJson(request, SETTINGS_URL)).settings.save_foreign_runs).toBe(false);
|
||||
// With Watch for new outputs drafted off nothing is watched: the row is disabled and its note says why.
|
||||
const monitoring = layout.getByRole('switch', { name: 'Watch for new outputs', exact: true });
|
||||
await monitoring.uncheck();
|
||||
await expect(foreign).toBeDisabled();
|
||||
const note = foreignRow.locator('p.small');
|
||||
await expect(note).toHaveText('Applies while Watch for new outputs is on.');
|
||||
await expect(foreign).toHaveAttribute('aria-describedby', await note.getAttribute('id'));
|
||||
await monitoring.check();
|
||||
await expect(foreign).toBeEnabled();
|
||||
await expect(note).toHaveCount(0);
|
||||
await expect(foreign).not.toHaveAttribute('aria-describedby', /./);
|
||||
await expect(statusOf(page)).toHaveText('No unsaved changes.');
|
||||
} finally {
|
||||
// The server is shared by the spec's tests: put the default back whatever happened.
|
||||
await request.post(SETTINGS_URL, { data: { save_foreign_runs: true } });
|
||||
}
|
||||
});
|
||||
|
||||
test('D158: saved off while the monitor still watches until a restart, the foreign row can still be set (D128)', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/system/monitor', { data: { watching: ['output', 'input'] } })).toBeOK();
|
||||
try {
|
||||
const layout = await openSources(page);
|
||||
await layout.getByRole('switch', { name: 'Watch for new outputs', exact: true }).uncheck();
|
||||
await saveButton(page).click();
|
||||
await expect(sourceList(page).locator('.operation p').first()).toHaveText('output · watching until restart');
|
||||
const foreign = layout.getByRole('switch', { name: 'Save prompts from other workflows', exact: true });
|
||||
await expect(foreign).toBeEnabled();
|
||||
await expect(layout.locator('.setting-row').filter({ has: page.locator('#foreign-runs') }).locator('p.small'))
|
||||
.toHaveText('Applies while Watch for new outputs is on.');
|
||||
await foreign.uncheck();
|
||||
const saved = page.waitForRequest(item => item.method() === 'POST' && item.url().endsWith(SETTINGS_URL));
|
||||
await saveButton(page).click();
|
||||
expect((await saved).postDataJSON()).toEqual({ save_foreign_runs: false, expected_revision: expect.any(String) });
|
||||
await expect(statusOf(page)).toHaveText('Saved to the server configuration.');
|
||||
} finally {
|
||||
await request.post(SETTINGS_URL, { data: { save_foreign_runs: true } });
|
||||
}
|
||||
});
|
||||
|
||||
test('worker range max is cpu_count', async ({ page, request }) => {
|
||||
const { settings } = await serverJson(request, SETTINGS_URL);
|
||||
const layout = await openSources(page);
|
||||
|
||||
@@ -73,7 +73,7 @@ const jobStart = page => page.waitForRequest(item => item.method() === 'POST' &&
|
||||
|
||||
test('selection bar shows Add tags then Auto-tag selected before Set category', async ({ page }) => {
|
||||
await select(page, FIRST);
|
||||
await expect(bar(page).locator(':scope > *')).toHaveText(['1 selected', 'Add tags', 'Auto-tag selected', 'Set category', 'Add to collection', 'Export metadata', 'Delete…', 'Clear']);
|
||||
await expect(bar(page).locator(':scope > *')).toHaveText(['1 selected', 'Add tags', 'Auto-tag selected', 'Set category', 'Add to collection', 'Export metadata', 'Clear']);
|
||||
});
|
||||
|
||||
test('Add tags runs a job and the cards show the tags after it finishes', async ({ page, request }) => {
|
||||
|
||||
@@ -309,21 +309,6 @@ test('fresh registered PM variants preserve exact raw whitespace and known empty
|
||||
expect(value.negative.raw).toBe('');
|
||||
expect(value.negative.effective).toBe('');
|
||||
expect(value.evidence.positive.raw).toMatchObject({ provenance: 'recorded', source: 'runtime_return' });
|
||||
// #438: the Library stores the stripped text; the run is in that prompt's history
|
||||
expect(value.positive_prompt_id, 'the run names the prompt the Library saved').toEqual(expect.any(Number));
|
||||
const runId = (await readExecutionRuns(execution.databasePath, id)).rows[0].run_id;
|
||||
const response = await execution.request.get(`/prompt_manager/library/${value.positive_prompt_id}/history`);expect(response.ok()).toBe(true);
|
||||
expect((await response.json()).items.map(item => item.run_id)).toContain(runId);
|
||||
if (kind !== 'PromptManager') continue;
|
||||
// #460: an upper-case re-queue counts toward the same prompt and joins its history
|
||||
const upper = await submit(execution.request, recipeGraph({ kind, text: `${kind.toUpperCase()} E\u0301` }));
|
||||
await history(execution.request, upper);
|
||||
const variant = (await capture(execution, upper)).contexts[0];
|
||||
expect(variant.positive.raw).toBe(`${kind.toUpperCase()} E\u0301`);
|
||||
expect(variant.positive_prompt_id, 'the case variant names the same prompt').toBe(value.positive_prompt_id);
|
||||
const variantRun = (await readExecutionRuns(execution.databasePath, upper)).rows[0].run_id;
|
||||
const scoped = await execution.request.get(`/prompt_manager/library/${value.positive_prompt_id}/history`);
|
||||
expect((await scoped.json()).items.map(item => item.run_id)).toContain(variantRun);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -334,15 +334,8 @@ async def image_monitor(request: web.Request) -> web.Response:
|
||||
|
||||
@PromptServer.instance.routes.get("/pm_execution_fixture/monitor")
|
||||
async def image_monitor_state(request: web.Request) -> web.Response:
|
||||
"""File names of images waiting for their run's recorded outputs: keyless
|
||||
ones (``awaiting``) and foreign ones (``foreign``, #247)."""
|
||||
"""File names of keyless images waiting for their run's recorded outputs."""
|
||||
monitor = _plugin_module("utils.image_monitor")._monitor_instance
|
||||
handler = getattr(monitor, "handler", None)
|
||||
waiting = handler.awaiting_history() if handler is not None else []
|
||||
foreign = handler.awaiting_foreign() if handler is not None else []
|
||||
return web.json_response(
|
||||
{
|
||||
"awaiting": [os.path.basename(p) for p in waiting],
|
||||
"foreign": [os.path.basename(p) for p in foreign],
|
||||
}
|
||||
)
|
||||
return web.json_response({"awaiting": [os.path.basename(p) for p in waiting]})
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// #342 on the real pinned host: the image monitor links an output only to a prompt
|
||||
// that provably produced it. An unknown node (PMHistoryCached) stands in for the
|
||||
// maintainer's LLM; KSampler is the fixture's deterministic stand-in. #247: an output
|
||||
// of a workflow without a PromptManager node gets its own prompt.
|
||||
// maintainer's LLM; KSampler is the fixture's deterministic stand-in.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
@@ -142,68 +141,6 @@ test('an output saved without metadata links once its recorded run names a store
|
||||
}
|
||||
});
|
||||
|
||||
// #247: no PromptManager node at all, only ComfyUI's own text encoders, sampler and
|
||||
// saver (the fixture's inputs and decoder stand in for the model files).
|
||||
function foreignGraph({ text, seed = 1, prefix }) {
|
||||
return {
|
||||
'10': { class_type: 'PMHistoryInputs', inputs: {} },
|
||||
'20': { class_type: 'CLIPTextEncode', inputs: { text, clip: ['10', 0] } },
|
||||
'30': { class_type: 'CLIPTextEncode', inputs: { text: '', clip: ['10', 0] } },
|
||||
'40': { class_type: 'KSampler', inputs: { model: ['10', 1], latent_image: ['10', 2], positive: ['20', 0],
|
||||
negative: ['30', 0], seed, steps: 1, cfg: 1, sampler_name: 'euler', scheduler: 'normal', denoise: 1 } },
|
||||
'60': { class_type: 'PMHistoryImages', inputs: { value: 64, batch: 1, latent: ['40', 0] } },
|
||||
'70': { class_type: 'SaveImage', inputs: { images: ['60', 0], filename_prefix: prefix } },
|
||||
};
|
||||
}
|
||||
|
||||
test('an output of a workflow without a PromptManager node creates and links its prompt; a second run reuses it', async ({ execution }, testInfo) => {
|
||||
const text = `foreign lighthouse at dusk ${testInfo.project.name} ${randomUUID()}`;
|
||||
const first = await saved(execution.request, foreignGraph({ text, prefix: 'foreign/run' }));
|
||||
await linkedTo(execution, first, text);
|
||||
// Only the seed changes: a new file, the same text, so the same stored prompt.
|
||||
const second = await saved(execution.request, foreignGraph({ text, seed: 2, prefix: 'foreign/run' }));
|
||||
await linkedTo(execution, second, text);
|
||||
const state = await readLinks(execution.databasePath, [first, second], [text]);
|
||||
expect(state).toEqual({ links: { [first]: text, [second]: text }, stored: { [text]: true } });
|
||||
// Decided from the graph at settle: nothing waits for the run's recorded outputs.
|
||||
expect((await (await execution.request.get('/pm_execution_fixture/monitor')).json()).foreign).toEqual([]);
|
||||
});
|
||||
|
||||
// CUJ 1: two branches saved under one prefix, so neither the graph nor the file name
|
||||
// names a file's branch; the run's recorded saver does, once its outputs are sealed.
|
||||
function sharedPrefixGraph(texts) {
|
||||
const nodes = {
|
||||
'10': { class_type: 'PMHistoryInputs', inputs: {} },
|
||||
'30': { class_type: 'CLIPTextEncode', inputs: { text: '', clip: ['10', 0] } },
|
||||
};
|
||||
texts.forEach((text, index) => {
|
||||
const base = (index + 1) * 100;
|
||||
nodes[String(base)] = { class_type: 'CLIPTextEncode', inputs: { text, clip: ['10', 0] } };
|
||||
nodes[String(base + 1)] = { class_type: 'KSampler', inputs: { model: ['10', 1], latent_image: ['10', 2],
|
||||
positive: [String(base), 0], negative: ['30', 0], seed: index + 1, steps: 1, cfg: 1, sampler_name: 'euler',
|
||||
scheduler: 'normal', denoise: 1 } };
|
||||
nodes[String(base + 2)] = { class_type: 'PMHistoryImages', inputs: { value: 64, batch: 1, latent: [String(base + 1), 0] } };
|
||||
nodes[String(base + 3)] = { class_type: 'SaveImage', inputs: { images: [String(base + 2), 0], filename_prefix: 'foreign/dataset' } };
|
||||
});
|
||||
return nodes;
|
||||
}
|
||||
|
||||
test('each branch of a shared-prefix workflow without a PromptManager node links to its own prompt after the run', async ({ execution }, testInfo) => {
|
||||
const tag = `${testInfo.project.name} ${randomUUID()}`;
|
||||
const texts = [`dataset branch one harbor ${tag}`, `dataset branch two meadow ${tag}`];
|
||||
const id = randomUUID();
|
||||
const response = await execution.request.post('/prompt', { data: { prompt: sharedPrefixGraph(texts), prompt_id: id } });
|
||||
expect(response.ok()).toBe(true);
|
||||
let item;
|
||||
await expect.poll(async () => {
|
||||
item = (await (await execution.request.get(`/history/${id}`)).json())[id];
|
||||
return item?.status?.messages?.some(([kind]) => kind === 'execution_success') || false;
|
||||
}, { timeout: 15000 }).toBe(true);
|
||||
const files = ['103', '203'].map(node => item.outputs[node].images[0].filename);
|
||||
for (const [index, file] of files.entries()) await linkedTo(execution, file, texts[index]);
|
||||
expect((await (await execution.request.get('/pm_execution_fixture/monitor')).json()).foreign).toEqual([]);
|
||||
});
|
||||
|
||||
// #332 on the real host: the migration ran at startup and the image category routes
|
||||
// set, list and remove an own category on a generated output.
|
||||
test('an output gets its own image category and loses it again', async ({ execution }, testInfo) => {
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
"""A fake system trash for tests (#480): renames into a folder outside every root.
|
||||
|
||||
``install(folder)`` replaces ``utils.trash.BACKEND`` and returns the fake;
|
||||
``uninstall`` puts the real one back. ``refuse[path] = code`` refuses one
|
||||
file with that code. The fake never copies or unlinks; it records each call.
|
||||
The disposable browser server installs it when ``PM_TEST_FAKE_TRASH=1``, so
|
||||
no test touches a real trash.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from utils import trash
|
||||
|
||||
|
||||
class FakeTrash:
|
||||
def __init__(self, folder):
|
||||
self.folder = str(folder)
|
||||
os.makedirs(self.folder, exist_ok=True)
|
||||
self.refuse = {}
|
||||
self.moved = [] # (source, destination)
|
||||
self.calls = []
|
||||
|
||||
def available(self, path, roots):
|
||||
self.calls.append(("available", str(path)))
|
||||
return self.refuse.get(str(path))
|
||||
|
||||
def move(self, path, roots):
|
||||
self.calls.append(("move", str(path)))
|
||||
code = self.refuse.get(str(path))
|
||||
if code:
|
||||
raise trash.TrashRefused(code)
|
||||
destination = os.path.join(
|
||||
self.folder, f"{len(self.moved)}-{os.path.basename(path)}"
|
||||
)
|
||||
if os.path.lexists(destination):
|
||||
raise trash.TrashRefused(trash.FAILED)
|
||||
os.rename(path, destination)
|
||||
self.moved.append((str(path), destination))
|
||||
return "fake trash"
|
||||
|
||||
|
||||
_ORIGINAL = trash.BACKEND
|
||||
|
||||
|
||||
def install(folder):
|
||||
fake = FakeTrash(folder)
|
||||
trash.BACKEND = fake
|
||||
return fake
|
||||
|
||||
|
||||
def uninstall():
|
||||
trash.BACKEND = _ORIGINAL
|
||||
@@ -118,7 +118,6 @@
|
||||
"POST /prompt_manager/image-annotations/observe",
|
||||
"POST /prompt_manager/image-annotations/suggest",
|
||||
"POST /prompt_manager/images/clear-thumbnails",
|
||||
"POST /prompt_manager/images/delete/preview",
|
||||
"POST /prompt_manager/images/generate-thumbnails",
|
||||
"POST /prompt_manager/images/generate-thumbnails/progress",
|
||||
"POST /prompt_manager/images/link",
|
||||
@@ -132,9 +131,6 @@
|
||||
"POST /prompt_manager/jobs/thumbnails",
|
||||
"POST /prompt_manager/jobs/{job_id}/cancel",
|
||||
"POST /prompt_manager/library",
|
||||
"POST /prompt_manager/library/delete",
|
||||
"POST /prompt_manager/library/delete/preview",
|
||||
"POST /prompt_manager/library/restore",
|
||||
"POST /prompt_manager/library/{positive_id}/variants",
|
||||
"POST /prompt_manager/logs/config",
|
||||
"POST /prompt_manager/logs/truncate",
|
||||
@@ -196,7 +192,6 @@
|
||||
"settings.monitoring_enabled",
|
||||
"settings.result_timeout",
|
||||
"settings.safety_backup",
|
||||
"settings.save_foreign_runs",
|
||||
"settings.thumbnail_edge",
|
||||
"settings.thumbnail_format",
|
||||
"settings.thumbnail_quality",
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
// #480 (D159): Images › Delete…: the review's rows, sentences, labels and outcome toasts.
|
||||
const loaded = await import('../../web/js/workspace/images-delete.mjs').catch(() => null);
|
||||
const module = () => { assert.ok(loaded, 'images-delete.mjs is not implemented'); return loaded; };
|
||||
const pageScope = () => {
|
||||
const controller = new AbortController();
|
||||
return { signal: controller.signal, cleanup: () => {}, abort: () => controller.abort() };
|
||||
};
|
||||
const plan = (over = {}) => ({
|
||||
token: 't', library: { count: 3 }, trash: { count: 2 },
|
||||
refused: { count: 1, counts: { 'not-listed': 1 }, rows: [{ path: 'sub/gone.png', code: 'not-listed', detail: 'No longer listed in Images.' }] },
|
||||
trash_refused: { count: 1, counts: { 'no-trash': 1 }, rows: [{ path: 'b.png', code: 'no-trash', detail: 'This drive has no trash.' }] },
|
||||
records: { prompt_links: 2, pair_links: 0, image_tags: 1, auto_tagged: 0, categories: 1, collections: 2, favorites: 0, review_items: 0 },
|
||||
orphaned_prompts: { count: 1, titles: ['Red fox'] },
|
||||
backup: { policy: 'on' }, disk: null, ...over,
|
||||
});
|
||||
|
||||
test('the words of the review and its routes', () => {
|
||||
const { PREVIEW_URL, MAX_DELETE, TOO_MANY, QUESTION, MODES } = module();
|
||||
assert.equal(PREVIEW_URL, '/prompt_manager/images/delete/preview');
|
||||
assert.equal(MAX_DELETE, 500);
|
||||
assert.equal(TOO_MANY, 'Select at most 500 images.');
|
||||
assert.equal(QUESTION, 'What should happen to the files?');
|
||||
assert.deepEqual(MODES.map(mode => mode.label), ['Remove from the library only', 'Also move the files to the trash']);
|
||||
});
|
||||
|
||||
test('library rows: scope, refusals, links, details, backup and media', () => {
|
||||
const { deleteRows } = module();
|
||||
const rows = Object.fromEntries(deleteRows(plan(), 'library'));
|
||||
assert.equal(rows.Scope, '3 images');
|
||||
assert.equal(rows['Not changed'], '1: gone.png. No longer listed in Images.');
|
||||
assert.equal(rows['Prompt links'], '2 links. 1 prompt loses its last image: Red fox. The prompt stays on the Library page.');
|
||||
assert.equal(rows['Image details'], '1 with image tags, 1 category, 2 collection memberships');
|
||||
assert.equal(rows.Backup, 'Create and verify before changes (the database, not the files)');
|
||||
assert.equal(rows['Media on disk'], 'Files stay on disk, hidden from Images.');
|
||||
});
|
||||
|
||||
test('trash rows add the trash refusals and the trash media line', () => {
|
||||
const { deleteRows } = module();
|
||||
const rows = Object.fromEntries(deleteRows(plan({ backup: { policy: 'off' } }), 'trash'));
|
||||
assert.equal(rows.Scope, '2 images');
|
||||
assert.match(rows['Not changed'], /^2: gone\.png, b\.png\. No longer listed in Images\. This drive has no trash\.$/);
|
||||
assert.equal(rows.Backup, 'Off: no safety backup (System › Backup & export › Recovery policy)');
|
||||
assert.match(rows['Media on disk'], /Nothing is deleted permanently/);
|
||||
});
|
||||
|
||||
test('rows leave out what is zero', () => {
|
||||
const { deleteRows } = module();
|
||||
const bare = plan({ refused: { count: 0, rows: [] }, trash_refused: { count: 0, rows: [] },
|
||||
records: { prompt_links: 0 }, orphaned_prompts: { count: 0, titles: [] } });
|
||||
assert.deepEqual(deleteRows(bare, 'library').map(([term]) => term), ['Scope', 'Backup', 'Media on disk']);
|
||||
const many = plan({ orphaned_prompts: { count: 5, titles: ['A', 'B', 'C'] } });
|
||||
assert.match(Object.fromEntries(deleteRows(many, 'library'))['Prompt links'],
|
||||
/5 prompts lose their last image: A, B, C and 2 more\. The prompts stay on the Library page\./);
|
||||
assert.equal(module().refusedValue({ count: 5, rows: [{ path: 'a.png' }] }), '5: a.png and 4 more');
|
||||
assert.equal(module().refusedValue({ count: 0 }), null);
|
||||
});
|
||||
|
||||
test('sentences follow the mode and the backup policy', () => {
|
||||
const { deleteSentences, SENTENCES, USE_LIBRARY_ONLY, NO_TRASH_HERE } = module();
|
||||
assert.deepEqual(deleteSentences(plan(), 'library'), [SENTENCES.libraryBackup]);
|
||||
assert.deepEqual(deleteSentences(plan({ backup: { policy: 'off' } }), 'library'), [SENTENCES.libraryNoBackup]);
|
||||
assert.deepEqual(deleteSentences(plan(), 'trash'), [SENTENCES.trash, USE_LIBRARY_ONLY]);
|
||||
const nowhere = plan({ trash: { count: 0 }, trash_refused: { count: 3, rows: [] } });
|
||||
assert.deepEqual(deleteSentences(nowhere, 'trash'), [SENTENCES.trash, NO_TRASH_HERE, USE_LIBRARY_ONLY]);
|
||||
// #500: no "yet" (Windows is unavailable by design); each file's reason says why.
|
||||
assert.equal(NO_TRASH_HERE, 'Moving to the trash is not available for these files; each one lists why.');
|
||||
assert.deepEqual(deleteSentences(plan({ trash_refused: { count: 0 } }), 'trash'), [SENTENCES.trash]);
|
||||
});
|
||||
|
||||
test('primary labels and the close-only reasons', () => {
|
||||
const { primaryLabel, closeOnlyReason, NONE_DELETABLE } = module();
|
||||
assert.equal(primaryLabel('library', 1), 'Remove 1 image from the library');
|
||||
assert.equal(primaryLabel('trash', 4), 'Move 4 images to the trash');
|
||||
assert.equal(closeOnlyReason(plan()), null);
|
||||
assert.equal(closeOnlyReason(plan({ library: { count: 0 }, trash: { count: 0 } })), NONE_DELETABLE);
|
||||
assert.equal(closeOnlyReason(plan({ disk: { sufficient: false, required_bytes: 2048, free_bytes: 1024 } })),
|
||||
'There is not enough free space for the safety backup (needs 2,048 bytes, has 1,024).');
|
||||
});
|
||||
|
||||
test('outcome toasts per mode and failure', () => {
|
||||
const { outcomeToast } = module();
|
||||
assert.equal(outcomeToast({ scope: { mode: 'library' }, counters: { removed: 3, not_changed: 0 }, total: 3 }),
|
||||
'Removed 3 images from the library.');
|
||||
assert.equal(outcomeToast({ scope: { mode: 'trash' }, counters: { trashed: 2, not_changed: 1 }, total: 3 }),
|
||||
'Moved 2 images to the trash. 1 was not changed (see Activity).');
|
||||
assert.equal(outcomeToast({ scope: { mode: 'trash' }, counters: { trashed: 0, not_changed: 2 }, total: 2 }),
|
||||
'Moved 0 images to the trash. 2 were not changed (see Activity).');
|
||||
assert.equal(outcomeToast({ scope: { mode: 'library', failure: 'records-failed' }, counters: { removed: 1 }, total: 4 }),
|
||||
'Stopped after 1 of 4 images; the rest were not changed. See Activity.');
|
||||
assert.equal(outcomeToast({ scope: { failure: 'backup-failed' } }), 'The safety backup failed; nothing was changed.');
|
||||
assert.equal(outcomeToast({ scope: { failure: 'sources-changed' } }), 'The sources changed; nothing was changed.');
|
||||
});
|
||||
|
||||
test('review failures name the cause', () => {
|
||||
const { reviewFailure, SOURCE_CHANGED, STORAGE_CHANGED, TOO_MANY, REVIEW_FAILED } = module();
|
||||
assert.equal(reviewFailure({ code: 'source-changed' }), SOURCE_CHANGED);
|
||||
assert.equal(reviewFailure({ code: 'storage-changed' }), STORAGE_CHANGED);
|
||||
assert.equal(reviewFailure({ code: 'too-many-images' }), TOO_MANY);
|
||||
assert.equal(reviewFailure({ code: 'busy', message: 'Scan running' }), 'Scan running');
|
||||
assert.equal(reviewFailure(new Error('boom')), REVIEW_FAILED);
|
||||
});
|
||||
|
||||
test('over 500 refuses with the toast before any request', async () => {
|
||||
const { openDeleteImages, TOO_MANY } = module();
|
||||
const toasts = [];
|
||||
const api = { post: () => assert.fail('no request') };
|
||||
await openDeleteImages({ api, scope: pageScope(), images: [], filter: {}, count: 501, notify: text => toasts.push(text) });
|
||||
assert.deepEqual(toasts, [TOO_MANY]);
|
||||
await openDeleteImages({ api, scope: pageScope(), images: [], notify: text => toasts.push(text) });
|
||||
assert.deepEqual(toasts, [TOO_MANY]);
|
||||
});
|
||||
|
||||
test('a failed review toasts and releases its trigger', async () => {
|
||||
const { openDeleteImages, SOURCE_CHANGED } = module();
|
||||
const toasts = [], sent = [];
|
||||
const attributes = new Map();
|
||||
const trigger = { textContent: 'Delete…', getAttribute: name => attributes.get(name) ?? null,
|
||||
setAttribute: (name, value) => attributes.set(name, value), removeAttribute: name => attributes.delete(name) };
|
||||
const api = { post: async (url, body) => { sent.push([url, body]); throw Object.assign(new Error('x'), { code: 'source-changed' }); } };
|
||||
await openDeleteImages({ api, scope: pageScope(), trigger, notify: text => toasts.push(text),
|
||||
images: [{ root_index: 0, root_scope: 'a'.repeat(64), path: 'a.png', filename: 'a.png' }] });
|
||||
assert.deepEqual(sent, [['/prompt_manager/images/delete/preview', { items: [{ root_index: 0, root_scope: 'a'.repeat(64), path: 'a.png' }] }]]);
|
||||
assert.deepEqual(toasts, [SOURCE_CHANGED]);
|
||||
assert.equal(trigger.textContent, 'Delete…');
|
||||
assert.equal(trigger.getAttribute('aria-disabled'), null);
|
||||
});
|
||||
|
||||
test('apply sends the token and the mode, and toasts that the job started', async () => {
|
||||
const { applyDelete } = module();
|
||||
const sent = [], toasts = [];
|
||||
const api = { post: async (url, body) => { sent.push([url, body]); return { job: { id: 'j', title: 'Delete images (move to the trash)' } }; } };
|
||||
const store = { track: job => job, load: () => {} };
|
||||
const job = await applyDelete({ api, store, plan: plan(), mode: 'trash', notify: text => toasts.push(text) });
|
||||
assert.equal(job.id, 'j');
|
||||
assert.deepEqual(sent, [['/prompt_manager/maintenance/apply', { token: 't', mode: 'trash' }]]);
|
||||
assert.deepEqual(toasts, ['Delete images (move to the trash) started.']);
|
||||
const refused = await applyDelete({ api: { post: async () => { throw Object.assign(new Error('Busy'), { code: 'busy', running: null }); } },
|
||||
store, plan: plan(), mode: 'library', notify: text => toasts.push(text) });
|
||||
assert.equal(refused, null);
|
||||
});
|
||||
|
||||
test('apply refuses to send a delete without an explicit mode', async () => {
|
||||
const { applyDelete } = module();
|
||||
const api = { post: () => assert.fail('no request without a mode') };
|
||||
for (const mode of [undefined, null, '', 'TRASH', 'delete']) {
|
||||
await assert.rejects(applyDelete({ api, store: null, plan: plan(), mode, notify: () => {} }), TypeError);
|
||||
}
|
||||
});
|
||||
@@ -1,150 +0,0 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
// #443 D152: the Prompt library's Delete… review (delete unpaired, archive paired, keep protected) and Restore.
|
||||
const source = import('../../web/js/workspace/library-delete.mjs');
|
||||
|
||||
const preview = (patch = {}) => ({success: true, library_epoch: 'e', delete: [], archive: [], protected: [],
|
||||
already_archived: 0, not_found: 0, image_links: 0, backup: 'on', disk: null, ...patch});
|
||||
|
||||
test('the routes are the workspace ones, never the legacy bulk delete', async () => {
|
||||
const {PREVIEW_URL, DELETE_URL, RESTORE_URL} = await source;
|
||||
assert.equal(PREVIEW_URL, '/prompt_manager/library/delete/preview');
|
||||
assert.equal(DELETE_URL, '/prompt_manager/library/delete');
|
||||
assert.equal(RESTORE_URL, '/prompt_manager/library/restore');
|
||||
});
|
||||
|
||||
test('the title, primary and body follow what the review will do', async () => {
|
||||
const {reviewOf, reviewTitle, primaryLabel, deleteBody} = await source;
|
||||
const both = reviewOf(preview({delete: [1, 2], archive: [3]}));
|
||||
assert.equal(reviewTitle(both), 'Delete or archive selected prompts');
|
||||
assert.equal(primaryLabel(both), 'Delete 2 and archive 1 prompt');
|
||||
assert.deepEqual(deleteBody(both), {archive_ids: [3], delete_ids: [1, 2], expected_epoch: 'e', backup: 'on'});
|
||||
const onlyDelete = reviewOf(preview({delete: [1]}));
|
||||
assert.equal(reviewTitle(onlyDelete), 'Delete selected prompts');
|
||||
assert.equal(primaryLabel(onlyDelete), 'Delete 1 prompt');
|
||||
const onlyArchive = reviewOf(preview({archive: [4, 5]}));
|
||||
assert.equal(reviewTitle(onlyArchive), 'Archive selected prompts');
|
||||
assert.equal(primaryLabel(onlyArchive), 'Archive 2 prompts');
|
||||
});
|
||||
|
||||
test('the grid shows each row only when it applies, with protected titles in selection order', async () => {
|
||||
const {reviewOf, reviewGrid} = await source;
|
||||
const names = {7: 'Seven', 8: 'Eight', 9: 'Nine', 10: 'Ten'};
|
||||
const review = reviewOf(preview({delete: [1], archive: [2], protected: [7, 8, 9, 10], already_archived: 1,
|
||||
not_found: 2, image_links: 3}));
|
||||
assert.deepEqual(reviewGrid(review, id => names[id]), [
|
||||
['Delete permanently', '1 prompt not in a saved pair'],
|
||||
['Archive', '1 prompt in a saved pair: hidden, restorable'],
|
||||
['Protected (kept)', '4: Seven, Eight, Nine and 1 more'],
|
||||
['Already archived', '1'],
|
||||
['No longer exist', '2'],
|
||||
['Image links removed', '3 images will no longer show these prompts (files stay on disk)'],
|
||||
['Backup', 'Create and verify before deleting'],
|
||||
]);
|
||||
const archiveOnly = reviewOf(preview({archive: [2]}));
|
||||
assert.deepEqual(reviewGrid(archiveOnly).map(([term]) => term), ['Delete permanently', 'Archive']);
|
||||
const off = reviewOf(preview({delete: [1], backup: 'off'}));
|
||||
assert.equal(reviewGrid(off).at(-1)[1], 'Off: no safety backup (System › Backup & export › Recovery policy)');
|
||||
});
|
||||
|
||||
test('the Archive row names the saved pair, never negative variants, for one or many prompts', async () => {
|
||||
const {reviewOf, reviewGrid} = await source;
|
||||
// A saved-pair prompt with 0 negative variants (live prompt 7901) is archive-only for its pair, not for variants.
|
||||
const one = reviewGrid(reviewOf(preview({archive: [2]}))).find(([term]) => term === 'Archive');
|
||||
assert.deepEqual(one, ['Archive', '1 prompt in a saved pair: hidden, restorable']);
|
||||
const many = reviewGrid(reviewOf(preview({archive: [2, 3, 4]}))).find(([term]) => term === 'Archive');
|
||||
assert.deepEqual(many, ['Archive', '3 prompts in a saved pair: hidden, restorable']);
|
||||
for (const [, value] of [one, many]) assert.doesNotMatch(value, /variant/);
|
||||
});
|
||||
|
||||
test('the sentences explain archive, irreversibility per backup policy, re-creation and protection', async () => {
|
||||
const {reviewOf, reviewSentences} = await source;
|
||||
const [archive, remove, guarded] = reviewSentences(reviewOf(preview({delete: [1], archive: [2], protected: [3]})));
|
||||
// Live test (#470): the archive reason is the saved pair, which a prompt with 0 negative variants has too.
|
||||
assert.match(archive, /^Prompts in a saved pair, as its positive or its negative, are archived instead of deleted, so the pair and its runs stay intact\./);
|
||||
assert.doesNotMatch(archive, /variant/);
|
||||
assert.match(archive, /Archived prompts cannot be deleted\. To bring them back, press Show archived prompts/);
|
||||
assert.match(remove, /the only way back is to restore the safety backup taken first \(System › Backup & export\), which returns the whole library to that moment\. It is kept among the last 10 safety backups\./);
|
||||
assert.match(remove, /A later scan, a new image or a node run with the same text adds it again as a new prompt\.$/);
|
||||
assert.equal(guarded, 'Protected prompts are tagged __protected__; remove the tag to delete them.');
|
||||
const [off] = reviewSentences(reviewOf(preview({delete: [1], backup: 'off'})));
|
||||
assert.match(off, /Deleted prompts cannot be brought back\. Image files/);
|
||||
assert.doesNotMatch(off, /safety backup/);
|
||||
});
|
||||
|
||||
test('a review that cannot start says why: nothing to change, or no room for the backup', async () => {
|
||||
const {reviewOf, blockedReason, COPY} = await source;
|
||||
assert.equal(blockedReason(reviewOf(preview({protected: [1]}))), COPY.none);
|
||||
assert.equal(blockedReason(reviewOf(preview({delete: [1], disk: {sufficient: false, message: 'Not enough. Nothing was changed.'}}))),
|
||||
'Not enough. Nothing was changed.');
|
||||
assert.match(blockedReason(reviewOf(preview({delete: [1], disk: {sufficient: false}}))), /not enough free space/);
|
||||
assert.equal(blockedReason(reviewOf(preview({delete: [1], disk: {sufficient: true}}))), null);
|
||||
assert.equal(blockedReason(reviewOf(preview({archive: [1]}))), null);
|
||||
});
|
||||
|
||||
test('a delete outcome counts each part, singular and plural, and names the backup', async () => {
|
||||
const {reviewOf, deleteOutcome} = await source;
|
||||
const review = reviewOf(preview({delete: [1, 2], archive: [3], protected: [9]}));
|
||||
assert.deepEqual(deleteOutcome({success: true, deleted: 2, archived: 3, already_archived: 0, not_found: 0, changed: 0,
|
||||
backup: 'prompts.db.backup_x'}, null, review), {committed: true, reload: true,
|
||||
toast: 'Deleted 2 prompts. Archived 3 prompts. 1 protected prompt was kept. Safety backup saved: prompts.db.backup_x (System › Backup & export).'});
|
||||
assert.equal(deleteOutcome({success: true, deleted: 1, archived: 0, already_archived: 1, not_found: 1, changed: 1,
|
||||
backup: null}, null, reviewOf(preview())).toast,
|
||||
'Deleted 1 prompt. 1 was already archived. 1 no longer exists. 1 changed since the review and was kept.');
|
||||
assert.equal(deleteOutcome({success: true, deleted: 0, archived: 0, already_archived: 2, not_found: 3, changed: 2,
|
||||
backup: null}, null, reviewOf(preview({protected: [1, 2]}))).toast,
|
||||
'Nothing was deleted or archived. 2 were already archived. 3 no longer exist. 2 changed since the review and were kept. 2 protected prompts were kept. The Library was reloaded.');
|
||||
assert.equal(deleteOutcome({success: true, deleted: 0, archived: 0}, null, reviewOf(preview())).committed, false);
|
||||
});
|
||||
|
||||
test('a failed delete says nothing changed, or that it may have, and keeps or reloads the selection', async () => {
|
||||
const {deleteOutcome, COPY} = await source;
|
||||
assert.deepEqual(deleteOutcome(null, {status: 409, code: 'storage-changed'}), {toast: COPY.storage, committed: false, reload: true});
|
||||
assert.equal(COPY.storage, 'The library was restored or moved since this page loaded. Nothing was deleted or archived.');
|
||||
assert.deepEqual(deleteOutcome(null, {status: 409, code: 'busy', running: {title: 'Scan'}}),
|
||||
{toast: 'Scan is running. Nothing was changed; try again when it finishes.', committed: false, reload: false});
|
||||
assert.deepEqual(deleteOutcome(null, {status: 500, code: 'backup-failed', message: 'The safety backup could not be taken and verified; nothing was changed.'}),
|
||||
{toast: 'The safety backup failed. The safety backup could not be taken and verified; nothing was changed.', committed: false, reload: false});
|
||||
assert.deepEqual(deleteOutcome(null, {status: 507, code: 'insufficient-space', message: 'Not enough free disk space. Nothing was changed.'}),
|
||||
{toast: 'Not enough free disk space. Nothing was changed.', committed: false, reload: false});
|
||||
for (const failure of [new TypeError('Failed to fetch'), {status: 502}, {status: 503}, {status: 504}]) {
|
||||
assert.deepEqual(deleteOutcome(null, failure), {toast: COPY.unconfirmed, committed: true, reload: true});
|
||||
}
|
||||
assert.deepEqual(deleteOutcome(null, {status: 400}), {toast: COPY.failed, committed: false, reload: false});
|
||||
});
|
||||
|
||||
test('a failed preview reloads only after a restore; Restore outcomes have their own words', async () => {
|
||||
const {previewOutcome, restoreOutcome, COPY} = await source;
|
||||
assert.deepEqual(previewOutcome({status: 409, code: 'storage-changed'}), {toast: COPY.storage, committed: false, reload: true});
|
||||
assert.deepEqual(previewOutcome({status: 500}), {toast: COPY.reviewFailed, committed: false, reload: false});
|
||||
assert.equal(previewOutcome({status: 409, code: 'busy', running: {title: 'Optimize'}}).toast,
|
||||
'Optimize is running. Nothing was changed; try again when it finishes.');
|
||||
assert.deepEqual(restoreOutcome({success: true, restored: 2, unchanged: 1, not_found: 1}, null),
|
||||
{toast: 'Restored 2 prompts to the Prompt library. 1 was not archived. 1 no longer exists.', committed: true, reload: true});
|
||||
assert.deepEqual(restoreOutcome({success: true, restored: 0, unchanged: 2, not_found: 0}, null),
|
||||
{toast: 'No prompt was restored. 2 were not archived.', committed: false, reload: true});
|
||||
assert.deepEqual(restoreOutcome(null, {status: 409, code: 'storage-changed'}), {toast: COPY.restoreStorage, committed: false, reload: true});
|
||||
assert.deepEqual(restoreOutcome(null, {status: 503}), {toast: COPY.restoreUnconfirmed, committed: true, reload: true});
|
||||
assert.deepEqual(restoreOutcome(null, {status: 400}), {toast: COPY.restoreFailed, committed: false, reload: false});
|
||||
});
|
||||
|
||||
test('reviewOf keeps only well-formed ids and counts from an answer', async () => {
|
||||
const {reviewOf} = await source;
|
||||
const review = reviewOf({library_epoch: 3, delete: [1, 'x', -2, 2], archive: null, protected: [1.5], already_archived: -1,
|
||||
not_found: 'x', image_links: 2, backup: 'maybe'});
|
||||
assert.deepEqual(review, {epoch: null, delete: [1, 2], archive: [], protected: [], alreadyArchived: 0, notFound: 0,
|
||||
imageLinks: 2, backup: 'on', disk: null});
|
||||
});
|
||||
|
||||
test('the archived notice says why (the saved pair) and names negative variants only when there are any', async () => {
|
||||
const {archivedNoticeText} = await source;
|
||||
assert.equal(archivedNoticeText(0), 'This prompt is archived because it is in a saved pair. It is hidden from the Prompt ' +
|
||||
'library; nothing was removed.');
|
||||
assert.equal(archivedNoticeText(1), 'This prompt is archived because it is in a saved pair. It and its 1 negative variant ' +
|
||||
'are hidden from the Prompt library; nothing was removed.');
|
||||
assert.equal(archivedNoticeText(3), 'This prompt is archived because it is in a saved pair. It and its 3 negative variants ' +
|
||||
'are hidden from the Prompt library; nothing was removed.');
|
||||
// An unknown count (an older server) never claims variants.
|
||||
assert.equal(archivedNoticeText(undefined), archivedNoticeText(0));
|
||||
});
|
||||
@@ -157,20 +157,3 @@ test('a valid configured source missing from the effective list is another spell
|
||||
assert.deepEqual(sourceRows({ sources, saved, current: saved }).rows.at(-1), { key: 'output/.', title: 'output',
|
||||
description: 'output/. · same folder as another source', scan: null, remove: 'output/.' });
|
||||
});
|
||||
|
||||
test('D158: "Save prompts from other workflows" drafts save_foreign_runs, disabled while monitoring is off (#247)', async () => {
|
||||
const { FOREIGN_RUNS, foreignRunsState } = await sourcesModule;
|
||||
assert.deepEqual(FOREIGN_RUNS, { id: 'foreign-runs', key: 'save_foreign_runs', label: 'Save prompts from other workflows',
|
||||
description: 'New outputs of workflows without a PromptManager node add their prompt to the library when it is certain.',
|
||||
off: 'Applies while Watch for new outputs is on.' });
|
||||
assert.deepEqual(foreignRunsState({ value: true, monitoring: true }), { disabled: false, checked: true, note: null });
|
||||
assert.deepEqual(foreignRunsState({ value: false, monitoring: true }), { disabled: false, checked: false, note: null });
|
||||
// Monitoring drafted or saved off: nothing is watched, so the row says why it is disabled and keeps its value.
|
||||
assert.deepEqual(foreignRunsState({ value: true, monitoring: false }), { disabled: true, checked: true, note: FOREIGN_RUNS.off });
|
||||
// Not read yet (or the host read failed): disabled without a value, the host rows' own note applies.
|
||||
assert.deepEqual(foreignRunsState({ value: undefined, monitoring: undefined }), { disabled: true, checked: null, note: null });
|
||||
assert.deepEqual(foreignRunsState({ value: undefined, monitoring: false }), { disabled: true, checked: null, note: null });
|
||||
// Saved off while the running monitor still watches until the restart (D128 `stopping`): it can still be set.
|
||||
assert.deepEqual(foreignRunsState({ value: true, monitoring: false, stopping: true }),
|
||||
{ disabled: false, checked: true, note: FOREIGN_RUNS.off });
|
||||
});
|
||||
|
||||
+2
-44
@@ -28,10 +28,6 @@ stack shows which). A dump after the summary is a hang at interpreter exit
|
||||
|
||||
A limit is a measurement, not a dial: change it only with new measurements
|
||||
written next to it (TESTING.md "Time limits and hangs").
|
||||
|
||||
PM_TEST_SLOWEST=N (opt-in, #487) lists the N slowest tests after the summary.
|
||||
A test's time runs from its ``start:`` line to the next one, so it includes
|
||||
the class and module fixtures in between, like the hang limit.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -40,7 +36,6 @@ import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
@@ -68,57 +63,23 @@ def read_limit(name, default):
|
||||
return int(text)
|
||||
|
||||
|
||||
class TestClock:
|
||||
"""Seconds from each test's start to the next start (or the run's end)."""
|
||||
|
||||
def __init__(self, clock=time.perf_counter):
|
||||
self._clock = clock
|
||||
self._current = None
|
||||
self._started = 0.0
|
||||
self.durations = []
|
||||
|
||||
def start(self, test_id):
|
||||
self.stop()
|
||||
self._current = test_id
|
||||
self._started = self._clock()
|
||||
|
||||
def stop(self):
|
||||
if self._current is not None:
|
||||
self.durations.append((self._clock() - self._started, self._current))
|
||||
self._current = None
|
||||
|
||||
def report(self, count):
|
||||
"""Return the lines that list the ``count`` slowest tests."""
|
||||
slowest = sorted(self.durations, key=lambda item: item[0], reverse=True)
|
||||
lines = [f"slowest {min(count, len(slowest))} of {len(slowest)} tests:"]
|
||||
lines += [f"{seconds:8.2f}s {test_id}" for seconds, test_id in slowest[:count]]
|
||||
return lines
|
||||
|
||||
|
||||
def make_result_class(stream, per_test_limit, shutdown_limit, slowest=0):
|
||||
def make_result_class(stream, per_test_limit, shutdown_limit):
|
||||
"""Build a TextTestResult that names each test and re-arms the watchdog."""
|
||||
|
||||
def arm(seconds):
|
||||
faulthandler.dump_traceback_later(seconds, exit=True, file=stream)
|
||||
|
||||
clock = TestClock()
|
||||
|
||||
class WatchdogResult(unittest.TextTestResult):
|
||||
def startTest(self, test):
|
||||
# At default verbosity the progress dots share a line; start ours.
|
||||
prefix = "\n" if self.dots else ""
|
||||
stream.write(f"{prefix}start: {test.id()}\n")
|
||||
stream.flush()
|
||||
clock.start(test.id())
|
||||
super().startTest(test)
|
||||
arm(per_test_limit)
|
||||
|
||||
def stopTestRun(self):
|
||||
clock.stop()
|
||||
super().stopTestRun()
|
||||
if slowest:
|
||||
stream.write("\n".join(["", *clock.report(slowest), ""]))
|
||||
stream.flush()
|
||||
arm(shutdown_limit)
|
||||
|
||||
return WatchdogResult
|
||||
@@ -177,7 +138,6 @@ def main(argv=None):
|
||||
try:
|
||||
per_test_limit = read_limit("PM_TEST_HANG_LIMIT", DEFAULT_PER_TEST_LIMIT)
|
||||
shutdown_limit = read_limit("PM_TEST_SHUTDOWN_LIMIT", DEFAULT_SHUTDOWN_LIMIT)
|
||||
slowest = read_limit("PM_TEST_SLOWEST", 0)
|
||||
except LimitError as error:
|
||||
stream.write(f"{error}\n")
|
||||
return USAGE_EXIT
|
||||
@@ -192,9 +152,7 @@ def main(argv=None):
|
||||
verbosity=args.verbosity,
|
||||
# As unittest.main does: show warnings unless -W/PYTHONWARNINGS set them.
|
||||
warnings=None if sys.warnoptions else "default",
|
||||
resultclass=make_result_class(
|
||||
stream, per_test_limit, shutdown_limit, slowest
|
||||
),
|
||||
resultclass=make_result_class(stream, per_test_limit, shutdown_limit),
|
||||
)
|
||||
result = runner.run(suite)
|
||||
finally:
|
||||
|
||||
@@ -684,132 +684,6 @@ class TestStaticRoutes(CoreRoutesTestCase):
|
||||
self.assertEqual(resp.status, 403)
|
||||
|
||||
|
||||
class TestRevalidatedCaching(CoreRoutesTestCase):
|
||||
"""Every served page and static file is revalidated on reload (#478).
|
||||
|
||||
``Cache-Control: no-cache`` makes the browser ask before it reuses a copy,
|
||||
and the ETag lets an unchanged file answer 304 without a body, so an update
|
||||
reaches the browser on a normal reload and an unchanged reload stays cheap.
|
||||
"""
|
||||
|
||||
STATIC_PATHS = (
|
||||
"/prompt_manager/js/workspace/index.mjs",
|
||||
"/prompt_manager/js/admin.js",
|
||||
"/prompt_manager/js/inspection-semantics.json",
|
||||
"/prompt_manager/lib/viewerjs/viewer.min.css",
|
||||
)
|
||||
|
||||
async def _assert_revalidated(self, path):
|
||||
resp = await self.client.request("GET", path)
|
||||
self.assertEqual(resp.status, 200, path)
|
||||
self.assertEqual(resp.headers.get("Cache-Control"), "no-cache", path)
|
||||
etag = resp.headers.get("ETag")
|
||||
self.assertTrue(etag, path)
|
||||
body = await resp.read()
|
||||
self.assertTrue(body, path)
|
||||
|
||||
again = await self.client.request("GET", path, headers={"If-None-Match": etag})
|
||||
self.assertEqual(again.status, 304, path)
|
||||
self.assertEqual(again.headers.get("ETag"), etag, path)
|
||||
self.assertEqual(again.headers.get("Cache-Control"), "no-cache", path)
|
||||
self.assertEqual(await again.read(), b"", path)
|
||||
|
||||
stale = await self.client.request(
|
||||
"GET", path, headers={"If-None-Match": 'W/"stale"'}
|
||||
)
|
||||
self.assertEqual(stale.status, 200, path)
|
||||
self.assertEqual(await stale.read(), body, path)
|
||||
return etag
|
||||
|
||||
async def test_static_files_are_revalidated(self):
|
||||
for path in self.STATIC_PATHS:
|
||||
await self._assert_revalidated(path)
|
||||
|
||||
async def test_pages_are_revalidated(self):
|
||||
for path in UI_PATHS:
|
||||
await self._assert_revalidated(path)
|
||||
|
||||
async def test_etag_matches_in_a_list_and_as_wildcard(self):
|
||||
path = "/prompt_manager/js/workspace/index.mjs"
|
||||
etag = (await self.client.request("GET", path)).headers["ETag"]
|
||||
for header in (f'W/"other", {etag}', etag.removeprefix("W/"), "*"):
|
||||
resp = await self.client.request(
|
||||
"GET", path, headers={"If-None-Match": header}
|
||||
)
|
||||
self.assertEqual(resp.status, 304, header)
|
||||
|
||||
async def test_changed_file_gets_a_new_etag_and_its_new_body(self):
|
||||
web_dir = self._fake_root()
|
||||
module = web_dir / "js" / "m.mjs"
|
||||
module.write_text("export const v = 1;")
|
||||
path = "/prompt_manager/js/m.mjs"
|
||||
first = await self.client.request("GET", path)
|
||||
etag = first.headers["ETag"]
|
||||
# Same size and, on coarse clocks, the same mtime: only the bytes differ.
|
||||
module.write_text("export const v = 2;")
|
||||
resp = await self.client.request("GET", path, headers={"If-None-Match": etag})
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertNotEqual(resp.headers["ETag"], etag)
|
||||
self.assertEqual(await resp.text(), "export const v = 2;")
|
||||
|
||||
async def test_gzipped_and_plain_bodies_share_a_weak_etag(self):
|
||||
"""The gzip middleware changes the bytes, so the tag must be weak."""
|
||||
resp = await self.client.request("GET", "/prompt_manager/js/admin.js")
|
||||
self.assertTrue(resp.headers["ETag"].startswith('W/"'))
|
||||
|
||||
async def test_errors_carry_no_validator(self):
|
||||
resp = await self.client.request("GET", "/prompt_manager/js/nope.js")
|
||||
self.assertEqual(resp.status, 404)
|
||||
self.assertNotIn("ETag", resp.headers)
|
||||
|
||||
|
||||
@web.middleware
|
||||
async def _comfyui_cache_control(request, handler):
|
||||
"""ComfyUI's own middleware (middleware/cache_middleware.py), JS/CSS branch.
|
||||
|
||||
It only fills a missing Cache-Control, and ``.mjs`` is not ``.js``: that is
|
||||
why the workspace modules went out with no header at all before #478.
|
||||
"""
|
||||
response = await handler(request)
|
||||
if request.path.endswith(".js") or request.path.endswith(".css"):
|
||||
response.headers.setdefault("Cache-Control", "no-store")
|
||||
return response
|
||||
|
||||
|
||||
class TestRevalidatedCachingBehindMiddlewares(CoreRoutesTestCase):
|
||||
"""The host's middlewares keep the header and the 304 intact."""
|
||||
|
||||
async def get_application(self):
|
||||
app = await super().get_application()
|
||||
app.middlewares.append(_comfyui_cache_control)
|
||||
app.middlewares.append(api_module._gzip_middleware)
|
||||
return app
|
||||
|
||||
async def test_gzipped_files_keep_no_cache_and_answer_304(self):
|
||||
for path in (
|
||||
"/prompt_manager/js/workspace/index.mjs",
|
||||
"/prompt_manager/js/admin.js",
|
||||
"/prompt_manager/lib/viewerjs/viewer.min.css",
|
||||
"/prompt_manager/workspace",
|
||||
):
|
||||
resp = await self.client.request(
|
||||
"GET", path, headers={"Accept-Encoding": "gzip"}
|
||||
)
|
||||
self.assertEqual(resp.status, 200, path)
|
||||
self.assertEqual(resp.headers.get("Content-Encoding"), "gzip", path)
|
||||
self.assertEqual(resp.headers["Cache-Control"], "no-cache", path)
|
||||
again = await self.client.request(
|
||||
"GET",
|
||||
path,
|
||||
headers={
|
||||
"Accept-Encoding": "gzip",
|
||||
"If-None-Match": resp.headers["ETag"],
|
||||
},
|
||||
)
|
||||
self.assertEqual(again.status, 304, path)
|
||||
self.assertEqual(again.headers["Cache-Control"], "no-cache", path)
|
||||
|
||||
|
||||
# ── instance helpers ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -45,9 +45,7 @@ SEEDED_PATHS = (
|
||||
"/srv/someone/renders",
|
||||
"C:\\Users\\someone\\Documents\\renders",
|
||||
"D:/Users/someone/prompts.db",
|
||||
# A share on this machine: on Windows the report resolves every source,
|
||||
# and an unknown server name made that wait about 2.9 s (#487).
|
||||
"\\\\localhost\\pm-no-such-share\\someone\\renders",
|
||||
"\\\\nas\\share\\someone\\renders",
|
||||
)
|
||||
_WINDOWS_ABSOLUTE = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
TOP_KEYS = {
|
||||
@@ -87,7 +85,6 @@ CONFIG = {
|
||||
"max_image_age_days",
|
||||
"max_concurrent_processing",
|
||||
"metadata_extraction_timeout",
|
||||
"save_foreign_runs",
|
||||
},
|
||||
"integrations": {"lora_manager_enabled", "has_civitai_api_key"},
|
||||
}
|
||||
|
||||
@@ -18,14 +18,11 @@ from unittest.mock import patch
|
||||
|
||||
from aiohttp import FormData, web
|
||||
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
try:
|
||||
from test_api_images import ImageAPITestCase
|
||||
except ImportError:
|
||||
from tests.test_api_images import ImageAPITestCase
|
||||
|
||||
from database import restore_stage
|
||||
from database.operations import PromptDatabase
|
||||
from py.api.library import _encode, _scope
|
||||
from tests.test_execution_context_database import bundle, snapshot
|
||||
@@ -44,21 +41,6 @@ OTHER_SESSION = "aaaaaaaa-3333-4333-8333-333333333333"
|
||||
MISSING_RUN = "bbbbbbbb-4444-4444-8444-444444444444"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def no_wal_truncate_wait():
|
||||
"""Restore with the restore notification held back, as these tests need.
|
||||
|
||||
The read under test keeps its snapshot open across the restore, so the
|
||||
publish's WAL truncate can never succeed; without the short wait it would
|
||||
sit out the full CHECKPOINT_BUSY_MS (30 s) each time (#487).
|
||||
"""
|
||||
with (
|
||||
patch("database.execution_history.notify_history_restore"),
|
||||
patch.object(restore_stage, "CHECKPOINT_BUSY_MS", 50),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
class HistoryAPITests(ImageAPITestCase):
|
||||
async def tearDownAsync(self):
|
||||
if hasattr(self.api, "close_history_reads"):
|
||||
@@ -209,11 +191,7 @@ class HistoryAPITests(ImageAPITestCase):
|
||||
context["evidence"]["positive"]["raw"]["provenance"], "recorded"
|
||||
)
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
# An edit changes the text and its dedup hash together (#460)
|
||||
conn.execute(
|
||||
"UPDATE prompts SET text='edited',hash=? WHERE id=?",
|
||||
(generate_prompt_hash("edited"), positive),
|
||||
)
|
||||
conn.execute("UPDATE prompts SET text='edited' WHERE id=?", (positive,))
|
||||
_, changed = await self.detail(start.run_id)
|
||||
self.assertEqual(changed["contexts"][0]["positive"], context["positive"])
|
||||
self.assertFalse(changed["contexts"][0]["navigation"]["positive_available"])
|
||||
@@ -362,7 +340,7 @@ class HistoryAPITests(ImageAPITestCase):
|
||||
|
||||
def during_snapshot(*args, **kwargs):
|
||||
result = original(*args, **kwargs)
|
||||
with no_wal_truncate_wait():
|
||||
with patch("database.execution_history.notify_history_restore"):
|
||||
writer.model.restore_from_file(source)
|
||||
return result
|
||||
|
||||
@@ -906,7 +884,7 @@ class HistoryAPITests(ImageAPITestCase):
|
||||
|
||||
def during_snapshot(*args, **kwargs):
|
||||
result = original(*args, **kwargs)
|
||||
with no_wal_truncate_wait():
|
||||
with patch("database.execution_history.notify_history_restore"):
|
||||
writer.model.restore_from_file(source)
|
||||
return result
|
||||
|
||||
|
||||
@@ -1015,122 +1015,6 @@ class HistoryOutputAPITests(ImageAPITestCase):
|
||||
response = await self.current(descriptor["inspection_url"])
|
||||
self.assert_error(response, await response.json(), 422, "response-limit")
|
||||
|
||||
async def tag_image(self, relative, tags):
|
||||
"""Save *tags* on Images root 0's *relative* file through the Tag routes.
|
||||
|
||||
The real observe and save routes store the locator the Tag dialog
|
||||
stores (``media_capture._resolve``), so History must find it (#457).
|
||||
"""
|
||||
self.addCleanup(self.api.close_annotation_captures)
|
||||
roots = self.api._get_all_output_dirs()
|
||||
target = {
|
||||
"root_index": 0,
|
||||
"root_scope": self.api._gallery_root_scope(roots[0], 0),
|
||||
"path": relative,
|
||||
}
|
||||
response = await self.client.post(
|
||||
"/prompt_manager/image-annotations/observe", json={"target": target}
|
||||
)
|
||||
observed = await response.json()
|
||||
self.assertEqual(response.status, 200, observed)
|
||||
token = observed["capture"]["token"]
|
||||
try:
|
||||
response = await self.client.put(
|
||||
"/prompt_manager/image-annotations/tags",
|
||||
json={
|
||||
"target": target,
|
||||
"capture_token": token,
|
||||
"sha256": observed["capture"]["sha256"],
|
||||
"expected_epoch": observed["library_epoch"],
|
||||
"expected_tag_revision": observed["tag_revision"],
|
||||
"tags": list(tags),
|
||||
},
|
||||
)
|
||||
saved = await response.json()
|
||||
self.assertEqual(response.status, 200, saved)
|
||||
finally:
|
||||
await self.client.delete(
|
||||
f"/prompt_manager/image-annotations/capture/{token}"
|
||||
)
|
||||
return saved["tags"]
|
||||
|
||||
async def test_current_inspection_carries_the_files_image_tags(self):
|
||||
"""#457: History reads the file's own image tags, as Images does (S1)."""
|
||||
sub = self.output_dir / "batch"
|
||||
sub.mkdir()
|
||||
tagged = png(sub / "tagged.png", prompt=graph("Tagged file"))
|
||||
png(self.output_dir / "plain.png", prompt=graph("Plain file"))
|
||||
self.assertEqual(
|
||||
await self.tag_image("batch/tagged.png", ["dunes", "warm_light"]),
|
||||
["dunes", "warm_light"],
|
||||
)
|
||||
start = self.capture(
|
||||
[
|
||||
self.ref("1", subfolder="batch", filename="tagged.png"),
|
||||
self.ref("2", filename="plain.png"),
|
||||
]
|
||||
)
|
||||
_, page = await self.outputs(start)
|
||||
urls = [
|
||||
item["current_candidate"]["media"]["inspection_url"]
|
||||
for item in page["items"]
|
||||
]
|
||||
bodies = []
|
||||
for url in urls:
|
||||
response = await self.current(url)
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
bodies.append(body)
|
||||
self.assertEqual(
|
||||
bodies[0].get("image_tags"),
|
||||
{"state": "current", "tags": ["dunes", "warm_light"]},
|
||||
)
|
||||
# Image tags are the file's own search metadata, never prompt tags.
|
||||
self.assertEqual(bodies[0]["prompts"]["positive"]["text"], "Tagged file")
|
||||
self.assertEqual(bodies[1].get("image_tags"), {"state": "none", "tags": []})
|
||||
# New bytes at the same path: the tags belong to the old byte record.
|
||||
stat = tagged.stat()
|
||||
os.utime(tagged, ns=(stat.st_atime_ns, stat.st_mtime_ns + 10**9))
|
||||
response = await self.current(urls[0])
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
self.assertEqual(body["image_tags"], {"state": "none", "tags": []})
|
||||
|
||||
async def test_current_inspection_finds_image_tags_under_a_symlinked_output(self):
|
||||
"""An output folder reached through a link (another drive) still matches."""
|
||||
link = self.comfy_dir / "linked_output"
|
||||
self._make_symlink(self.output_dir, link, target_is_directory=True)
|
||||
self.api._get_all_output_dirs = lambda: [link]
|
||||
self.host_output = str(link)
|
||||
png(self.output_dir / "image.png", prompt=graph("Linked folder"))
|
||||
self.assertEqual(await self.tag_image("image.png", ["dunes"]), ["dunes"])
|
||||
start = self.capture([self.ref()])
|
||||
descriptor = await self.media(start)
|
||||
response = await self.current(descriptor["inspection_url"])
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
self.assertEqual(body["image_tags"], {"state": "current", "tags": ["dunes"]})
|
||||
|
||||
async def test_current_inspection_image_tag_failure_is_unavailable(self):
|
||||
png(self.output_dir / "image.png", prompt=graph("File evidence"))
|
||||
await self.tag_image("image.png", ["dunes"])
|
||||
start = self.capture([self.ref()])
|
||||
descriptor = await self.media(start)
|
||||
with (
|
||||
self.assertLogs(self.api.logger, "WARNING") as logs,
|
||||
patch.object(
|
||||
type(self.api.db),
|
||||
"current_image_tags",
|
||||
side_effect=sqlite3.OperationalError("locked"),
|
||||
),
|
||||
):
|
||||
response = await self.current(descriptor["inspection_url"])
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
self.assertEqual(body["image_tags"], {"state": "unavailable", "tags": []})
|
||||
self.assertEqual(body["prompts"]["positive"]["text"], "File evidence")
|
||||
self.assertNotIn(str(self.output_dir), "\n".join(logs.output))
|
||||
|
||||
async def test_optional_candidate_failure_preserves_every_occurrence_without_urls(
|
||||
self,
|
||||
):
|
||||
|
||||
@@ -35,7 +35,6 @@ from aiohttp import web # noqa: E402
|
||||
from aiohttp.test_utils import AioHTTPTestCase # noqa: E402
|
||||
from PIL import Image # noqa: E402
|
||||
|
||||
from database.models import PromptModel # noqa: E402
|
||||
from database.operations import PromptDatabase # noqa: E402
|
||||
import py.api.images as images_module
|
||||
from py.api import gallery_listing # noqa: E402
|
||||
@@ -93,41 +92,14 @@ class ImageAPITestCase(AioHTTPTestCase):
|
||||
self._anchor_patch.stop()
|
||||
# Windows refuses to unlink a database that still has open handles.
|
||||
self.api.db.close_all()
|
||||
try:
|
||||
leaked = self._close_leaked_connections()
|
||||
finally:
|
||||
for path in (
|
||||
self._temp_db.name,
|
||||
self._temp_db.name + "-wal",
|
||||
self._temp_db.name + "-shm",
|
||||
):
|
||||
if os.path.exists(path):
|
||||
os.unlink(path)
|
||||
self._output_tmp.cleanup()
|
||||
if leaked:
|
||||
raise AssertionError(
|
||||
f"{leaked} connection(s) to the test database were still open at "
|
||||
"teardown; Windows cannot unlink it (WinError 32). Close every "
|
||||
"extra PromptDatabase before tearDownAsync, not in a cleanup."
|
||||
)
|
||||
|
||||
def _close_leaked_connections(self):
|
||||
"""Close and count connections other models still hold on the temp DB.
|
||||
|
||||
Linux unlinks an open SQLite file, Windows does not; counting here makes
|
||||
a handle left open past teardown fail on every OS (#475).
|
||||
"""
|
||||
key = os.path.normcase(os.path.realpath(self._temp_db.name))
|
||||
with PromptModel._instances_lock:
|
||||
models = list(PromptModel._instances)
|
||||
leaked = 0
|
||||
for model in models:
|
||||
if model._schema_key() != key:
|
||||
continue
|
||||
with model._conn_lock:
|
||||
leaked += len(model._connections)
|
||||
model.close_all()
|
||||
return leaked
|
||||
for path in (
|
||||
self._temp_db.name,
|
||||
self._temp_db.name + "-wal",
|
||||
self._temp_db.name + "-shm",
|
||||
):
|
||||
if os.path.exists(path):
|
||||
os.unlink(path)
|
||||
self._output_tmp.cleanup()
|
||||
|
||||
# -- helpers ---------------------------------------------------------
|
||||
|
||||
|
||||
@@ -38,25 +38,11 @@ def _payload(token):
|
||||
|
||||
class TestRestartKeepsTheOpenPage(ImageAPITestCase):
|
||||
async def get_application(self):
|
||||
self._restarts = []
|
||||
app = await super().get_application()
|
||||
for index in range(3):
|
||||
make_png(self.output_dir / f"image_{index}.png")
|
||||
return app
|
||||
|
||||
async def tearDownAsync(self):
|
||||
# Before the base teardown unlinks the database: cleanups run after
|
||||
# tearDownAsync, and Windows cannot unlink a file still open (#475).
|
||||
errors = []
|
||||
while self._restarts:
|
||||
try:
|
||||
await self._restarts.pop()()
|
||||
except Exception as error: # every closer runs; the first error wins
|
||||
errors.append(error)
|
||||
await super().tearDownAsync()
|
||||
if errors:
|
||||
raise errors[0]
|
||||
|
||||
async def first_page(self, client=None, **params):
|
||||
response = await (client or self.client).get(
|
||||
LISTING, params={"mode": "workspace", "limit": "1", **params}
|
||||
@@ -81,7 +67,7 @@ class TestRestartKeepsTheOpenPage(ImageAPITestCase):
|
||||
api.close_auto_thumbnails()
|
||||
api.db.close_all()
|
||||
|
||||
self._restarts.append(close)
|
||||
self.addAsyncCleanup(close)
|
||||
return api, client
|
||||
|
||||
async def test_a_restart_keeps_every_held_scope_and_cursor_valid(self):
|
||||
@@ -166,13 +152,8 @@ class TestRestartKeepsTheOpenPage(ImageAPITestCase):
|
||||
other_db = self.comfy_dir / "elsewhere" / "prompts.db"
|
||||
other_db.parent.mkdir()
|
||||
api = PromptManagerAPI(db=PromptDatabase(str(other_db)))
|
||||
|
||||
async def close():
|
||||
api.close_auto_thumbnails()
|
||||
api.db.close_all()
|
||||
|
||||
# Closed before teardown removes the folder that holds its database.
|
||||
self._restarts.append(close)
|
||||
self.addCleanup(api.db.close_all)
|
||||
self.addCleanup(api.close_auto_thumbnails)
|
||||
self.assertNotEqual(
|
||||
api._gallery_root_scope(self.output_dir, 0), page["images"][0]["root_scope"]
|
||||
)
|
||||
|
||||
@@ -8,7 +8,6 @@ import contextlib
|
||||
import csv
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
@@ -16,18 +15,7 @@ import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, _REPO_ROOT)
|
||||
|
||||
# Run in a child interpreter: import py.autotag with a stub ComfyUI ``server``
|
||||
# and print every ML backend that the import pulled in.
|
||||
_IMPORT_AUTOTAG_REPORT_BACKENDS = """
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
sys.modules["server"] = MagicMock()
|
||||
import py.autotag
|
||||
print(*sorted({"onnxruntime", "llama_cpp"} & set(sys.modules)))
|
||||
"""
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from PIL import Image
|
||||
|
||||
@@ -154,26 +142,17 @@ class TestModelInjection(EngineTestCase):
|
||||
self.assertEqual(service._model_loader, service._load_model)
|
||||
|
||||
def test_importing_autotag_does_not_import_ml_backends(self):
|
||||
# A fresh interpreter: earlier tests in this process may have imported
|
||||
# an installed backend, so only a clean import can prove this (#472).
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", _IMPORT_AUTOTAG_REPORT_BACKENDS],
|
||||
cwd=_REPO_ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.split(), [])
|
||||
for heavy in ("onnxruntime", "llama_cpp"):
|
||||
self.assertNotIn(heavy, sys.modules)
|
||||
|
||||
def test_default_wd14_loader_reports_missing_onnxruntime(self):
|
||||
_touch_model_files(self.models_dir, "wd14-vit")
|
||||
service = AutoTagService(models_dir=self.models_dir)
|
||||
if "onnxruntime" in sys.modules:
|
||||
self.skipTest("onnxruntime is installed in this environment")
|
||||
|
||||
# A None entry makes onnxruntime unimportable even where it is installed.
|
||||
with _patch_modules({"onnxruntime": None}):
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
service.load_model("wd14-vit")
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
service.load_model("wd14-vit")
|
||||
|
||||
self.assertIn("onnxruntime", str(ctx.exception))
|
||||
self.assertFalse(service.is_model_loaded())
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
"""#424: each tagging backend reports the Python packages it cannot import.
|
||||
|
||||
``find_spec`` is patched throughout, and a backend an earlier test imported is
|
||||
hidden where it matters, so these pass with or without the ML packages (#472).
|
||||
``find_spec`` is patched throughout: no ML backend is installed or imported.
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -76,13 +75,8 @@ class TestMissingPackages(unittest.TestCase):
|
||||
def test_find_spec_errors_count_as_missing(self):
|
||||
for error in (ValueError("no spec"), ImportError("broken"), OSError("io")):
|
||||
with self.subTest(error=type(error).__name__):
|
||||
# Hide an onnxruntime an earlier test imported, so the check
|
||||
# reaches find_spec (#472).
|
||||
with (
|
||||
_patch_modules({"onnxruntime": None}),
|
||||
patch.object(
|
||||
autotag.importlib.util, "find_spec", side_effect=error
|
||||
),
|
||||
with patch.object(
|
||||
autotag.importlib.util, "find_spec", side_effect=error
|
||||
):
|
||||
self.assertEqual(missing_packages("wd14-vit"), ["onnxruntime"])
|
||||
|
||||
|
||||
@@ -1,526 +0,0 @@
|
||||
"""#443 (D152): delete selected Library prompts; paired ones are archived.
|
||||
|
||||
The DB commands, and what every reader does with an archived parent.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from database.execution_history_schema import rotate_history_epoch
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.library_folders import folder_prefixes
|
||||
from database.operations import PromptDatabase
|
||||
from database.prompt_archive import TABLE as ARCHIVE
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
|
||||
def save(db, text, **metadata):
|
||||
return db.save_prompt(text, prompt_hash=generate_prompt_hash(text), **metadata)
|
||||
|
||||
|
||||
def rows(db, sql, params=()):
|
||||
with db.model.get_connection() as conn:
|
||||
return [tuple(row) for row in conn.execute(sql, params).fetchall()]
|
||||
|
||||
|
||||
def archived_ids(db):
|
||||
return sorted(row[0] for row in rows(db, f"SELECT prompt_id FROM {ARCHIVE}"))
|
||||
|
||||
|
||||
def prompt_ids(db):
|
||||
return sorted(row[0] for row in rows(db, "SELECT id FROM prompts"))
|
||||
|
||||
|
||||
_STAMP = re.compile(r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}")
|
||||
|
||||
|
||||
def _stable(value):
|
||||
return "<time>" if isinstance(value, str) and _STAMP.match(value) else value
|
||||
|
||||
|
||||
def table_dump(db):
|
||||
"""Every row of every table (times masked), to compare two databases."""
|
||||
tables = [
|
||||
row[0]
|
||||
for row in rows(
|
||||
db,
|
||||
"SELECT name FROM sqlite_master WHERE type='table' "
|
||||
"AND name NOT LIKE 'sqlite_%' AND name!='execution_history_metadata' "
|
||||
"ORDER BY name",
|
||||
)
|
||||
]
|
||||
dump = {}
|
||||
for table in tables:
|
||||
dump[table] = sorted(
|
||||
(tuple(map(_stable, row)) for row in rows(db, f"SELECT * FROM {table}")),
|
||||
key=repr,
|
||||
)
|
||||
return dump
|
||||
|
||||
|
||||
class LibraryDeleteCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
# Cleanups run last-in first-out and after every cleanup a test adds,
|
||||
# so a database a test opens (and closes via addCleanup) is closed
|
||||
# before the folder goes; Windows cannot delete an open file (#510).
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
self.db = PromptDatabase(str(self.root / "delete.db"))
|
||||
self.addCleanup(self.db.close_all)
|
||||
self.epoch = self.db.get_library_options()["library_epoch"]
|
||||
|
||||
def paired(self, positive, negative="blur"):
|
||||
pair = self.db.save_prompt_pair(positive, negative)
|
||||
found = self.db.get_prompt_pair(pair)
|
||||
return found["positive_prompt_id"], found["negative_prompt_id"], pair
|
||||
|
||||
def image(self, prompt_id, name):
|
||||
path = self.root / name
|
||||
path.write_bytes(b"png")
|
||||
return self.db.link_image_to_prompt(prompt_id, str(path))
|
||||
|
||||
def apply(self, ids, epoch=None, review=None):
|
||||
"""Review ``ids`` (or use ``review``), then write what was reviewed."""
|
||||
review = review or self.classify(ids)
|
||||
return self.db.archive_or_delete_library_prompts(
|
||||
review["archive"],
|
||||
review["delete"],
|
||||
expected_epoch=epoch or self.epoch,
|
||||
candidates=review["fingerprints"],
|
||||
)
|
||||
|
||||
def classify(self, ids, epoch=None):
|
||||
return self.db.classify_library_delete(ids, expected_epoch=epoch or self.epoch)
|
||||
|
||||
def restore(self, ids, epoch=None):
|
||||
return self.db.restore_library_prompts(ids, expected_epoch=epoch or self.epoch)
|
||||
|
||||
|
||||
class TestClassifyAndApply(LibraryDeleteCase):
|
||||
def test_each_class_in_order(self):
|
||||
plain = save(self.db, "Plain")
|
||||
protected = save(self.db, "Guarded", tags=["__protected__"])
|
||||
protected_paired, _neg, _pair = self.paired("Guarded pair")
|
||||
self.db.bulk_add_tags([protected_paired], ["__protected__"])
|
||||
positive, negative_only, _pair = self.paired("Cat", "only negative")
|
||||
both = save(self.db, "Both roles")
|
||||
self.image(both, "both.png") # a parent through its image
|
||||
self.db.save_prompt_pair("Other", "Both roles")
|
||||
archived_pair_positive, _n, old_pair = self.paired("Old")
|
||||
self.db.archive_prompt_pair(old_pair, True)
|
||||
already, _n2, _p2 = self.paired("Already")
|
||||
self.apply([already])
|
||||
self.image(plain, "plain.png")
|
||||
ids = [
|
||||
plain,
|
||||
protected,
|
||||
protected_paired,
|
||||
positive,
|
||||
negative_only,
|
||||
both,
|
||||
archived_pair_positive,
|
||||
already,
|
||||
99999,
|
||||
]
|
||||
result = self.classify(ids)
|
||||
self.assertEqual(
|
||||
result,
|
||||
{
|
||||
"library_epoch": self.epoch,
|
||||
"delete": [plain],
|
||||
"archive": [positive, both, archived_pair_positive],
|
||||
"protected": [protected, protected_paired],
|
||||
"already_archived": [already],
|
||||
"not_found": [negative_only, 99999],
|
||||
"image_links": 1,
|
||||
"fingerprints": result["fingerprints"],
|
||||
},
|
||||
)
|
||||
self.assertEqual(list(result["fingerprints"]), [plain])
|
||||
|
||||
def test_apply_archives_paired_deletes_unpaired_and_skips_protected(self):
|
||||
plain = save(self.db, "Plain", tags=["red"])
|
||||
protected = save(self.db, "Guarded", tags=["__protected__"])
|
||||
positive, negative, pair = self.paired("Cat")
|
||||
self.image(plain, "a.png")
|
||||
self.image(positive, "b.png")
|
||||
before = rows(self.db, "SELECT updated_at FROM prompts WHERE id=?", (positive,))
|
||||
result = self.apply([plain, protected, positive, 99999])
|
||||
self.assertEqual(
|
||||
result,
|
||||
{
|
||||
"library_epoch": self.epoch,
|
||||
"deleted": 1,
|
||||
"archived": 1,
|
||||
"already_archived": 0,
|
||||
"not_found": 0,
|
||||
"changed": 0,
|
||||
"image_links": 1,
|
||||
},
|
||||
)
|
||||
self.assertEqual(prompt_ids(self.db), sorted([protected, positive, negative]))
|
||||
self.assertEqual(archived_ids(self.db), [positive])
|
||||
# Archive keeps everything: the pair, its image link and the prompt row.
|
||||
self.assertEqual(
|
||||
len(rows(self.db, "SELECT 1 FROM prompt_pairs WHERE id=?", (pair,))), 1
|
||||
)
|
||||
self.assertEqual(
|
||||
len(
|
||||
rows(
|
||||
self.db,
|
||||
"SELECT 1 FROM generated_images WHERE prompt_id=?",
|
||||
(positive,),
|
||||
)
|
||||
),
|
||||
1,
|
||||
)
|
||||
after = rows(self.db, "SELECT updated_at FROM prompts WHERE id=?", (positive,))
|
||||
self.assertNotEqual(before, after)
|
||||
# The image file stays on disk.
|
||||
self.assertTrue((self.root / "a.png").exists())
|
||||
|
||||
def test_delete_has_exactly_the_legacy_row_effects(self):
|
||||
twin = PromptDatabase(str(self.root / "twin.db"))
|
||||
self.addCleanup(twin.close_all)
|
||||
twin_epoch = twin.get_library_options()["library_epoch"]
|
||||
for db in (self.db, twin):
|
||||
a = save(db, "Alpha", tags=["x", "y"], category="c")
|
||||
save(db, "Beta")
|
||||
path = self.root / "shared.png"
|
||||
path.write_bytes(b"png")
|
||||
db.link_image_to_prompt(a, str(path), {"workflow": {"nodes": []}})
|
||||
self.db.bulk_delete_prompts([a])
|
||||
twin.archive_or_delete_library_prompts(
|
||||
[], [a], expected_epoch=twin_epoch, candidates=None
|
||||
)
|
||||
self.assertEqual(table_dump(self.db), table_dump(twin))
|
||||
|
||||
def test_already_archived_and_restore(self):
|
||||
positive, _neg, _pair = self.paired("Cat")
|
||||
self.apply([positive])
|
||||
again = self.classify([positive])
|
||||
self.assertEqual(
|
||||
(again["archive"], again["already_archived"]), ([], [positive])
|
||||
)
|
||||
# Archived between the review and the write: counted, not archived twice.
|
||||
other, _n, _p = self.paired("Dog")
|
||||
review = self.classify([other])
|
||||
self.apply([other])
|
||||
late = self.apply([], review=review)
|
||||
self.assertEqual((late["archived"], late["already_archived"]), (0, 1))
|
||||
plain = save(self.db, "Plain")
|
||||
result = self.restore([positive, plain, 99999])
|
||||
self.assertEqual(
|
||||
result,
|
||||
{
|
||||
"library_epoch": self.epoch,
|
||||
"restored": 1,
|
||||
"unchanged": 1,
|
||||
"not_found": 1,
|
||||
},
|
||||
)
|
||||
self.assertEqual(archived_ids(self.db), [other])
|
||||
|
||||
def test_the_write_executes_only_what_was_reviewed(self):
|
||||
plain = save(self.db, "Plain")
|
||||
guarded = save(self.db, "Guarded later")
|
||||
touched = save(self.db, "Touched later")
|
||||
review = self.classify([plain, guarded, touched])
|
||||
# Between the review and the write: one becomes protected, one is used.
|
||||
self.db.bulk_add_tags([guarded], ["__protected__"])
|
||||
self.db.record_prompt_use(touched)
|
||||
result = self.apply([], review=review)
|
||||
self.assertEqual((result["deleted"], result["changed"]), (1, 2))
|
||||
self.assertEqual(prompt_ids(self.db), sorted([guarded, touched]))
|
||||
self.assertEqual(archived_ids(self.db), [])
|
||||
|
||||
def test_a_new_image_link_after_the_review_keeps_the_prompt(self):
|
||||
plain = save(self.db, "Plain")
|
||||
review = self.classify([plain])
|
||||
self.image(plain, "late.png")
|
||||
self.assertEqual(self.apply([], review=review)["changed"], 1)
|
||||
self.assertIn(plain, prompt_ids(self.db))
|
||||
|
||||
def test_reviewed_deletes_without_a_backup_are_kept(self):
|
||||
plain = save(self.db, "Plain")
|
||||
review = self.classify([plain])
|
||||
result = self.db.archive_or_delete_library_prompts(
|
||||
[], [plain], expected_epoch=self.epoch, candidates={}
|
||||
)
|
||||
self.assertEqual((result["deleted"], result["changed"]), (0, 1))
|
||||
self.assertEqual(review["delete"], [plain])
|
||||
|
||||
def test_a_reviewed_delete_that_became_paired_is_archived(self):
|
||||
plain = save(self.db, "Plain")
|
||||
review = self.classify([plain])
|
||||
self.db.save_prompt_pair("Plain", "late negative")
|
||||
result = self.apply([], review=review)
|
||||
self.assertEqual((result["deleted"], result["archived"]), (0, 1))
|
||||
|
||||
def test_a_reviewed_archive_that_lost_its_pair_is_archived_not_deleted(self):
|
||||
positive, _negative, pair = self.paired("Cat")
|
||||
review = self.classify([positive])
|
||||
self.db.delete_prompt_pair(pair)
|
||||
result = self.apply([], review=review)
|
||||
self.assertEqual((result["deleted"], result["archived"]), (0, 1))
|
||||
self.assertEqual(archived_ids(self.db), [positive])
|
||||
|
||||
def test_one_id_in_both_lists_or_no_ids_is_refused(self):
|
||||
plain = save(self.db, "Plain")
|
||||
for archive, delete in (([plain], [plain]), ([], [])):
|
||||
with self.subTest(archive=archive, delete=delete):
|
||||
with self.assertRaises(ValueError):
|
||||
self.db.archive_or_delete_library_prompts(
|
||||
archive, delete, expected_epoch=self.epoch, candidates=None
|
||||
)
|
||||
|
||||
def test_archiving_repairs_a_missing_or_stale_hash(self):
|
||||
missing, _n1, _p1 = self.paired("Hashless")
|
||||
stale, _n2, _p2 = self.paired("Stale hash")
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute("UPDATE prompts SET hash=NULL WHERE id=?", (missing,))
|
||||
conn.execute("UPDATE prompts SET hash='stale' WHERE id=?", (stale,))
|
||||
self.apply([missing, stale])
|
||||
for pid, text in ((missing, "Hashless"), (stale, "Stale hash")):
|
||||
self.assertEqual(
|
||||
rows(self.db, "SELECT hash FROM prompts WHERE id=?", (pid,)),
|
||||
[(generate_prompt_hash(text),)],
|
||||
)
|
||||
# So every ingestion lookup finds the archived row.
|
||||
self.assertEqual(save(self.db, text), pid)
|
||||
|
||||
def test_a_canonical_hash_another_row_holds_is_left_alone(self):
|
||||
stale, _n, _p = self.paired("Twin text")
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute("UPDATE prompts SET hash='stale' WHERE id=?", (stale,))
|
||||
twin = save(self.db, "twin text")
|
||||
self.apply([stale])
|
||||
self.assertEqual(
|
||||
rows(self.db, "SELECT hash FROM prompts WHERE id=?", (stale,)), [("stale",)]
|
||||
)
|
||||
self.assertEqual(save(self.db, "Twin text"), twin)
|
||||
|
||||
def test_restore_unhides_protected_and_non_parent_archived_prompts(self):
|
||||
guarded, _n, _p = self.paired("Guarded")
|
||||
self.apply([guarded])
|
||||
self.db.bulk_add_tags([guarded], ["__protected__"])
|
||||
negative_only = save(self.db, "Only a negative")
|
||||
self.db.save_prompt_pair("Owner", "Only a negative")
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
f"INSERT INTO {ARCHIVE} VALUES (?, '2026-10-28T00:00:00+00:00')",
|
||||
(negative_only,),
|
||||
)
|
||||
result = self.restore([guarded, negative_only])
|
||||
self.assertEqual(result["restored"], 2)
|
||||
self.assertEqual(archived_ids(self.db), [])
|
||||
|
||||
def test_restored_storage_changes_nothing(self):
|
||||
plain = save(self.db, "Plain")
|
||||
positive, _neg, _pair = self.paired("Cat")
|
||||
rotate_history_epoch(self.db.model.get_connection())
|
||||
for call in (
|
||||
lambda: self.apply([plain, positive]),
|
||||
lambda: self.classify([plain]),
|
||||
lambda: self.restore([positive]),
|
||||
):
|
||||
with self.assertRaises(LibraryCommandError) as caught:
|
||||
call()
|
||||
self.assertEqual(caught.exception.code, "storage-changed")
|
||||
self.assertIn(plain, prompt_ids(self.db))
|
||||
self.assertEqual(archived_ids(self.db), [])
|
||||
|
||||
def test_a_failure_mid_write_rolls_everything_back(self):
|
||||
plain = save(self.db, "Plain")
|
||||
positive, _neg, _pair = self.paired("Cat")
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"CREATE TRIGGER fail_delete BEFORE DELETE ON prompts "
|
||||
"BEGIN SELECT RAISE(ABORT, 'boom'); END"
|
||||
)
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
self.apply([positive, plain])
|
||||
self.assertIn(plain, prompt_ids(self.db))
|
||||
self.assertEqual(archived_ids(self.db), [])
|
||||
|
||||
def test_many_ids_are_handled_in_batches(self):
|
||||
ids = [save(self.db, f"Many {i}") for i in range(1000)]
|
||||
result = self.apply(ids)
|
||||
self.assertEqual(result["deleted"], 1000)
|
||||
|
||||
def test_rejects_invalid_arguments(self):
|
||||
plain = save(self.db, "Plain")
|
||||
for ids, epoch in (([], self.epoch), ([True], self.epoch), ([plain], "x")):
|
||||
for call in (self.apply, self.classify, self.restore):
|
||||
with self.subTest(ids=ids, epoch=epoch, call=call.__name__):
|
||||
with self.assertRaises(ValueError):
|
||||
call(ids, epoch)
|
||||
|
||||
|
||||
class TestArchivedReaders(LibraryDeleteCase):
|
||||
"""What each reader does with an archived parent (design §4.4)."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.kept, _kn, _kp = self.paired("Kept cat", "kept blur")
|
||||
self.gone, _gn, _gp = self.paired("Gone cat", "gone blur")
|
||||
for pid, category, tag in ((self.kept, "keep", "k"), (self.gone, "gone", "g")):
|
||||
self.db.update_prompt_metadata(pid, category=category, tags=[tag])
|
||||
self.apply([self.gone])
|
||||
|
||||
def ids_of(self, page):
|
||||
return [item["id"] for item in page["items"]]
|
||||
|
||||
def test_listing_search_and_totals(self):
|
||||
active = self.db.list_library()
|
||||
self.assertEqual(self.ids_of(active), [self.kept])
|
||||
self.assertEqual(
|
||||
(active["total"], active["variant_total"], active["archived_total"]),
|
||||
(1, 1, 1),
|
||||
)
|
||||
archived = self.db.list_library(shelf="archived")
|
||||
self.assertEqual(self.ids_of(archived), [self.gone])
|
||||
self.assertEqual((archived["total"], archived["variant_total"]), (1, 1))
|
||||
self.assertEqual(self.ids_of(self.db.list_library(text="Gone")), [])
|
||||
self.assertEqual(
|
||||
self.ids_of(self.db.list_library(text="Gone", shelf="archived")),
|
||||
[self.gone],
|
||||
)
|
||||
self.assertEqual(self.ids_of(self.db.list_library(tags=["g"])), [])
|
||||
self.assertEqual(self.db.list_library(category="gone")["archived_total"], 1)
|
||||
with self.assertRaises(ValueError):
|
||||
self.db.list_library(shelf="all")
|
||||
|
||||
def test_the_444_filters_and_tag_counts_hide_it(self):
|
||||
# #444's folder filter, tag match and tag ranking keep #443's shelves.
|
||||
bare, _bn, _bp = self.paired("Bare cat", "bare blur")
|
||||
self.apply([bare])
|
||||
folder = self.root / "out" / "cats"
|
||||
folder.mkdir(parents=True)
|
||||
for pid, name in ((self.kept, "kept.png"), (self.gone, "gone.png")):
|
||||
self.image(pid, str(Path("out") / "cats" / name))
|
||||
prefixes = folder_prefixes([str(self.root / "out")], "cats")
|
||||
self.assertEqual(
|
||||
self.ids_of(self.db.list_library(folder_prefixes=prefixes)), [self.kept]
|
||||
)
|
||||
self.assertEqual(
|
||||
self.ids_of(
|
||||
self.db.list_library(folder_prefixes=prefixes, shelf="archived")
|
||||
),
|
||||
[self.gone],
|
||||
)
|
||||
self.assertEqual(
|
||||
self.ids_of(self.db.list_library(tags=["k", "g"], tag_mode="any")),
|
||||
[self.kept],
|
||||
)
|
||||
self.assertEqual(self.ids_of(self.db.list_library(tag_mode="untagged")), [])
|
||||
self.assertEqual(
|
||||
self.ids_of(self.db.list_library(tag_mode="untagged", shelf="archived")),
|
||||
[bare],
|
||||
)
|
||||
tags = self.db.list_vocabulary_tags(sort="prompts")
|
||||
self.assertEqual(
|
||||
{row["name"]: row["prompt_count"] for row in tags["tags"]},
|
||||
{"k": 1, "g": 0},
|
||||
)
|
||||
self.assertEqual(tags["untagged_count"], 0)
|
||||
self.assertEqual(
|
||||
self.db.list_vocabulary_tags(search="g", sort="prompts")["tags"][0][
|
||||
"prompt_count"
|
||||
],
|
||||
0,
|
||||
)
|
||||
|
||||
def test_detail_and_options(self):
|
||||
self.assertTrue(self.db.get_library_positive(self.gone)["positive"]["archived"])
|
||||
self.assertFalse(
|
||||
self.db.get_library_positive(self.kept)["positive"]["archived"]
|
||||
)
|
||||
# Suggestions keep archived parents' categories: the archived view
|
||||
# filters by them.
|
||||
self.assertEqual(self.db.get_library_options()["categories"], ["gone", "keep"])
|
||||
|
||||
def test_counts(self):
|
||||
self.assertEqual(self.db.library_counts()["positive_prompts"], 1)
|
||||
self.assertEqual(self.db.library_counts()["negative_variants"], 1)
|
||||
tags = {
|
||||
row["name"]: row["prompt_count"]
|
||||
for row in self.db.list_vocabulary_tags()["tags"]
|
||||
}
|
||||
self.assertEqual((tags["k"], tags["g"]), (1, 0))
|
||||
categories = {
|
||||
row["value"]: row["prompt_count"]
|
||||
for row in self.db.list_vocabulary_categories()["categories"]
|
||||
}
|
||||
# A value only archived parents hold stays listed with 0, as a tag does.
|
||||
self.assertEqual(categories, {"keep": 1, "gone": 0})
|
||||
|
||||
def test_restore_summary_and_export(self):
|
||||
from database.restore_stage import library_summary
|
||||
|
||||
summary = library_summary(self.db.model.get_connection())
|
||||
self.assertEqual(summary["positives"], 1)
|
||||
exported = {
|
||||
item["positive_text"]: item["archived"]
|
||||
for item in self.db.export_library()["positives"]
|
||||
}
|
||||
self.assertEqual(exported, {"Kept cat": False, "Gone cat": True})
|
||||
|
||||
def test_usage_statistics_keep_archived_runs(self):
|
||||
self.db.record_prompt_use(self.gone, 3)
|
||||
runs = {row["category"]: row["runs"] for row in self.db.usage_by_category()}
|
||||
self.assertEqual(runs["gone"], 3)
|
||||
|
||||
def test_runs_node_saves_and_new_prompt_find_the_archived_row(self):
|
||||
count = len(prompt_ids(self.db))
|
||||
self.assertTrue(self.db.record_prompt_use(self.gone, 2))
|
||||
existing = self.db.get_prompt_by_hash(generate_prompt_hash("Gone cat"))
|
||||
self.assertEqual(existing["id"], self.gone)
|
||||
self.assertEqual(save(self.db, "Gone cat"), self.gone)
|
||||
self.db.save_prompt_pair("Gone cat", "new blur")
|
||||
self.assertEqual(len(prompt_ids(self.db)), count + 1) # only the new negative
|
||||
self.assertEqual(archived_ids(self.db), [self.gone])
|
||||
|
||||
def test_legacy_reads_still_list_it_and_legacy_delete_still_refuses(self):
|
||||
from database.pairs import PairDependencyError
|
||||
|
||||
texts = [p["text"] for p in self.db.get_recent_prompts(limit=50)["prompts"]]
|
||||
self.assertIn("Gone cat", texts)
|
||||
with self.assertRaises(PairDependencyError):
|
||||
self.db.bulk_delete_prompts([self.gone])
|
||||
|
||||
def test_as_another_parents_negative_it_stays_a_visible_variant(self):
|
||||
owner, _neg, pair = self.paired("Owner cat", "Gone cat")
|
||||
detail = self.db.get_library_positive(owner)
|
||||
self.assertEqual([v["id"] for v in detail["variants"]], [pair])
|
||||
self.assertEqual(self.ids_of(self.db.list_library(text="Gone")), [owner])
|
||||
|
||||
def test_maintenance_never_prunes_an_archived_prompt(self):
|
||||
lonely = save(self.db, "Lonely")
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
f"INSERT INTO {ARCHIVE} VALUES (?, '2026-10-28T00:00:00+00:00')",
|
||||
(lonely,),
|
||||
)
|
||||
orphaned = [row[0] for row in conn.execute(self.orphaned_sql()).fetchall()]
|
||||
self.assertNotIn(lonely, orphaned)
|
||||
|
||||
@staticmethod
|
||||
def orphaned_sql():
|
||||
from database.maintenance import ORPHANED_PROMPTS_SQL
|
||||
|
||||
return ORPHANED_PROMPTS_SQL
|
||||
|
||||
def test_bulk_metadata_writes_still_reach_an_archived_parent(self):
|
||||
result = self.db.bulk_set_library_category(
|
||||
[self.gone], "moved", expected_epoch=self.epoch
|
||||
)
|
||||
self.assertEqual(result["updated"], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -59,28 +59,6 @@ class TestGalleryConfig(unittest.TestCase):
|
||||
self.assertFalse(GalleryConfig.MONITORING_ENABLED)
|
||||
self.assertEqual(GalleryConfig.PROCESSING_DELAY, 5.0)
|
||||
|
||||
def test_save_foreign_runs_defaults_on_and_only_booleans_apply(self):
|
||||
"""#247: gallery.monitoring.save_foreign_runs, default true."""
|
||||
self.assertIs(
|
||||
GalleryConfig.get_config()["monitoring"]["save_foreign_runs"], True
|
||||
)
|
||||
GalleryConfig.update_config({"monitoring": {"save_foreign_runs": False}})
|
||||
self.assertIs(GalleryConfig.SAVE_FOREIGN_RUNS, False)
|
||||
for junk in ("true", 1, None):
|
||||
with self.subTest(value=junk):
|
||||
with self.assertLogs("prompt_manager.config", "WARNING") as logs:
|
||||
GalleryConfig.update_config(
|
||||
{"monitoring": {"save_foreign_runs": junk}}
|
||||
)
|
||||
self.assertIn(
|
||||
"Ignoring gallery monitoring save_foreign_runs: expected true "
|
||||
"or false",
|
||||
"\n".join(logs.output),
|
||||
)
|
||||
self.assertIs(GalleryConfig.SAVE_FOREIGN_RUNS, False)
|
||||
GalleryConfig.update_config({"monitoring": {"save_foreign_runs": True}})
|
||||
self.assertIs(GalleryConfig.SAVE_FOREIGN_RUNS, True)
|
||||
|
||||
def test_update_tracking(self):
|
||||
GalleryConfig.update_config({"tracking": {"prompt_timeout": 300}})
|
||||
self.assertEqual(GalleryConfig.PROMPT_TIMEOUT, 300)
|
||||
|
||||
@@ -15,7 +15,6 @@ from database import execution_contexts
|
||||
from database.execution_history import HistoryEpochChanged
|
||||
from database.execution_context_schema import context_schema_error
|
||||
from database.execution_contexts import LIVE_HINT_PROMPT_IDS_SQL
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from database.execution_history_schema import rotate_history_epoch
|
||||
from utils.execution_contexts import (
|
||||
ContextBundle,
|
||||
@@ -467,11 +466,7 @@ class ContextDatabaseTests(unittest.TestCase):
|
||||
with conn:
|
||||
conn.execute("DELETE FROM prompts WHERE id=3")
|
||||
self.assertEqual(assert_same(), {1, 2, 4})
|
||||
# A case-only edit keeps the prompt's hash, so the run still counts
|
||||
# toward it (#460); another text drops the hint.
|
||||
db.update_prompt_text(4, "Other")
|
||||
self.assertEqual(assert_same(), {1, 2, 4})
|
||||
db.update_prompt_text(4, "another")
|
||||
self.assertEqual(assert_same(), {1, 2})
|
||||
with conn:
|
||||
conn.execute("INSERT INTO prompts(id,text) VALUES(5,'none')")
|
||||
@@ -479,131 +474,6 @@ class ContextDatabaseTests(unittest.TestCase):
|
||||
rotate_history_epoch(conn)
|
||||
self.assertEqual(assert_same(), set())
|
||||
|
||||
def test_hints_resolve_the_stored_form_of_outer_whitespace_raw_text(self):
|
||||
# #438: the Library stores text.strip(); the raw run text keeps its spaces.
|
||||
db = self.database()
|
||||
conn = db.model.get_connection()
|
||||
with conn:
|
||||
conn.execute(
|
||||
"INSERT INTO prompts(id,text) VALUES(1,'positive'),(2,'negative')"
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) "
|
||||
"VALUES(7,1,2)"
|
||||
)
|
||||
start = self.capture(db, 0, bundle(" negative\n", positive="positive\n\t"))
|
||||
context = db.get_execution_contexts(start.run_id)["contexts"][0]
|
||||
self.assertEqual(context["positive"]["raw"], "positive\n\t")
|
||||
self.assertEqual(
|
||||
(
|
||||
context["positive_prompt_id"],
|
||||
context["negative_prompt_id"],
|
||||
context["pair_id"],
|
||||
),
|
||||
(1, 2, 7),
|
||||
)
|
||||
live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)}
|
||||
self.assertEqual(live, {1, 2})
|
||||
stored = conn.execute(
|
||||
"SELECT positive_prompt_id FROM execution_run_contexts WHERE run_id=?",
|
||||
(start.run_id,),
|
||||
).fetchone()[0]
|
||||
self.assertEqual(stored, 1)
|
||||
# A legacy row that kept outer whitespace keeps its exact-text hint and
|
||||
# with it the recorded-run protection from pruning.
|
||||
with conn:
|
||||
conn.execute("INSERT INTO prompts(id,text) VALUES(3,' legacy ')")
|
||||
legacy = self.capture(db, 1, bundle("negative", positive=" legacy "))
|
||||
hinted = db.get_execution_contexts(legacy.run_id)["contexts"][0]
|
||||
self.assertEqual(hinted["positive_prompt_id"], 3)
|
||||
live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)}
|
||||
self.assertEqual(live, {1, 2, 3})
|
||||
self.assertEqual(db.prune_orphaned_prompts(), 0)
|
||||
# Both forms present make the hint ambiguous, in both rule forms.
|
||||
with conn:
|
||||
conn.execute("INSERT INTO prompts(id,text) VALUES(4,'legacy')")
|
||||
hinted = db.get_execution_contexts(legacy.run_id)["contexts"][0]
|
||||
self.assertIsNone(hinted["positive_prompt_id"])
|
||||
live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)}
|
||||
self.assertEqual(live, {1, 2})
|
||||
|
||||
def test_hints_follow_the_hash_for_case_variant_raw_text(self):
|
||||
# #460: the run names the prompt the Library counted it toward.
|
||||
db = self.database()
|
||||
first = db.save_prompt(
|
||||
"\u00e9t\u00e9 positive",
|
||||
prompt_hash=generate_prompt_hash("\u00e9t\u00e9 positive"),
|
||||
)
|
||||
second = db.save_prompt(
|
||||
"negative \u03c3", prompt_hash=generate_prompt_hash("negative \u03c3")
|
||||
)
|
||||
with db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) "
|
||||
"VALUES(7,?,?)",
|
||||
(first, second),
|
||||
)
|
||||
start = self.capture(
|
||||
db, 0, bundle("NEGATIVE \u03a3", positive="\u00c9T\u00c9 Positive")
|
||||
)
|
||||
context = db.get_execution_contexts(start.run_id)["contexts"][0]
|
||||
self.assertEqual(
|
||||
(
|
||||
context["positive_prompt_id"],
|
||||
context["negative_prompt_id"],
|
||||
context["pair_id"],
|
||||
),
|
||||
(first, second, 7),
|
||||
)
|
||||
conn = db.model.get_connection()
|
||||
live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)}
|
||||
self.assertEqual(live, {first, second})
|
||||
# A prompt that only shares SQLite's ASCII lower() is not a match
|
||||
other = db.save_prompt("other", prompt_hash=generate_prompt_hash("other"))
|
||||
start = self.capture(db, 1, bundle("x", positive="\u00d6THER"))
|
||||
context = db.get_execution_contexts(start.run_id)["contexts"][0]
|
||||
self.assertIsNone(context["positive_prompt_id"])
|
||||
self.assertNotIn(
|
||||
other, {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)}
|
||||
)
|
||||
|
||||
def test_hash_is_only_a_fallback_so_a_text_hint_stays_unambiguous(self):
|
||||
# #460 review: a hash match must never make a #438 text hint ambiguous,
|
||||
# or its prompt would lose the recorded-run prune protection.
|
||||
db = self.database()
|
||||
conn = db.model.get_connection()
|
||||
with conn:
|
||||
# A legacy row without a hash, and a case variant with the hash
|
||||
conn.execute("INSERT INTO prompts(id,text) VALUES(1,'a lighthouse')")
|
||||
conn.execute(
|
||||
"INSERT INTO prompts(id,text,hash) VALUES(2,'A Lighthouse',?)",
|
||||
(generate_prompt_hash("a lighthouse"),),
|
||||
)
|
||||
exact = self.capture(db, 0, bundle("x", positive="a lighthouse"))
|
||||
variant = self.capture(db, 1, bundle("x", positive="A LIGHTHOUSE"))
|
||||
hints = [
|
||||
db.get_execution_contexts(start.run_id)["contexts"][0]["positive_prompt_id"]
|
||||
for start in (exact, variant)
|
||||
]
|
||||
self.assertEqual(hints, [1, 2])
|
||||
live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)}
|
||||
self.assertEqual(live, {1, 2})
|
||||
self.assertEqual(db.prune_orphaned_prompts(), 0)
|
||||
|
||||
def test_live_hint_prompt_lookups_use_indexes(self):
|
||||
db = self.database()
|
||||
conn = db.model.get_connection()
|
||||
plan = [
|
||||
row[3]
|
||||
for row in conn.execute("EXPLAIN QUERY PLAN " + LIVE_HINT_PROMPT_IDS_SQL)
|
||||
]
|
||||
prompt_steps = [
|
||||
step for step in plan if " p " in f" {step} " or "TABLE p" in step
|
||||
]
|
||||
self.assertTrue(prompt_steps, plan)
|
||||
self.assertEqual([step for step in plan if step.startswith("SCAN p")], [])
|
||||
self.assertEqual([step for step in plan if step.startswith("SCAN q")], [])
|
||||
|
||||
def test_hints_require_exact_unique_raw_text_and_never_refresh(self):
|
||||
db = self.database()
|
||||
conn = db.model.get_connection()
|
||||
@@ -634,13 +504,12 @@ class ContextDatabaseTests(unittest.TestCase):
|
||||
db.get_execution_contexts(fresh.run_id)["contexts"][0]["positive_prompt_id"]
|
||||
)
|
||||
|
||||
def test_null_negative_pair_only_matches_a_known_stored_empty(self):
|
||||
def test_null_negative_pair_only_matches_known_exact_empty(self):
|
||||
db = self.database()
|
||||
with db.model.get_connection() as conn:
|
||||
conn.execute("INSERT INTO prompts(id,text) VALUES(1,'positive')")
|
||||
conn.execute("INSERT INTO prompt_pairs(id,positive_prompt_id) VALUES(7,1)")
|
||||
# A whitespace-only negative is the stored empty one (#438)
|
||||
for item, raw, expected in ((0, "", 7), (1, " \n", 7), (2, None, None)):
|
||||
for item, raw, expected in ((0, "", 7), (1, " ", None), (2, None, None)):
|
||||
candidate = bundle(negative=raw or "")
|
||||
if raw is None:
|
||||
ctx = candidate.contexts[0]
|
||||
|
||||
@@ -19,7 +19,6 @@ from unittest.mock import patch
|
||||
from tests.history_waits import drain_writer
|
||||
from database import PromptDatabase
|
||||
from utils import usage_tracking
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.execution_history_writer import HistoryWriter
|
||||
|
||||
try:
|
||||
@@ -917,42 +916,3 @@ class CachedRuntimeHistoryTests(HooksCase):
|
||||
from database.run_attribution import named_prompt
|
||||
|
||||
self.assertIsNone(named_prompt(self.db, self.rows()[1]["run_id"]))
|
||||
|
||||
|
||||
class CachedRuntimeCaseHistoryTests(CachedRuntimeHistoryTests):
|
||||
"""Raw text that differs only in case joins the saved prompt's history (#460).
|
||||
|
||||
The hash folds case, so the run counts toward the lower-case prompt saved
|
||||
first; the run keeps its exact raw spelling.
|
||||
"""
|
||||
|
||||
TEXT = "\u00c9 LIGHTHOUSE the LLM wrote"
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.db.delete_prompt(self.prompt_id)
|
||||
lower = self.TEXT.lower()
|
||||
self.prompt_id = self.db.save_prompt(
|
||||
text=lower, prompt_hash=generate_prompt_hash(lower)
|
||||
)
|
||||
|
||||
def test_the_library_counts_the_case_variant_as_this_prompt(self):
|
||||
self.assertEqual(
|
||||
self.db.get_prompt_by_hash(generate_prompt_hash(self.TEXT))["id"],
|
||||
self.prompt_id,
|
||||
)
|
||||
|
||||
|
||||
class CachedRuntimeWhitespaceHistoryTests(CachedRuntimeHistoryTests):
|
||||
"""Raw text with outer whitespace joins the history of the prompt it saves (#438).
|
||||
|
||||
The Library stores ``"a lighthouse the LLM wrote"``; the run keeps the exact
|
||||
frozen raw text, newline included, yet is a tracked run of that prompt.
|
||||
"""
|
||||
|
||||
TEXT = "\u3000 a lighthouse the LLM wrote\n"
|
||||
|
||||
def test_the_library_stores_the_stripped_text(self):
|
||||
self.assertEqual(
|
||||
self.db.get_prompt_by_id(self.prompt_id)["text"], self.TEXT.strip()
|
||||
)
|
||||
|
||||
@@ -1,330 +0,0 @@
|
||||
"""#247 PR 2 x #90: the monitor's foreign path never stores switched-off text.
|
||||
|
||||
A file whose run's prompt has Save to library off is the opted-out path's,
|
||||
not the foreign path's; a foreign graph whose ladder decision is
|
||||
``not_saved`` (a switched-off node the #90 walk does not reach) stores
|
||||
nothing at settle, at the re-check after the seal, or from recorded history,
|
||||
and leaves no wait behind.
|
||||
"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from tests import saver_anchor_graphs as shapes
|
||||
from tests import test_run_attribution as attribution
|
||||
from tests import test_save_to_library as save_tests
|
||||
from tests.test_pair_scan import sampled_graph
|
||||
from utils.foreign_runs import ForeignRecheck
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from py.config import GalleryConfig # after tests: it stubs the host
|
||||
|
||||
LOGGER = "prompt_manager.image_monitor"
|
||||
OFF_TEXT = "SECRET OFF TEXT"
|
||||
SKIPPED = "Save to library is off for this run's prompt, skipping image"
|
||||
|
||||
# The class dict, not the class: a TestCase bound here would run again
|
||||
Fixture = vars(attribution.TestMonitorForeignRuns)
|
||||
|
||||
|
||||
def off_behind(middle="MyNode", key="value"):
|
||||
"""A switched-off PromptManagerText reaching a core CLIPTextEncode through
|
||||
``middle``'s ``key`` input: the #90 walk does not follow ``key``."""
|
||||
graph = save_tests.run_graph()
|
||||
graph["1"] = {
|
||||
"class_type": "PromptManagerText",
|
||||
"inputs": {"text": OFF_TEXT, "save_to_library": False},
|
||||
}
|
||||
graph["7"] = {"class_type": middle, "inputs": {key: ["1", 0]}}
|
||||
graph["8"] = {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": ["7", 0], "clip": ["4", 1]},
|
||||
}
|
||||
graph["3"]["inputs"]["positive"] = ["8", 0]
|
||||
return graph
|
||||
|
||||
|
||||
def off_text_unwired():
|
||||
"""Only history can decide; the legacy answer is a dangling switched-off
|
||||
node's own text (the ladder says ``not_saved`` once it may ask history)."""
|
||||
graph = sampled_graph("unused", "blurry")
|
||||
graph["1"]["inputs"]["text"] = ["30", 0]
|
||||
graph["30"] = {"class_type": "VendorPromptMaker", "inputs": {"prompt": OFF_TEXT}}
|
||||
graph["40"] = {
|
||||
"class_type": "PromptManagerText",
|
||||
"inputs": {"text": OFF_TEXT, "save_to_library": False},
|
||||
}
|
||||
return graph
|
||||
|
||||
|
||||
class TestForeignPathKeepsSaveToLibrary(attribution.RecordedRuns, unittest.TestCase):
|
||||
setUp = Fixture["setUp"]
|
||||
close_databases = Fixture["close_databases"]
|
||||
wait_for = Fixture["wait_for"]
|
||||
image = Fixture["image"]
|
||||
linked_text = Fixture["linked_text"]
|
||||
key = Fixture["key"]
|
||||
prompt_count = Fixture["prompt_count"]
|
||||
settled = Fixture["settled"]
|
||||
|
||||
def reference(self, path):
|
||||
return {
|
||||
"folder_type": "output",
|
||||
"subfolder": "dataset",
|
||||
"filename": os.path.basename(path),
|
||||
}
|
||||
|
||||
def seal(self, path, positives=(), nodes=("9",)):
|
||||
"""Record one run reporting ``path`` (a new run id for every call)."""
|
||||
self.runs = getattr(self, "runs", -1) + 1
|
||||
self.record_run(
|
||||
self.db,
|
||||
self.output,
|
||||
[("dataset", os.path.basename(path))],
|
||||
list(positives),
|
||||
item=self.runs,
|
||||
nodes=list(nodes),
|
||||
)
|
||||
|
||||
def recheck(self, path):
|
||||
"""The re-check the seal schedules, run through the monitor's entry."""
|
||||
with self.assertLogs(LOGGER, "INFO") as logs:
|
||||
self.handler.process_new_image(path, ForeignRecheck(self.reference(path)))
|
||||
return "\n".join(logs.output)
|
||||
|
||||
def assert_nothing_stored(self, path):
|
||||
self.assertIsNone(self.linked_text(path))
|
||||
self.assertEqual(self.prompt_count(), 0)
|
||||
self.assertIsNone(self.db.get_prompt_by_hash(generate_prompt_hash(OFF_TEXT)))
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [])
|
||||
self.assertEqual(list(self.handler._foreign_due), [])
|
||||
|
||||
def test_q1_the_only_prompt_manager_node_off_is_opted_out_not_foreign(self):
|
||||
graph = save_tests.run_graph(save=False)
|
||||
# The switched-off positive is the only PromptManager node
|
||||
graph["2"] = {"class_type": "CLIPTextEncode", "inputs": {"text": "blurry"}}
|
||||
sampler_less = {
|
||||
"1": {
|
||||
"class_type": "PromptManagerText",
|
||||
"inputs": {"text": OFF_TEXT, "save_to_library": False},
|
||||
}
|
||||
}
|
||||
for name, graph in (("sampled", graph), ("sampler-less", sampler_less)):
|
||||
with self.subTest(name=name):
|
||||
self.metadata = {"prompt": graph}
|
||||
state, _ = self.handler._find_prompt_from_metadata(self.metadata)
|
||||
self.assertEqual(state, "opted_out")
|
||||
path = self.image(f"IMG_GEN_{name}_00001_.png")
|
||||
with self.assertLogs(LOGGER, "INFO") as logs:
|
||||
self.handler.process_new_image(path)
|
||||
self.assertIn(SKIPPED, "\n".join(logs.output))
|
||||
self.seal(path)
|
||||
self.assertTrue(self.settled())
|
||||
self.assertIn(SKIPPED, self.recheck(path))
|
||||
self.assert_nothing_stored(path)
|
||||
|
||||
def test_q2_an_off_text_node_behind_a_pass_through_stores_nothing(self):
|
||||
for middle, key in (("MyNode", "value"), ("easy showAnything", "anything")):
|
||||
with self.subTest(middle=middle):
|
||||
self.metadata = {"prompt": off_behind(middle, key)}
|
||||
state, _ = self.handler._find_prompt_from_metadata(self.metadata)
|
||||
self.assertEqual(state, "foreign")
|
||||
path = self.image(f"IMG_GEN_{key}_00001_.png")
|
||||
with self.assertLogs(LOGGER, "INFO") as logs:
|
||||
self.handler.process_new_image(path)
|
||||
self.assertIn(SKIPPED, "\n".join(logs.output))
|
||||
self.assert_nothing_stored(path)
|
||||
self.seal(path)
|
||||
self.assertTrue(self.settled())
|
||||
self.assertIn(SKIPPED, self.recheck(path))
|
||||
self.assert_nothing_stored(path)
|
||||
|
||||
def test_q2_a_wired_switch_behind_a_pass_through_stores_nothing(self):
|
||||
graph = off_behind()
|
||||
graph["1"]["inputs"]["save_to_library"] = ["20", 0]
|
||||
graph["20"] = {"class_type": "SomeBoolNode", "inputs": {}}
|
||||
self.metadata = {"prompt": graph}
|
||||
path = self.image("IMG_GEN_wired_00001_.png")
|
||||
wired = "Save to library is wired from a node the image cannot show"
|
||||
with self.assertLogs(LOGGER, "INFO") as logs:
|
||||
self.handler.process_new_image(path)
|
||||
self.assertIn(wired, "\n".join(logs.output))
|
||||
self.seal(path)
|
||||
self.assertTrue(self.settled())
|
||||
self.assertIn(wired, self.recheck(path))
|
||||
self.assert_nothing_stored(path)
|
||||
|
||||
def test_q3_an_undecided_not_saved_waits_for_the_record_then_skips(self):
|
||||
"""No record yet: a not_saved of the undecided step is not final (a
|
||||
recorded saver may resolve another branch), so the file waits for its
|
||||
run; the decision with the record stores nothing and logs the skip."""
|
||||
wired = off_text_unwired()
|
||||
wired["40"]["inputs"]["save_to_library"] = ["20", 0]
|
||||
wired["20"] = {"class_type": "SomeBoolNode", "inputs": {}}
|
||||
cases = (
|
||||
("off", off_text_unwired(), SKIPPED),
|
||||
("wired", wired, "Save to library is wired from a node"),
|
||||
)
|
||||
for name, graph, line in cases:
|
||||
with self.subTest(name=name):
|
||||
self.metadata = {"prompt": graph}
|
||||
path = self.image(f"vendor_{name}_00001_.png")
|
||||
self.handler.process_new_image(path)
|
||||
self.assertIsNone(self.linked_text(path))
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [self.key(path)])
|
||||
with self.assertLogs(LOGGER, "INFO") as logs:
|
||||
self.seal(path)
|
||||
self.assertTrue(
|
||||
self.wait_for(lambda: not self.handler.awaiting_foreign())
|
||||
)
|
||||
self.assertTrue(self.settled())
|
||||
self.assertIn(line, "\n".join(logs.output))
|
||||
self.assert_nothing_stored(path)
|
||||
|
||||
def test_q3_an_early_not_saved_never_waits_even_with_a_saver_branch(self):
|
||||
"""Review: a switched-off node feeding one branch's positive is the
|
||||
ladder's early, final not_saved (PR 1's #90 rule covers every
|
||||
sampler's positive), so the file never waits and nothing is stored,
|
||||
not even the plain branch the recorded saver names."""
|
||||
graph = shapes.shared_prefix()
|
||||
graph["20"]["inputs"]["text"] = ["50", 0]
|
||||
graph["50"] = {
|
||||
"class_type": "PromptManagerText",
|
||||
"inputs": {"text": OFF_TEXT, "save_to_library": False},
|
||||
}
|
||||
self.metadata = {"prompt": graph}
|
||||
path = self.image("IMG_GEN_00001_.png")
|
||||
with self.assertLogs(LOGGER, "INFO") as logs:
|
||||
self.handler.process_new_image(path)
|
||||
self.assertIn(SKIPPED, "\n".join(logs.output))
|
||||
self.assert_nothing_stored(path)
|
||||
self.seal(path, nodes=("13",))
|
||||
self.assertTrue(self.settled())
|
||||
self.assertIn(SKIPPED, self.recheck(path))
|
||||
self.assert_nothing_stored(path)
|
||||
self.assertIsNone(
|
||||
self.db.get_prompt_by_hash(generate_prompt_hash(shapes.BRANCH_TEXTS[0]))
|
||||
)
|
||||
|
||||
def test_q3_an_off_node_in_the_recorded_saver_branch_is_never_foreign(self):
|
||||
"""PR 3's parity case: a switched-off node feeds the very branch the
|
||||
recorded saver names (saver "23" is branch 2). A literal switch is the
|
||||
opted-out path's; a wired one is the node's own run's (#90), never the
|
||||
foreign path's. Nothing is stored at settle or after the run, and no
|
||||
file waits for a foreign re-check."""
|
||||
cases = (
|
||||
("off", False, "opted_out", SKIPPED),
|
||||
("wired", ["52", 0], "unknown", "No certain prompt for"),
|
||||
)
|
||||
for name, save, state, line in cases:
|
||||
with self.subTest(name=name):
|
||||
graph = shapes.shared_prefix()
|
||||
graph["50"] = {
|
||||
"class_type": "PromptManagerText",
|
||||
"inputs": {"text": OFF_TEXT, "save_to_library": save},
|
||||
}
|
||||
graph["20"]["inputs"]["text"] = ["50", 0]
|
||||
if isinstance(save, list):
|
||||
graph["52"] = {"class_type": "SomeBoolNode", "inputs": {}}
|
||||
self.metadata = {"prompt": graph}
|
||||
found, _ = self.handler._find_prompt_from_metadata(self.metadata)
|
||||
self.assertEqual(found, state)
|
||||
path = self.image(f"IMG_GEN_{name}_00001_.png")
|
||||
with self.assertLogs(LOGGER, "INFO") as logs:
|
||||
self.handler.process_new_image(path)
|
||||
self.assertIn(line, "\n".join(logs.output))
|
||||
self.assert_nothing_stored(path)
|
||||
self.seal(path, positives=[OFF_TEXT], nodes=("23",))
|
||||
self.assertTrue(self.settled())
|
||||
self.handler.process_new_image(path)
|
||||
self.assert_nothing_stored(path)
|
||||
self.assertIsNone(
|
||||
self.db.get_prompt_by_hash(
|
||||
generate_prompt_hash(shapes.BRANCH_TEXTS[1])
|
||||
)
|
||||
)
|
||||
|
||||
def test_q4_an_off_node_on_the_negative_only_still_links_the_positive(self):
|
||||
positive = "a lighthouse on a cliff at dusk"
|
||||
plain = sampled_graph(positive, "unused")
|
||||
plain["2"] = {
|
||||
"class_type": "PromptManager",
|
||||
"inputs": {"text": OFF_TEXT, "save_to_library": False},
|
||||
}
|
||||
behind = sampled_graph(positive, "unused")
|
||||
behind["2"]["inputs"]["text"] = ["7", 0]
|
||||
behind["7"] = {
|
||||
"class_type": "PromptManagerText",
|
||||
"inputs": {"text": OFF_TEXT, "save_to_library": False},
|
||||
}
|
||||
for name, graph in (("direct", plain), ("behind an encoder", behind)):
|
||||
with self.subTest(name=name):
|
||||
self.metadata = {"prompt": graph}
|
||||
state, _ = self.handler._find_prompt_from_metadata(self.metadata)
|
||||
self.assertEqual(state, "foreign")
|
||||
path = self.image(f"plain_{len(name)}_00001_.png")
|
||||
self.handler.process_new_image(path)
|
||||
self.assertEqual(self.linked_text(path), positive)
|
||||
self.assertIsNone(
|
||||
self.db.get_prompt_by_hash(generate_prompt_hash(OFF_TEXT))
|
||||
)
|
||||
pairs = self.db.list_prompt_pairs()["pairs"]
|
||||
self.assertEqual(pairs, [])
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [])
|
||||
|
||||
def test_q5_recorded_history_never_brings_back_switched_off_text(self):
|
||||
"""Even a recorded run whose contexts hold the off text (an older
|
||||
history row) cannot name it, at settle or at the re-check."""
|
||||
cases = {"pass-through": off_behind(), "dangling": off_text_unwired()}
|
||||
for name, graph in cases.items():
|
||||
with self.subTest(name=name):
|
||||
self.metadata = {"prompt": graph}
|
||||
path = self.image(f"IMG_GEN_{len(name)}_00001_.png")
|
||||
self.seal(path, positives=[OFF_TEXT], nodes=("9",))
|
||||
self.handler.process_new_image(path)
|
||||
self.assert_nothing_stored(path)
|
||||
self.assertIn(SKIPPED, self.recheck(path))
|
||||
self.assert_nothing_stored(path)
|
||||
|
||||
def test_q6_with_foreign_saving_off_save_to_library_is_unchanged(self):
|
||||
GalleryConfig.SAVE_FOREIGN_RUNS = False
|
||||
opted_out = save_tests.run_graph(save=False)
|
||||
self.metadata = {"prompt": opted_out}
|
||||
skipped = self.image("IMG_GEN_off_00001_.png")
|
||||
with self.assertLogs(LOGGER, "INFO") as logs:
|
||||
self.handler.process_new_image(skipped)
|
||||
self.assertIn(SKIPPED, "\n".join(logs.output))
|
||||
self.metadata = {"prompt": off_behind()}
|
||||
foreign = self.image("IMG_GEN_foreign_00001_.png")
|
||||
with self.assertLogs(LOGGER, "INFO") as logs:
|
||||
self.handler.process_new_image(foreign)
|
||||
self.assertIn("No PromptManager prompt in workflow", "\n".join(logs.output))
|
||||
for path in (skipped, foreign):
|
||||
self.assert_nothing_stored(path)
|
||||
# A switched-on PromptManager positive still links by its metadata
|
||||
stored = self.db.save_prompt(
|
||||
text=save_tests.POSITIVE,
|
||||
prompt_hash=generate_prompt_hash(save_tests.POSITIVE),
|
||||
)
|
||||
self.metadata = {"prompt": save_tests.run_graph(save=True)}
|
||||
on = self.image("IMG_GEN_on_00001_.png")
|
||||
self.handler.process_new_image(on)
|
||||
self.assertEqual(self.linked_text(on), save_tests.POSITIVE)
|
||||
self.assertEqual(
|
||||
self.db.get_prompt_by_hash(generate_prompt_hash(save_tests.POSITIVE))["id"],
|
||||
stored,
|
||||
)
|
||||
|
||||
def test_the_shared_prefix_dataset_still_links_after_the_seal(self):
|
||||
"""Regression guard (passes before the fix): the wait still works."""
|
||||
self.metadata = {"prompt": shapes.shared_prefix()}
|
||||
path = self.image("IMG_GEN_00001_.png")
|
||||
self.handler.process_new_image(path)
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [self.key(path)])
|
||||
self.seal(path, nodes=("13",))
|
||||
self.assertTrue(self.wait_for(lambda: self.linked_text(path)))
|
||||
self.assertTrue(self.settled())
|
||||
self.assertEqual(self.linked_text(path), shapes.BRANCH_TEXTS[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -16,11 +16,6 @@ from tests.background_stops import stop_tracker # noqa: E402
|
||||
from utils.hashing import (
|
||||
generate_prompt_hash,
|
||||
is_duplicate_prompt,
|
||||
STRIPPED_CODE_POINTS,
|
||||
prompt_key,
|
||||
register_prompt_functions,
|
||||
stored_prompt_text,
|
||||
stored_prompt_text_sql,
|
||||
)
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
@@ -120,96 +115,5 @@ class TestGeneratePromptHashEdgeCases(unittest.TestCase):
|
||||
self.assertEqual(len(h), 64)
|
||||
|
||||
|
||||
class TestStoredPromptText(unittest.TestCase):
|
||||
"""SQLite strips exactly the characters Python's str.strip() does (#438)."""
|
||||
|
||||
WHITESPACE = "".join(c for c in map(chr, range(0x110000)) if c.isspace())
|
||||
|
||||
def sql_strip(self, value):
|
||||
import sqlite3
|
||||
|
||||
conn = sqlite3.connect(":memory:")
|
||||
try:
|
||||
return conn.execute(
|
||||
f"SELECT {stored_prompt_text_sql('?')}", (value,)
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def test_code_points_are_exactly_python_whitespace(self):
|
||||
self.assertEqual("".join(map(chr, STRIPPED_CODE_POINTS)), self.WHITESPACE)
|
||||
|
||||
def test_python_rule_is_the_hash_strip(self):
|
||||
self.assertEqual(stored_prompt_text(" a lighthouse\n"), "a lighthouse")
|
||||
self.assertEqual(
|
||||
generate_prompt_hash(stored_prompt_text(" A b\u3000")),
|
||||
generate_prompt_hash(" A b\u3000"),
|
||||
)
|
||||
|
||||
def test_sql_strips_every_python_whitespace_character(self):
|
||||
for char in self.WHITESPACE:
|
||||
with self.subTest(code=hex(ord(char))):
|
||||
value = f"{char}a{char}b{char}"
|
||||
self.assertEqual(self.sql_strip(value), stored_prompt_text(value))
|
||||
mixed = self.WHITESPACE + "a lighthouse é" + self.WHITESPACE
|
||||
self.assertEqual(self.sql_strip(mixed), "a lighthouse é")
|
||||
|
||||
def test_sql_keeps_what_python_keeps(self):
|
||||
for value in ("\u200ba\u200b", "\ufeffa", "a\u0301", "", "x"):
|
||||
with self.subTest(value=repr(value)):
|
||||
self.assertEqual(self.sql_strip(value), stored_prompt_text(value))
|
||||
|
||||
|
||||
class TestPromptKey(unittest.TestCase):
|
||||
"""History compares the hash's own key; SQL calls Python for it (#460)."""
|
||||
|
||||
CASES = (
|
||||
"A Lighthouse",
|
||||
" \u00c9t\u00c9\n",
|
||||
"\u03a3\u03a3",
|
||||
"\u0130stanbul",
|
||||
"\u1e9e",
|
||||
"\u212a",
|
||||
"ok",
|
||||
)
|
||||
|
||||
def test_key_is_the_hash_normalization(self):
|
||||
import hashlib
|
||||
|
||||
for value in self.CASES:
|
||||
with self.subTest(value=repr(value)):
|
||||
self.assertEqual(
|
||||
generate_prompt_hash(value),
|
||||
hashlib.sha256(prompt_key(value).encode("utf-8")).hexdigest(),
|
||||
)
|
||||
self.assertEqual(prompt_key(" A Lighthouse\n"), "a lighthouse")
|
||||
|
||||
def test_sql_functions_give_pythons_result_where_sql_lower_does_not(self):
|
||||
import sqlite3
|
||||
|
||||
conn = sqlite3.connect(":memory:")
|
||||
try:
|
||||
register_prompt_functions(conn)
|
||||
for value in self.CASES:
|
||||
with self.subTest(value=repr(value)):
|
||||
key, digest = conn.execute(
|
||||
"SELECT pm_prompt_key(?), pm_prompt_hash(?)", (value, value)
|
||||
).fetchone()
|
||||
self.assertEqual(key, prompt_key(value))
|
||||
self.assertEqual(digest, generate_prompt_hash(value))
|
||||
self.assertEqual(
|
||||
conn.execute(
|
||||
"SELECT pm_prompt_key(1), pm_prompt_hash(NULL)"
|
||||
).fetchone(),
|
||||
(None, None),
|
||||
)
|
||||
# SQLite's own lower() folds ASCII only
|
||||
self.assertNotEqual(
|
||||
conn.execute("SELECT lower(?)", ("\u00c9",)).fetchone()[0], "\u00e9"
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -7,7 +7,6 @@ from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from database import restore_stage
|
||||
from database.execution_history import HistoryEpochChanged, notify_history_restore
|
||||
from tests import test_execution_history_database as history_tests
|
||||
|
||||
@@ -42,12 +41,7 @@ class HistoryReadFenceTests(unittest.TestCase):
|
||||
generation = fence.begin()
|
||||
with reader.recorded_history_snapshot() as epoch:
|
||||
original = reader.get_history_run(start.run_id)
|
||||
# The open snapshot keeps the WAL from being truncated; do not
|
||||
# sit out the full 30 s checkpoint wait for it (#487).
|
||||
with (
|
||||
patch("database.execution_history.notify_history_restore"),
|
||||
patch.object(restore_stage, "CHECKPOINT_BUSY_MS", 50),
|
||||
):
|
||||
with patch("database.execution_history.notify_history_restore"):
|
||||
writer.model.restore_from_file(source)
|
||||
self.assertEqual(reader.get_execution_epoch(), epoch)
|
||||
self.assertEqual(original["run"]["storage_epoch"], epoch)
|
||||
|
||||
+5
-161
@@ -9,15 +9,9 @@ from unittest.mock import patch
|
||||
|
||||
from database.execution_history import HistoryEpochChanged
|
||||
from database.execution_history_schema import history_schema_error
|
||||
from utils.execution_contexts import (
|
||||
ContextBundle,
|
||||
RecipeContext,
|
||||
TextEvidence,
|
||||
content_digest,
|
||||
)
|
||||
from utils.execution_contexts import ContextBundle, RecipeContext, TextEvidence
|
||||
from utils.execution_outputs import OutputBundle
|
||||
from tests.test_execution_context_database import bundle, snapshot
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from tests.test_execution_output_database import bundle as outputs
|
||||
from tests import test_execution_history_database as history_tests
|
||||
|
||||
@@ -34,10 +28,6 @@ class HistoryReadTests(unittest.TestCase):
|
||||
conn.execute("INSERT INTO prompts(id,text) VALUES(?,?)", (identity, text))
|
||||
return identity
|
||||
|
||||
def hashed(self, db, text):
|
||||
"""A prompt saved as the nodes save it: with its dedup hash."""
|
||||
return db.save_prompt(text, prompt_hash=generate_prompt_hash(text))
|
||||
|
||||
def capture(self, db, item=0, context=None, output=None, time=100):
|
||||
start = self.start(db, item, time)
|
||||
db.write_execution_start(start)
|
||||
@@ -156,11 +146,10 @@ class HistoryReadTests(unittest.TestCase):
|
||||
)
|
||||
self.require_reads(db)
|
||||
with db.recorded_history_snapshot():
|
||||
# Outer whitespace (#438) and case (#460) join one scope;
|
||||
# Unicode normalization stays distinct.
|
||||
expected = (5, 5, 5, 5, 1, 1, 2, 2)
|
||||
for i, count in enumerate(expected, 1):
|
||||
self.assertEqual(db.list_history_window(i)["recorded_run_count"], count)
|
||||
for i in range(1, len(values) + 1):
|
||||
self.assertEqual(
|
||||
db.list_history_window(i)["recorded_run_count"], 2 if i == 1 else 1
|
||||
)
|
||||
self.assertIsNone(db.list_history_window(100))
|
||||
self.assertIsNone(db.list_history_window(101))
|
||||
page = db.list_history_window(99)
|
||||
@@ -400,32 +389,6 @@ class HistoryReadTests(unittest.TestCase):
|
||||
pass
|
||||
self.assertFalse(conn.in_transaction)
|
||||
|
||||
def test_a_deleted_negative_is_unavailable_and_a_negative_only_one_is_not(self):
|
||||
# #443: a Library delete can remove an unpaired prompt a run used as
|
||||
# its negative. The run's prompt ids are live hints, so a deleted one
|
||||
# reads unavailable; a negative-only prompt stays available.
|
||||
db = self.database()
|
||||
self.saved(db)
|
||||
self.saved(db, "negative", 2)
|
||||
start = self.capture(db)
|
||||
with db.recorded_history_snapshot():
|
||||
context = db.get_history_run(start.run_id)["contexts"][0]
|
||||
self.assertTrue(context["navigation"]["negative_available"])
|
||||
with db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) "
|
||||
"VALUES(9,1,2)"
|
||||
)
|
||||
with db.recorded_history_snapshot():
|
||||
context = db.get_history_run(start.run_id)["contexts"][0]
|
||||
self.assertTrue(context["navigation"]["negative_available"])
|
||||
with db.model.get_connection() as conn:
|
||||
conn.execute("DELETE FROM prompt_pairs WHERE id=9")
|
||||
conn.execute("DELETE FROM prompts WHERE id=2")
|
||||
with db.recorded_history_snapshot():
|
||||
context = db.get_history_run(start.run_id)["contexts"][0]
|
||||
self.assertFalse(context["navigation"]["negative_available"])
|
||||
|
||||
def test_detail_preserves_full_text_snapshots_hints_and_archive(self):
|
||||
db = self.database()
|
||||
self.saved(db)
|
||||
@@ -1026,122 +989,3 @@ class HistoryReadTests(unittest.TestCase):
|
||||
db.delete_prompt(saved["positive_prompt_id"])
|
||||
with db.recorded_history_snapshot():
|
||||
self.assertIsNone(db.get_newest_pair_run(saved["positive_prompt_id"], pair))
|
||||
|
||||
def test_outer_whitespace_raw_text_joins_the_stored_prompt_scope(self):
|
||||
# #438: the Library stores text.strip(); the run keeps its raw text.
|
||||
db = self.database()
|
||||
positive = db.save_prompt("a lighthouse")
|
||||
raws = ("a lighthouse\n", "a lighthouse", " a lighthouse \t")
|
||||
starts = [
|
||||
self.capture(db, i, bundle(positive=raw), time=100 + i)
|
||||
for i, raw in enumerate(raws)
|
||||
]
|
||||
self.saved(db, "a lighthouse at dusk", 50)
|
||||
self.capture(db, 9, bundle(positive="a lighthouse at dusk\n"), time=200)
|
||||
with db.recorded_history_snapshot():
|
||||
page = db.list_history_window(positive)
|
||||
first = db.list_history_window(positive, limit=2)
|
||||
rest = db.list_history_window(
|
||||
positive, limit=2, after=first["next_position"]
|
||||
)
|
||||
details = [db.get_history_run(start.run_id) for start in starts]
|
||||
self.assertEqual(page["recorded_run_count"], 3)
|
||||
self.assertEqual(page["group"]["scope_status"], "matched")
|
||||
self.assertEqual(
|
||||
page["group"]["raw_text_id"], content_digest("text", ("a lighthouse",))
|
||||
)
|
||||
self.assertEqual(
|
||||
[r["run_id"] for r in page["items"]], [s.run_id for s in starts]
|
||||
)
|
||||
self.assertTrue(all(r["matching_anchors"] for r in page["items"]))
|
||||
self.assertEqual(len(page["context_summaries"]), 3)
|
||||
self.assertEqual(
|
||||
[r["run_id"] for r in first["items"] + rest["items"]],
|
||||
[s.run_id for s in starts[1:]] + [starts[0].run_id],
|
||||
)
|
||||
for raw, detail in zip(raws, details):
|
||||
context = detail["contexts"][0]
|
||||
self.assertEqual(context["positive"]["raw"], raw)
|
||||
self.assertEqual(
|
||||
context["positive_raw_text_id"], page["group"]["raw_text_id"]
|
||||
)
|
||||
self.assertEqual(context["positive_prompt_id"], positive)
|
||||
self.assertTrue(context["navigation"]["positive_available"])
|
||||
|
||||
def test_newest_pair_run_follows_outer_whitespace_raw_text(self):
|
||||
db = self.database()
|
||||
pair = db.save_prompt_pair("a lighthouse", "fog")
|
||||
positive = db.get_prompt_pair(pair)["positive_prompt_id"]
|
||||
newest = self.capture(
|
||||
db, 0, bundle(" fog\n", positive="a lighthouse\n"), time=100
|
||||
)
|
||||
with db.recorded_history_snapshot():
|
||||
found = db.get_newest_pair_run(positive, pair)
|
||||
self.assertEqual(found["run"]["run_id"], newest.run_id)
|
||||
|
||||
def test_case_variant_raw_text_joins_the_saved_prompt_scope(self):
|
||||
# #460: the hash folds case, so these runs count toward this prompt.
|
||||
db = self.database()
|
||||
positive = self.hashed(db, "\u00e9t\u00e9 lighthouse \u03c3")
|
||||
raws = (
|
||||
"\u00c9T\u00c9 Lighthouse \u03a3\n",
|
||||
"\u00e9t\u00e9 lighthouse \u03c3",
|
||||
" \u00c9t\u00e9 LIGHTHOUSE \u03a3",
|
||||
)
|
||||
starts = [
|
||||
self.capture(db, i, bundle(positive=raw), time=100 + i)
|
||||
for i, raw in enumerate(raws)
|
||||
]
|
||||
# Unicode normalization and inner text stay distinct
|
||||
self.capture(db, 7, bundle(positive="e\u0301te\u0301 lighthouse \u03c3"))
|
||||
self.capture(db, 8, bundle(positive="\u00e9t\u00e9 lighthouse \u03c3"))
|
||||
dotted = self.hashed(db, "\u0130stanbul")
|
||||
dotted_run = self.capture(db, 9, bundle(positive="\u0130STANBUL"), time=300)
|
||||
with db.recorded_history_snapshot():
|
||||
page = db.list_history_window(positive)
|
||||
details = [db.get_history_run(start.run_id) for start in starts]
|
||||
turkish = db.list_history_window(dotted)
|
||||
self.assertEqual(page["recorded_run_count"], 3)
|
||||
self.assertEqual(
|
||||
[r["run_id"] for r in page["items"]], [s.run_id for s in starts]
|
||||
)
|
||||
for raw, detail in zip(raws, details):
|
||||
context = detail["contexts"][0]
|
||||
self.assertEqual(context["positive"]["raw"], raw)
|
||||
self.assertEqual(
|
||||
context["positive_raw_text_id"], page["group"]["raw_text_id"]
|
||||
)
|
||||
self.assertEqual(context["positive_prompt_id"], positive)
|
||||
self.assertEqual([r["run_id"] for r in turkish["items"]], [dotted_run.run_id])
|
||||
|
||||
def test_newest_pair_run_follows_case_variant_raw_text(self):
|
||||
db = self.database()
|
||||
pair = db.save_prompt_pair("a lighthouse", "fog")
|
||||
positive = db.get_prompt_pair(pair)["positive_prompt_id"]
|
||||
newest = self.capture(db, 0, bundle("FOG", positive="A Lighthouse"), time=100)
|
||||
with db.recorded_history_snapshot():
|
||||
found = db.get_newest_pair_run(positive, pair)
|
||||
self.assertEqual(found["run"]["run_id"], newest.run_id)
|
||||
|
||||
def test_a_position_from_before_460_resumes_the_same_scope(self):
|
||||
db = self.database()
|
||||
positive = self.hashed(db, "A Lighthouse")
|
||||
for i in range(3):
|
||||
self.capture(db, i, bundle(positive="a lighthouse"), time=100 + i)
|
||||
with db.recorded_history_snapshot():
|
||||
first = db.list_history_window(positive, limit=2)
|
||||
position = first["next_position"]
|
||||
legacy = replace(
|
||||
position, raw_text_id=content_digest("text", ("A Lighthouse",))
|
||||
)
|
||||
rest = db.list_history_window(positive, limit=2, after=legacy)
|
||||
with self.assertRaises(ValueError):
|
||||
db.list_history_window(
|
||||
positive,
|
||||
after=replace(position, raw_text_id=content_digest("text", ("x",))),
|
||||
)
|
||||
self.assertEqual(first["recorded_run_count"], 3)
|
||||
self.assertEqual(len(rest["items"]), 1)
|
||||
self.assertEqual(
|
||||
rest["group"]["raw_text_id"], content_digest("text", ("a lighthouse",))
|
||||
)
|
||||
|
||||
@@ -1,409 +0,0 @@
|
||||
"""Delete selected images: the review and the apply job (#480).
|
||||
|
||||
``POST /prompt_manager/images/delete/preview`` reviews both modes;
|
||||
``POST /prompt_manager/maintenance/apply {token, mode}`` removes the files'
|
||||
records (library mode, with a removal mark) or moves the files to the trash
|
||||
first (trash mode, the fake trash here). No path unlinks a media file.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests import fake_trash
|
||||
from tests.disk_waits import DISK_BOUND_SECONDS
|
||||
from tests.test_api_duplicate_files import DuplicateFilesTestCase
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
import fake_trash
|
||||
from disk_waits import DISK_BOUND_SECONDS
|
||||
from test_api_duplicate_files import DuplicateFilesTestCase
|
||||
|
||||
from py.api import image_delete
|
||||
from py.api.image_delete import file_check, lexical_problem
|
||||
|
||||
PREVIEW = "/prompt_manager/images/delete/preview"
|
||||
APPLY = "/prompt_manager/maintenance/apply"
|
||||
LISTING = "/prompt_manager/images/output"
|
||||
|
||||
|
||||
class ImageDeleteTestCase(DuplicateFilesTestCase):
|
||||
async def get_application(self):
|
||||
app = await super().get_application()
|
||||
self.fake = fake_trash.install(Path(self.tmpdir) / "fake-trash")
|
||||
self.addCleanup(fake_trash.uninstall)
|
||||
return app
|
||||
|
||||
def target(self, path, index=0):
|
||||
roots = self.api._get_all_output_dirs()
|
||||
return {
|
||||
"root_index": index,
|
||||
"root_scope": self.api._gallery_root_scope(roots[index], index),
|
||||
"path": path,
|
||||
}
|
||||
|
||||
async def review(self, paths, status=200):
|
||||
items = [p if isinstance(p, dict) else self.target(p) for p in paths]
|
||||
return await self.post(PREVIEW, {"items": items}, status)
|
||||
|
||||
async def delete(self, paths, mode):
|
||||
plan = await self.review(paths)
|
||||
body = await self.post(APPLY, {"token": plan["token"], "mode": mode}, 202)
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
return plan, job
|
||||
|
||||
async def listed(self):
|
||||
response = await self.client.get(LISTING, params={"mode": "workspace"})
|
||||
page = await response.json()
|
||||
return [image["path"] for image in page["images"]]
|
||||
|
||||
def link(self, name, text="a prompt"):
|
||||
prompt_id = self.save(text)
|
||||
return self.api.db.link_image_to_prompt(
|
||||
str(prompt_id), str(self.output_dir / name)
|
||||
)
|
||||
|
||||
def count(self, sql):
|
||||
conn = sqlite3.connect(self.api.db.model.db_path)
|
||||
try:
|
||||
return conn.execute(sql).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@staticmethod
|
||||
def details(job, group):
|
||||
return [row["detail"] for row in job.detail()["outcomes"][group]]
|
||||
|
||||
|
||||
class TestReview(ImageDeleteTestCase):
|
||||
async def test_review_counts_records_and_refuses_what_it_cannot_delete(self):
|
||||
self.write("a.png", b"aaa", mtime=1_000_000)
|
||||
self.write("b.png", b"bbb", mtime=2_000_000)
|
||||
self.link("a.png", "only a")
|
||||
self.fake.refuse[str(self.output_dir / "b.png")] = "no-trash"
|
||||
plan = await self.review(["a.png", "b.png", "missing.png"])
|
||||
self.assertEqual(plan["library"], {"count": 2})
|
||||
self.assertEqual(plan["trash"], {"count": 1})
|
||||
self.assertEqual(plan["refused"]["counts"], {"not-listed": 1})
|
||||
self.assertEqual(plan["trash_refused"]["counts"], {"no-trash": 1})
|
||||
self.assertEqual(plan["records"]["prompt_links"], 1)
|
||||
self.assertEqual(plan["orphaned_prompts"]["count"], 1)
|
||||
self.assertIn(plan["backup"]["policy"], ("on", "off"))
|
||||
self.assertNotIn(self.tmpdir, str(plan))
|
||||
# The review wrote nothing.
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 1)
|
||||
self.assertEqual(self.fake.moved, [])
|
||||
|
||||
async def test_body_shapes_and_the_cap(self):
|
||||
await self.post(PREVIEW, {}, 400)
|
||||
await self.post(PREVIEW, {"items": [], "mode": "trash"}, 400)
|
||||
too_many = [self.target(f"{n}.png") for n in range(501)]
|
||||
await self.post(PREVIEW, {"items": too_many}, 400)
|
||||
|
||||
async def test_mode_is_required_and_only_for_this_plan(self):
|
||||
self.write("a.png", b"aaa")
|
||||
self.link("a.png")
|
||||
plan = await self.review(["a.png"])
|
||||
# No explicit, exact mode: refused, never a default (least of all trash).
|
||||
for body in (
|
||||
{"token": plan["token"]},
|
||||
{"token": plan["token"], "mode": None},
|
||||
{"token": plan["token"], "mode": ""},
|
||||
{"token": plan["token"], "mode": "TRASH"},
|
||||
{"token": plan["token"], "mode": "delete"},
|
||||
{"token": plan["token"], "mode": ["trash"]},
|
||||
{"token": plan["token"], "mode": True},
|
||||
):
|
||||
with self.subTest(body=body):
|
||||
await self.post(APPLY, body, 400)
|
||||
self.assertTrue((self.output_dir / "a.png").exists())
|
||||
self.assertEqual(self.fake.moved, [])
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 1)
|
||||
# The refusals left the token usable: an explicit mode still applies.
|
||||
body = await self.post(APPLY, {"token": plan["token"], "mode": "library"}, 202)
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
self.assertEqual(job.scope["mode"], "library")
|
||||
self.assertTrue((self.output_dir / "a.png").exists())
|
||||
self.assertEqual(self.fake.moved, [])
|
||||
|
||||
async def test_mode_on_another_plan_is_refused(self):
|
||||
plan = await self.post(
|
||||
"/prompt_manager/maintenance/preview",
|
||||
{"operation": "missing_links", "options": {}},
|
||||
200,
|
||||
)
|
||||
await self.post(APPLY, {"token": plan["token"], "mode": "trash"}, 400)
|
||||
|
||||
|
||||
class TestLibraryMode(ImageDeleteTestCase):
|
||||
async def test_records_go_and_the_file_stays_hidden(self):
|
||||
self.write("a.png", b"aaa", mtime=1_000_000)
|
||||
self.write("b.png", b"bbb", mtime=2_000_000)
|
||||
self.link("a.png")
|
||||
self.assertEqual(sorted(await self.listed()), ["a.png", "b.png"])
|
||||
before = os.stat(self.output_dir / "a.png")
|
||||
_plan, job = await self.delete(["a.png"], "library")
|
||||
self.assertEqual(job.state, "finished", job.summary())
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 0)
|
||||
after = os.stat(self.output_dir / "a.png")
|
||||
self.assertEqual(
|
||||
(before.st_size, before.st_mtime_ns), (after.st_size, after.st_mtime_ns)
|
||||
)
|
||||
self.assertEqual(await self.listed(), ["b.png"])
|
||||
# A later scan or link never brings it back.
|
||||
self.assertEqual(self.link("a.png", "rescanned"), 0)
|
||||
self.assertEqual(
|
||||
self.api._removed_paths([self.output_dir / "a.png"]),
|
||||
{str(self.output_dir / "a.png")},
|
||||
)
|
||||
self.assertEqual(self.fake.moved, [])
|
||||
|
||||
async def test_a_modified_file_shows_again(self):
|
||||
self.write("a.png", b"aaa", mtime=1_000_000)
|
||||
await self.delete(["a.png"], "library")
|
||||
self.assertEqual(await self.listed(), [])
|
||||
self.write("a.png", b"new and longer", mtime=3_000_000)
|
||||
self.api.invalidate_gallery_cache()
|
||||
self.assertEqual(await self.listed(), ["a.png"])
|
||||
|
||||
async def test_a_safety_backup_is_taken_first(self):
|
||||
self.write("a.png", b"aaa")
|
||||
with patch.object(self.api, "_safety_backup_policy", return_value="on"):
|
||||
await self.delete(["a.png"], "library")
|
||||
self.assertEqual(len(self.backups()), 1)
|
||||
|
||||
|
||||
class TestTrashMode(ImageDeleteTestCase):
|
||||
async def test_the_file_moves_first_then_its_records_go(self):
|
||||
self.write("a.png", b"aaa")
|
||||
self.link("a.png")
|
||||
seen = {}
|
||||
original = self.fake.move
|
||||
|
||||
def move(path, roots):
|
||||
# The records still exist while the file moves.
|
||||
seen["links"] = self.count("SELECT COUNT(*) FROM generated_images")
|
||||
return original(path, roots)
|
||||
|
||||
self.fake.move = move
|
||||
_plan, job = await self.delete(["a.png"], "trash")
|
||||
self.assertEqual(job.state, "finished", job.summary())
|
||||
self.assertEqual(seen["links"], 1)
|
||||
self.assertFalse((self.output_dir / "a.png").exists())
|
||||
[(source, destination)] = self.fake.moved
|
||||
self.assertTrue(os.path.exists(destination))
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 0)
|
||||
self.assertEqual(self.api.db.removal_marks(), {})
|
||||
|
||||
async def test_a_refused_move_changes_nothing(self):
|
||||
self.write("a.png", b"aaa")
|
||||
self.link("a.png")
|
||||
plan = await self.review(["a.png"])
|
||||
self.fake.refuse[str(self.output_dir / "a.png")] = "locked"
|
||||
body = await self.post(APPLY, {"token": plan["token"], "mode": "trash"}, 202)
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
self.assertTrue((self.output_dir / "a.png").exists())
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 1)
|
||||
self.assertEqual(self.api.db.removal_marks(), {})
|
||||
self.assertEqual(job.counters["not_changed"], 1)
|
||||
self.assertEqual(
|
||||
self.details(job, "not_changed"), [image_delete.REASONS["locked"]]
|
||||
)
|
||||
|
||||
async def test_a_file_changed_after_the_review_is_not_changed(self):
|
||||
self.write("a.png", b"aaa", mtime=1_000_000)
|
||||
plan = await self.review(["a.png"])
|
||||
self.write("a.png", b"changed bytes", mtime=2_000_000)
|
||||
body = await self.post(APPLY, {"token": plan["token"], "mode": "trash"}, 202)
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
self.assertTrue((self.output_dir / "a.png").exists())
|
||||
self.assertEqual(self.fake.moved, [])
|
||||
self.assertEqual(
|
||||
self.details(job, "not_changed"), [image_delete.REASONS["changed"]]
|
||||
)
|
||||
|
||||
async def test_a_records_failure_stops_the_job(self):
|
||||
self.write("a.png", b"aaa", mtime=1_000_000)
|
||||
self.write("b.png", b"bbb", mtime=2_000_000)
|
||||
with patch.object(
|
||||
type(self.api.db),
|
||||
"remove_image_records",
|
||||
side_effect=sqlite3.OperationalError("locked"),
|
||||
):
|
||||
_plan, job = await self.delete(["a.png", "b.png"], "library")
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertEqual(job.scope.get("failure"), "records-failed")
|
||||
self.assertEqual(job.counters["errors"], 1)
|
||||
self.assertEqual(
|
||||
self.details(job, "not_changed"), [image_delete.REASONS["not-attempted"]]
|
||||
)
|
||||
|
||||
async def test_nothing_is_ever_unlinked(self):
|
||||
self.write("a.png", b"aaa")
|
||||
self.write("b.png", b"bbb")
|
||||
media = str(self.output_dir)
|
||||
real_remove, real_unlink = os.remove, os.unlink
|
||||
|
||||
def guard(real):
|
||||
def call(path, *args, **kwargs):
|
||||
text = os.path.realpath(str(path))
|
||||
if (
|
||||
text.startswith(os.path.realpath(media))
|
||||
and "thumbnails" not in text
|
||||
):
|
||||
raise AssertionError(f"unlinked a media file: {path}")
|
||||
return real(path, *args, **kwargs)
|
||||
|
||||
return call
|
||||
|
||||
with (
|
||||
patch("os.remove", guard(real_remove)),
|
||||
patch("os.unlink", guard(real_unlink)),
|
||||
):
|
||||
await self.delete(["a.png"], "trash")
|
||||
await self.delete(["b.png"], "library")
|
||||
self.assertTrue((self.output_dir / "b.png").exists())
|
||||
|
||||
|
||||
class TestFileCheck(ImageDeleteTestCase):
|
||||
def test_lexical_codes(self):
|
||||
self.assertEqual(lexical_problem("thumbnails/a.png"), "thumbnail")
|
||||
self.assertEqual(lexical_problem(".Trash-1000/files/a.png"), "trash-folder")
|
||||
self.assertEqual(lexical_problem("x/$RECYCLE.BIN/a.png"), "trash-folder")
|
||||
self.assertEqual(lexical_problem("a.txt"), "not-media")
|
||||
self.assertIsNone(lexical_problem(".Trashy/a.png"))
|
||||
self.assertIsNone(lexical_problem("sub/a.png"))
|
||||
|
||||
def test_links_and_folders_leading_outside_are_refused(self):
|
||||
root = str(self.output_dir)
|
||||
outside = Path(self.tmpdir) / "elsewhere"
|
||||
outside.mkdir()
|
||||
(outside / "x.png").write_bytes(b"x")
|
||||
try:
|
||||
os.symlink(outside / "x.png", self.output_dir / "link.png")
|
||||
os.symlink(outside, self.output_dir / "linked", target_is_directory=True)
|
||||
except OSError:
|
||||
self.skipTest("symlinks are not permitted here")
|
||||
self.assertEqual(
|
||||
file_check(str(self.output_dir / "link.png"), root, "link.png")[0], "link"
|
||||
)
|
||||
self.assertEqual(
|
||||
file_check(str(self.output_dir / "linked/x.png"), root, "linked/x.png")[0],
|
||||
"outside",
|
||||
)
|
||||
self.assertEqual(
|
||||
file_check(str(self.output_dir / "gone.png"), root, "gone.png")[0], "gone"
|
||||
)
|
||||
self.write("ok.png", b"ok")
|
||||
self.assertIsNone(
|
||||
file_check(str(self.output_dir / "ok.png"), root, "ok.png")[0]
|
||||
)
|
||||
|
||||
def test_windows_reason_says_why_plainly(self):
|
||||
self.assertEqual(
|
||||
image_delete.REASONS["windows-recycle"],
|
||||
"Moving files to the Recycle Bin isn't available on Windows: files "
|
||||
"larger than the bin's capacity would be deleted permanently.",
|
||||
)
|
||||
|
||||
def test_reasons_cover_every_code(self):
|
||||
from utils import trash
|
||||
|
||||
for code in trash.CODES:
|
||||
self.assertIn(code, image_delete.REASONS)
|
||||
|
||||
|
||||
class TestMacOSOutcomes(ImageDeleteTestCase):
|
||||
"""The codes and the unconfirmed outcome the macOS backend adds (#499)."""
|
||||
|
||||
async def test_permission_and_cloud_refusals_show_their_sentences(self):
|
||||
self.write("a.png", b"aaa")
|
||||
self.write("b.png", b"bbb")
|
||||
self.fake.refuse[str(self.output_dir / "a.png")] = "no-permission"
|
||||
self.fake.refuse[str(self.output_dir / "b.png")] = "cloud-sync"
|
||||
plan = await self.review(["a.png", "b.png"])
|
||||
self.assertEqual(plan["trash"], {"count": 0})
|
||||
self.assertEqual(
|
||||
plan["trash_refused"]["counts"], {"no-permission": 1, "cloud-sync": 1}
|
||||
)
|
||||
details = {row["detail"] for row in plan["trash_refused"]["rows"]}
|
||||
self.assertEqual(
|
||||
details,
|
||||
{image_delete.REASONS["no-permission"], image_delete.REASONS["cloud-sync"]},
|
||||
)
|
||||
|
||||
async def test_an_unconfirmed_move_keeps_records_and_stops_the_job(self):
|
||||
from utils import trash
|
||||
|
||||
self.write("a.png", b"aaa", mtime=1_000_000)
|
||||
self.write("b.png", b"bbb", mtime=2_000_000)
|
||||
self.link("a.png")
|
||||
original = self.fake.move
|
||||
|
||||
def move(path, roots):
|
||||
if path.endswith("a.png"):
|
||||
original(path, roots) # the file did leave its folder
|
||||
raise trash.TrashUnconfirmed("resultingItemURL=None")
|
||||
return original(path, roots)
|
||||
|
||||
self.fake.move = move
|
||||
_plan, job = await self.delete(["a.png", "b.png"], "trash")
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertEqual(job.scope.get("failure"), "records-failed")
|
||||
self.assertEqual(
|
||||
self.details(job, "errors"), [image_delete.REASONS["trash-unconfirmed"]]
|
||||
)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 1)
|
||||
self.assertEqual(
|
||||
self.details(job, "not_changed"), [image_delete.REASONS["not-attempted"]]
|
||||
)
|
||||
self.assertTrue((self.output_dir / "b.png").exists())
|
||||
|
||||
async def test_a_refusal_detail_is_logged(self):
|
||||
from utils import trash
|
||||
|
||||
self.write("a.png", b"aaa")
|
||||
|
||||
def move(path, roots):
|
||||
raise trash.TrashRefused("no-permission", "NSCocoaErrorDomain 513")
|
||||
|
||||
self.fake.move = move
|
||||
with self.assertLogs("prompt_manager.api", level="INFO") as logs:
|
||||
_plan, job = await self.delete(["a.png"], "trash")
|
||||
self.assertEqual(
|
||||
self.details(job, "not_changed"), [image_delete.REASONS["no-permission"]]
|
||||
)
|
||||
self.assertTrue(any("NSCocoaErrorDomain 513" in line for line in logs.output))
|
||||
self.assertTrue((self.output_dir / "a.png").exists())
|
||||
|
||||
|
||||
class TestNativeBackend(ImageDeleteTestCase):
|
||||
"""The real backend of this OS: Windows has no trash yet (#500).
|
||||
|
||||
Only library mode is applied, so even on macOS and Linux no file reaches
|
||||
a real trash.
|
||||
"""
|
||||
|
||||
async def test_library_only_works_and_the_trash_is_unavailable_here(self):
|
||||
fake_trash.uninstall()
|
||||
self.write("a.png", b"aaa")
|
||||
self.link("a.png")
|
||||
plan = await self.review(["a.png"])
|
||||
native_trash = sys.platform != "win32"
|
||||
if not native_trash:
|
||||
self.assertEqual(plan["trash"], {"count": 0})
|
||||
self.assertEqual(plan["trash_refused"]["counts"], {"windows-recycle": 1})
|
||||
body = await self.post(APPLY, {"token": plan["token"], "mode": "library"}, 202)
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
self.assertEqual(job.counters["removed"], 1, job.summary())
|
||||
self.assertTrue((self.output_dir / "a.png").exists())
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 0)
|
||||
self.assertEqual(await self.listed(), [])
|
||||
@@ -172,40 +172,17 @@ class TestForeignWorkflows(LinkingTestCase):
|
||||
self.tracker.set_current_prompt(text, {"prompt_id": prompt_id})
|
||||
return prompt_id
|
||||
|
||||
def _linked_text(self):
|
||||
row = (
|
||||
self.db.model.get_connection()
|
||||
.execute(
|
||||
"SELECT p.text FROM generated_images g "
|
||||
"JOIN prompts p ON p.id = g.prompt_id WHERE g.image_path = ?",
|
||||
(self.image,),
|
||||
)
|
||||
.fetchone()
|
||||
)
|
||||
return row[0] if row else None
|
||||
|
||||
def test_foreign_image_leaves_the_batch_queue_aligned(self):
|
||||
"""#247: it links to its own prompt and never takes a queue entry."""
|
||||
first = self._queue_batch_item("item one")
|
||||
self.assertIsNone(self._process({"prompt": foreign_graph()}))
|
||||
self.assertEqual(self.linked, [])
|
||||
self.assertEqual(self.tracker.pop_next_prompt()["id"], first)
|
||||
self.assertIsNone(self.tracker.pop_next_prompt())
|
||||
self.assertEqual(self._linked_text(), "a cat")
|
||||
prompt = self.db.get_prompt_by_hash(generate_prompt_hash("a cat"))
|
||||
self.assertEqual(prompt["run_count"] or 0, 0)
|
||||
|
||||
def test_a_foreign_image_is_linked_only_by_the_ladder(self):
|
||||
def test_foreign_image_is_not_linked_by_the_later_fallbacks_either(self):
|
||||
self._queue_batch_item("item one") # live tracker and recent prompt exist
|
||||
untraceable = foreign_graph()
|
||||
untraceable["9"]["inputs"]["text"] = ["30", 0]
|
||||
untraceable["30"] = {"class_type": "VendorPromptMaker", "inputs": {}}
|
||||
self.assertIsNone(self._process({"prompt": untraceable}))
|
||||
self.assertEqual(self.linked, [])
|
||||
self.assertIsNone(self._linked_text())
|
||||
self.assertIsNone(self._process({"prompt": foreign_graph()}))
|
||||
self.assertEqual(self.linked, [])
|
||||
self.assertEqual(self._linked_text(), "a cat")
|
||||
|
||||
def test_negative_only_workflow_is_foreign(self):
|
||||
self._save("blurry")
|
||||
|
||||
@@ -597,11 +597,6 @@ class TestHandlerScheduling(unittest.TestCase):
|
||||
self.handler._schedule_recheck(path)
|
||||
self.assertIsNone(self.handler._worker)
|
||||
self.assertEqual(list(self.handler._history_due), [])
|
||||
# #247: a foreign file's sealed re-check is dropped the same way
|
||||
self.handler._await_foreign(path)
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [])
|
||||
self.handler._schedule_recheck(path, {"filename": "late.png"})
|
||||
self.assertEqual(list(self.handler._foreign_due), [])
|
||||
self.assertEqual(self._live_workers() - before, set())
|
||||
|
||||
def test_a_sealed_run_starts_a_worker_on_an_open_handler(self):
|
||||
|
||||
@@ -1,319 +0,0 @@
|
||||
"""Removing one file's records from the library, and removal marks (#480)."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.image_removal_schema import (
|
||||
image_removals_schema_error,
|
||||
migrate_image_removals,
|
||||
)
|
||||
from database.models import PromptModel
|
||||
from database.operations import PromptDatabase
|
||||
|
||||
|
||||
def _locator(path):
|
||||
return os.path.normcase(os.path.realpath(str(path)))
|
||||
|
||||
|
||||
class _RemovalCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.addCleanup(assert_closed, self, self.temp.name)
|
||||
self.addCleanup(PromptModel.reset_schema_cache)
|
||||
base = Path(self.temp.name)
|
||||
self.root = base / "output"
|
||||
self.root.mkdir()
|
||||
# A second spelling of the folder (a symlink); Windows without the
|
||||
# privilege has none, and the records then carry one spelling.
|
||||
self.alias = base / "alias"
|
||||
try:
|
||||
self.alias.symlink_to(self.root, target_is_directory=True)
|
||||
except OSError:
|
||||
self.alias = None
|
||||
self.db = PromptDatabase(str(base / "prompts.db"))
|
||||
self.addCleanup(self.db.close_all)
|
||||
self.epoch = self.db.get_execution_epoch()
|
||||
|
||||
def file(self, name, data=b"image-bytes"):
|
||||
path = self.root / name
|
||||
path.write_bytes(data)
|
||||
return path
|
||||
|
||||
def removal(self, path):
|
||||
root_key = os.path.normcase(os.path.realpath(str(self.root)))
|
||||
return {
|
||||
"spellings": [str(path)]
|
||||
+ ([str(self.alias / path.name)] if self.alias else []),
|
||||
"locator": _locator(path),
|
||||
"favorites": [(root_key, path.name.lower())],
|
||||
"category": (root_key, path.name.lower()),
|
||||
}
|
||||
|
||||
def link(self, path, text="a prompt", spelled=None):
|
||||
prompt_id = self.db.save_prompt(text)
|
||||
image_id = self.db.link_image_to_prompt(str(prompt_id), str(spelled or path))
|
||||
self.assertTrue(image_id)
|
||||
return prompt_id, image_id
|
||||
|
||||
def tag(self, path, tags):
|
||||
data = path.read_bytes()
|
||||
info = os.stat(path)
|
||||
return self.db.replace_image_tags(
|
||||
locator=_locator(path),
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
size=info.st_size,
|
||||
mtime_ns=info.st_mtime_ns,
|
||||
expected_epoch=self.epoch,
|
||||
expected_tag_revision=0,
|
||||
tags=tags,
|
||||
)
|
||||
|
||||
def queue(self, path):
|
||||
data = path.read_bytes()
|
||||
info = os.stat(path)
|
||||
return self.db.record_review_item(
|
||||
locator=_locator(path),
|
||||
spelled_path=os.path.realpath(str(path)),
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
size=info.st_size,
|
||||
mtime_ns=info.st_mtime_ns,
|
||||
epoch=self.epoch,
|
||||
resolver_version=1,
|
||||
evidence={"candidates": []},
|
||||
candidate_count=0,
|
||||
limited=True,
|
||||
)
|
||||
|
||||
def count(self, sql, *params):
|
||||
conn = sqlite3.connect(self.db.model.db_path)
|
||||
try:
|
||||
return conn.execute(sql, params).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def decorate(self, path):
|
||||
"""Give *path* one record of every kind."""
|
||||
removal = self.removal(path)
|
||||
root_key, relative = removal["favorites"][0]
|
||||
self.tag(path, ["portrait"])
|
||||
self.db.set_image_favorite(root_key, relative)
|
||||
self.db.set_image_categories([removal["category"]], "keepers")
|
||||
self.db.add_to_collection(items=[(root_key, relative)], name="Set")
|
||||
return removal
|
||||
|
||||
|
||||
class ImageRemovalRecordsTests(_RemovalCase):
|
||||
def test_counts_and_removes_every_record_under_every_spelling(self):
|
||||
path = self.file("a.png")
|
||||
other = self.file("b.png", b"other")
|
||||
orphan, _ = self.link(path, "only this image")
|
||||
shared, _ = self.link(path, "two images")
|
||||
self.db.link_image_to_prompt(str(shared), str(other))
|
||||
# A record under the symlinked folder spelling (or a third prompt).
|
||||
self.link(path, "aliased", spelled=(self.alias or self.root) / "a.png")
|
||||
removal = self.decorate(path)
|
||||
self.assertTrue(self.queue(path))
|
||||
|
||||
counted = self.db.count_removal_records([removal])
|
||||
self.assertEqual(
|
||||
counted["records"],
|
||||
{
|
||||
"prompt_links": 3,
|
||||
"pair_links": 0,
|
||||
"image_tags": 1,
|
||||
"auto_tagged": 0,
|
||||
"categories": 1,
|
||||
"collections": 1,
|
||||
"favorites": 1,
|
||||
"review_items": 1,
|
||||
},
|
||||
)
|
||||
self.assertEqual(counted["orphaned_prompts"]["count"], 2)
|
||||
|
||||
removed = self.db.remove_image_records(removal)
|
||||
self.assertEqual(removed["prompt_links"], 3)
|
||||
self.assertEqual(removed["review_items"], 1)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 1)
|
||||
for table in (
|
||||
"media_image_tags",
|
||||
"media_annotation_files",
|
||||
"media_review_items",
|
||||
"image_favorites",
|
||||
"image_categories",
|
||||
"image_collection_items",
|
||||
):
|
||||
with self.subTest(table=table):
|
||||
self.assertEqual(self.count(f"SELECT COUNT(*) FROM {table}"), 0)
|
||||
# Prompts and the collection itself stay.
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM prompts"), 3)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM image_collections"), 1)
|
||||
self.assertTrue(path.exists())
|
||||
self.assertIsNotNone(self.db.get_prompt_by_id(orphan))
|
||||
|
||||
def test_a_kept_review_item_keeps_the_annotation_rows(self):
|
||||
path = self.file("a.png")
|
||||
removal = self.decorate(path)
|
||||
self.assertTrue(self.queue(path))
|
||||
conn = sqlite3.connect(self.db.model.db_path)
|
||||
try:
|
||||
# A revised item is not settled; it still cites the bytes, whose
|
||||
# foreign key is ON DELETE RESTRICT.
|
||||
conn.execute("UPDATE media_review_items SET revision=1")
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
removed = self.db.remove_image_records(removal)
|
||||
self.assertEqual(removed["review_items"], 0)
|
||||
self.assertEqual(removed["image_tags"], 1)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM media_image_tags"), 0)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM media_review_items"), 1)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM media_annotation_files"), 1)
|
||||
|
||||
def test_mark_is_written_with_the_records_and_bumps_the_revision(self):
|
||||
path = self.file("a.png")
|
||||
self.link(path)
|
||||
before = self.db.removal_revision()
|
||||
info = os.stat(path)
|
||||
self.db.remove_image_records(
|
||||
self.removal(path), mark=(info.st_size, info.st_mtime_ns)
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.removal_marks(),
|
||||
{_locator(path): (info.st_size, info.st_mtime_ns)},
|
||||
)
|
||||
self.assertEqual(self.db.removal_revision(), before + 1)
|
||||
self.assertEqual(self.db.clear_removal_marks([_locator(path)]), 1)
|
||||
self.assertEqual(self.db.removal_marks(), {})
|
||||
self.assertEqual(self.db.removal_revision(), before + 2)
|
||||
self.assertEqual(self.db.clear_removal_marks([_locator(path)]), 0)
|
||||
|
||||
def test_a_failure_rolls_back_everything(self):
|
||||
path = self.file("a.png")
|
||||
self.link(path)
|
||||
removal = self.decorate(path)
|
||||
broken = {**removal, "category": ("only-one-value",)}
|
||||
with self.assertRaises(sqlite3.Error):
|
||||
self.db.remove_image_records(broken, mark=(1, 1))
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 1)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM media_image_tags"), 1)
|
||||
self.assertEqual(self.db.removal_marks(), {})
|
||||
|
||||
def test_parity_with_the_legacy_single_delete(self):
|
||||
path = self.file("a.png")
|
||||
_prompt, image_id = self.link(path)
|
||||
twin = PromptDatabase(str(Path(self.temp.name) / "twin.db"))
|
||||
try:
|
||||
twin_prompt = twin.save_prompt("a prompt")
|
||||
twin_image = twin.link_image_to_prompt(str(twin_prompt), str(path))
|
||||
self.assertTrue(twin.delete_image(twin_image))
|
||||
self.db.remove_image_records(self.removal(path))
|
||||
for table in (
|
||||
"generated_images",
|
||||
"image_prompt_pairs",
|
||||
"workflow_documents",
|
||||
):
|
||||
with self.subTest(table=table):
|
||||
conn = sqlite3.connect(twin.model.db_path)
|
||||
try:
|
||||
legacy = conn.execute(
|
||||
f"SELECT COUNT(*) FROM {table}"
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(
|
||||
self.count(f"SELECT COUNT(*) FROM {table}"), legacy
|
||||
)
|
||||
finally:
|
||||
twin.close_all()
|
||||
self.assertIsNotNone(image_id)
|
||||
|
||||
|
||||
class RemovalMarkGateTests(_RemovalCase):
|
||||
"""A marked file is never linked again; any record write shows it again."""
|
||||
|
||||
def hide(self, path):
|
||||
info = os.stat(path)
|
||||
self.db.remove_image_records(
|
||||
self.removal(path), mark=(info.st_size, info.st_mtime_ns)
|
||||
)
|
||||
|
||||
def test_a_marked_file_is_not_linked_again(self):
|
||||
path = self.file("a.png")
|
||||
self.hide(path)
|
||||
prompt_id = self.db.save_prompt("scanned again")
|
||||
self.assertEqual(self.db.link_image_to_prompt(str(prompt_id), str(path)), 0)
|
||||
self.assertEqual(self.count("SELECT COUNT(*) FROM generated_images"), 0)
|
||||
self.assertIn(_locator(path), self.db.removal_marks())
|
||||
|
||||
def test_a_changed_file_drops_its_stale_mark_and_links(self):
|
||||
path = self.file("a.png")
|
||||
self.hide(path)
|
||||
path.write_bytes(b"new bytes, new size")
|
||||
prompt_id = self.db.save_prompt("a new image at the same path")
|
||||
self.assertTrue(self.db.link_image_to_prompt(str(prompt_id), str(path)))
|
||||
self.assertEqual(self.db.removal_marks(), {})
|
||||
|
||||
def test_explicit_record_writes_clear_the_mark(self):
|
||||
writers = {
|
||||
"favorite": lambda path, key: self.db.set_image_favorite(*key),
|
||||
"category": lambda path, key: self.db.set_image_categories([key], "c"),
|
||||
"collection": lambda path, key: self.db.add_to_collection(
|
||||
items=[key], name="Set"
|
||||
),
|
||||
"tags": lambda path, key: self.tag(path, ["t"]),
|
||||
}
|
||||
for kind, write in writers.items():
|
||||
with self.subTest(kind=kind):
|
||||
path = self.file(f"{kind}.png", kind.encode())
|
||||
self.hide(path)
|
||||
self.assertIn(_locator(path), self.db.removal_marks())
|
||||
write(path, self.removal(path)["favorites"][0])
|
||||
self.assertNotIn(_locator(path), self.db.removal_marks())
|
||||
|
||||
|
||||
class ImageRemovalSchemaTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.conn = sqlite3.connect(":memory:")
|
||||
self.addCleanup(self.conn.close)
|
||||
|
||||
def test_missing_is_allowed_only_for_old_backups(self):
|
||||
self.assertIsNone(image_removals_schema_error(self.conn))
|
||||
self.assertIsNotNone(
|
||||
image_removals_schema_error(self.conn, allow_missing=False)
|
||||
)
|
||||
|
||||
def test_install_is_valid_and_idempotent(self):
|
||||
migrate_image_removals(self.conn)
|
||||
migrate_image_removals(self.conn)
|
||||
self.assertIsNone(image_removals_schema_error(self.conn, allow_missing=False))
|
||||
self.assertEqual(
|
||||
self.conn.execute("SELECT revision FROM image_removals_meta").fetchall(),
|
||||
[(0,)],
|
||||
)
|
||||
|
||||
def test_invalid_values_and_objects_are_refused(self):
|
||||
migrate_image_removals(self.conn)
|
||||
self.conn.execute("INSERT INTO image_removals VALUES ('', 1, 1, 'now')")
|
||||
self.assertIn("invalid values", image_removals_schema_error(self.conn))
|
||||
self.conn.execute("DELETE FROM image_removals")
|
||||
self.conn.execute("CREATE INDEX extra ON image_removals(size)")
|
||||
self.assertIn("unsupported", image_removals_schema_error(self.conn))
|
||||
|
||||
def test_a_foreign_table_definition_is_refused(self):
|
||||
self.conn.execute("CREATE TABLE image_removals (locator TEXT)")
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
migrate_image_removals(self.conn)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,251 +0,0 @@
|
||||
"""Removal-mark gaps after #480, and trash folders inside output roots (#501).
|
||||
|
||||
- Thumbnail jobs and the legacy auto-tag listing skip a library-removed file.
|
||||
- Missing-file links also forgets the marks of files that are gone (only under
|
||||
a source that can be read now).
|
||||
- Every walker skips a trash folder that sits inside an output root.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.disk_waits import DISK_BOUND_SECONDS
|
||||
from tests.test_image_delete_routes import ImageDeleteTestCase
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from disk_waits import DISK_BOUND_SECONDS
|
||||
from test_image_delete_routes import ImageDeleteTestCase
|
||||
|
||||
from py.api import images as images_module
|
||||
from py.api.admin import _collect_media_files
|
||||
from py.api.autotag_routes import AutotagRoutesMixin
|
||||
from utils.image_monitor import ImageGenerationHandler
|
||||
|
||||
MAINTENANCE_PREVIEW = "/prompt_manager/maintenance/preview"
|
||||
MAINTENANCE_APPLY = "/prompt_manager/maintenance/apply"
|
||||
|
||||
|
||||
class _GapCase(ImageDeleteTestCase):
|
||||
def marks(self):
|
||||
conn = sqlite3.connect(self.api.db.model.db_path)
|
||||
try:
|
||||
return [
|
||||
row[0] for row in conn.execute("SELECT locator FROM image_removals")
|
||||
]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def revision(self):
|
||||
conn = sqlite3.connect(self.api.db.model.db_path)
|
||||
try:
|
||||
return conn.execute(
|
||||
"SELECT revision FROM image_removals_meta WHERE singleton_id=1"
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class TestThumbnailJobsSkipHiddenFiles(_GapCase):
|
||||
async def test_missing_and_rebuild_plans_leave_a_removed_file_out(self):
|
||||
self.write("hidden.png", b"hidden bytes", mtime=1_000_000)
|
||||
self.write("shown.png", b"shown bytes", mtime=2_000_000)
|
||||
await self.delete(["hidden.png"], "library")
|
||||
thumbs = self.output_dir / "thumbnails"
|
||||
for mode in ("missing", "rebuild"):
|
||||
with self.subTest(mode=mode):
|
||||
plan = self.api._plan_thumbnails(self.output_dir, thumbs, mode=mode)
|
||||
sources = [src.name for src, _dst, _video in plan["targets"]]
|
||||
self.assertEqual(sources, ["shown.png"])
|
||||
self.assertEqual(plan["missing"], 1)
|
||||
|
||||
async def test_a_changed_file_gets_its_preview_again(self):
|
||||
self.write("hidden.png", b"hidden bytes", mtime=1_000_000)
|
||||
await self.delete(["hidden.png"], "library")
|
||||
self.write("hidden.png", b"new and longer bytes", mtime=3_000_000)
|
||||
plan = self.api._plan_thumbnails(
|
||||
self.output_dir, self.output_dir / "thumbnails"
|
||||
)
|
||||
self.assertEqual([t[0].name for t in plan["targets"]], ["hidden.png"])
|
||||
|
||||
|
||||
class TestLegacyAutotagListing(_GapCase):
|
||||
async def test_scan_output_dir_leaves_a_removed_file_out(self):
|
||||
self.write("hidden.png", b"hidden bytes", mtime=1_000_000)
|
||||
self.write("shown.png", b"shown bytes", mtime=2_000_000)
|
||||
await self.delete(["hidden.png"], "library")
|
||||
with patch.object(
|
||||
self.api, "_find_comfyui_output_dir", return_value=str(self.output_dir)
|
||||
):
|
||||
response = await self.client.get("/prompt_manager/scan_output_dir")
|
||||
body = await response.json()
|
||||
self.assertEqual([image["filename"] for image in body["images"]], ["shown.png"])
|
||||
|
||||
|
||||
class TestMissingFileLinksPrunesMarks(_GapCase):
|
||||
async def links_review(self, status=200):
|
||||
return await self.post(
|
||||
MAINTENANCE_PREVIEW, {"operation": "missing_links", "options": {}}, status
|
||||
)
|
||||
|
||||
async def apply(self, token, status=202):
|
||||
body = await self.post(MAINTENANCE_APPLY, {"token": token}, status)
|
||||
if status != 202:
|
||||
return body
|
||||
job = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
return job
|
||||
|
||||
async def removed(self, name):
|
||||
self.write("other.png", b"another output") # the source lists something
|
||||
self.write(name, f"bytes of {name}".encode(), mtime=1_000_000)
|
||||
await self.delete([name], "library")
|
||||
self.assertEqual(len(self.marks()), 1)
|
||||
|
||||
async def test_the_mark_of_a_gone_file_is_counted_and_forgotten(self):
|
||||
await self.removed("gone.png")
|
||||
# Not an unlink by PromptManager: the user deleted it elsewhere.
|
||||
os.remove(self.output_dir / "gone.png")
|
||||
plan = await self.links_review()
|
||||
self.assertEqual(plan["scope"]["count"], 1)
|
||||
self.assertEqual(
|
||||
plan["scope"]["label"], "1 removed image whose file is gone or changed"
|
||||
)
|
||||
before = self.revision()
|
||||
job = await self.apply(plan["token"])
|
||||
self.assertEqual(job.state, "finished", job.summary())
|
||||
self.assertEqual(self.marks(), [])
|
||||
self.assertGreater(self.revision(), before)
|
||||
|
||||
async def test_the_mark_of_a_present_file_stays(self):
|
||||
await self.removed("kept.png")
|
||||
plan = await self.links_review()
|
||||
self.assertEqual(plan["scope"]["count"], 0)
|
||||
self.assertEqual(plan["scope"]["label"], "No links to missing files")
|
||||
self.assertEqual(len(self.marks()), 1)
|
||||
|
||||
async def test_a_mark_under_an_unreadable_source_stays(self):
|
||||
await self.removed("away.png")
|
||||
os.remove(self.output_dir / "away.png")
|
||||
with patch(
|
||||
"database.maintenance.readable_roots",
|
||||
side_effect=lambda r: [False] * len(r),
|
||||
):
|
||||
plan = await self.links_review()
|
||||
self.assertEqual(plan["scope"]["count"], 0)
|
||||
self.assertEqual(len(self.marks()), 1)
|
||||
|
||||
async def test_the_mark_of_a_changed_file_is_forgotten(self):
|
||||
await self.removed("changed.png")
|
||||
self.write("changed.png", b"other and longer bytes", mtime=4_000_000)
|
||||
plan = await self.links_review()
|
||||
self.assertEqual(plan["scope"]["count"], 1)
|
||||
job = await self.apply(plan["token"])
|
||||
self.assertEqual(job.state, "finished", job.summary())
|
||||
self.assertEqual(self.marks(), [])
|
||||
|
||||
async def test_marks_under_an_empty_source_stay(self):
|
||||
# An unplugged drive leaves its mount point: an empty, readable folder.
|
||||
await self.removed("mounted.png")
|
||||
for child in list(self.output_dir.iterdir()):
|
||||
if child.is_dir():
|
||||
shutil.rmtree(child)
|
||||
else:
|
||||
os.remove(child)
|
||||
plan = await self.links_review()
|
||||
self.assertEqual(plan["scope"]["count"], 0)
|
||||
self.assertEqual(len(self.marks()), 1)
|
||||
|
||||
async def test_a_name_the_folder_still_lists_is_not_gone(self):
|
||||
# A case-sensitive NTFS folder: lstat of the normcased locator fails.
|
||||
await self.removed("Listed.png")
|
||||
real_lstat = os.lstat
|
||||
locator = self.marks()[0]
|
||||
|
||||
def lstat(path, *args, **kwargs):
|
||||
if os.fspath(path) == locator:
|
||||
raise FileNotFoundError(path)
|
||||
return real_lstat(path, *args, **kwargs)
|
||||
|
||||
with (
|
||||
patch("database.maintenance.os.lstat", side_effect=lstat),
|
||||
patch("database.maintenance.os.path.normcase", side_effect=str.lower),
|
||||
):
|
||||
from database.maintenance import _stale_marks
|
||||
|
||||
stale = _stale_marks([(locator, 1, 1)], [str(self.output_dir)])
|
||||
self.assertEqual(stale, [])
|
||||
|
||||
async def test_a_file_back_before_the_apply_changes_the_plan(self):
|
||||
await self.removed("back.png")
|
||||
path = self.output_dir / "back.png"
|
||||
saved = Path(self.tmpdir) / "saved.png"
|
||||
shutil.copy2(path, saved)
|
||||
os.remove(path)
|
||||
plan = await self.links_review()
|
||||
shutil.copy2(saved, path)
|
||||
job = await self.apply(plan["token"])
|
||||
self.assertEqual(job.state, "failed", job.summary())
|
||||
self.assertEqual(len(self.marks()), 1)
|
||||
|
||||
|
||||
class TestWalkersSkipTrashFolders(_GapCase):
|
||||
def seed(self):
|
||||
self.write("kept.png", b"kept")
|
||||
self.write(
|
||||
"x.png", b"trashed x", folder=self.output_dir / ".Trash-1000" / "files"
|
||||
)
|
||||
self.write(
|
||||
"y.png", b"trashed y", folder=self.output_dir / "sub" / "$RECYCLE.BIN"
|
||||
)
|
||||
self.write("z.png", b"trashed z", folder=self.output_dir / ".Trashes")
|
||||
|
||||
def test_walk_media_and_the_settled_entry(self):
|
||||
self.seed()
|
||||
walked = sorted(
|
||||
Path(entry.path).name
|
||||
for entry in images_module._walk_media(self.output_dir, 100)
|
||||
)
|
||||
self.assertEqual(walked, ["kept.png"])
|
||||
for relative in (
|
||||
os.path.join(".Trash-1000", "files", "x.png"),
|
||||
os.path.join("sub", "$RECYCLE.BIN", "y.png"),
|
||||
):
|
||||
with self.subTest(relative=relative):
|
||||
self.assertIsNone(
|
||||
images_module._settled_entry(self.output_dir, relative)
|
||||
)
|
||||
|
||||
def test_scan_duplicate_and_autotag_walkers(self):
|
||||
self.seed()
|
||||
collected = [
|
||||
Path(p).name for p in _collect_media_files([self.output_dir], [".png"])
|
||||
]
|
||||
self.assertEqual(collected, ["kept.png"])
|
||||
found, _truncated = AutotagRoutesMixin._collect_scan_files(self.output_dir)
|
||||
self.assertEqual([p.name for p in found], ["kept.png"])
|
||||
|
||||
def test_the_image_monitor(self):
|
||||
monitor = ImageGenerationHandler.__new__(ImageGenerationHandler)
|
||||
monitor.supported_extensions = (".png",)
|
||||
base = str(self.output_dir)
|
||||
self.assertTrue(monitor.is_image_file(os.path.join(base, "a.png")))
|
||||
for folder in (".Trash-1000", "$Recycle.Bin", ".Trashes", ".trash"):
|
||||
with self.subTest(folder=folder):
|
||||
self.assertFalse(
|
||||
monitor.is_image_file(os.path.join(base, folder, "files", "a.png"))
|
||||
)
|
||||
|
||||
async def test_the_workspace_listing(self):
|
||||
self.seed()
|
||||
self.assertEqual(await self.listed(), ["kept.png"])
|
||||
|
||||
def test_a_root_named_like_a_trash_folder_still_lists(self):
|
||||
root = Path(self.tmpdir) / ".Trash-1000"
|
||||
(root / "inner").mkdir(parents=True)
|
||||
(root / "inner" / "a.png").write_bytes(b"a")
|
||||
walked = [Path(e.path).name for e in images_module._walk_media(root, 10)]
|
||||
self.assertEqual(walked, ["a.png"])
|
||||
@@ -887,36 +887,6 @@ class TestImageTagsFacet(unittest.TestCase):
|
||||
self.assertNotIn("image_tags", json.loads(encode(inspection({}))))
|
||||
self.assertNotIn("image_tags", self.encoded(None))
|
||||
|
||||
def history_encoded(self, facet):
|
||||
from utils.inspection_response import encode_history_inspection_response
|
||||
|
||||
body = encode_history_inspection_response(
|
||||
current_output={"history_epoch": "epoch"},
|
||||
file={"observation": "current-file", "size": 100},
|
||||
prompts=PROMPTS,
|
||||
resolution={"state": "resolved", "sampler_id": "3", "reason": ""},
|
||||
inspection=inspection({"prompt": graph_text()}),
|
||||
library={"status": "none"},
|
||||
image_tags=facet,
|
||||
)
|
||||
return json.loads(body)
|
||||
|
||||
def test_history_envelope_validates_image_tags_like_the_rooted_one(self):
|
||||
"""#457: History's current-file read carries the same facet."""
|
||||
for facet in (
|
||||
{"state": "current", "tags": ["dunes", "warm_light"]},
|
||||
{"state": "none", "tags": []},
|
||||
{"state": "unavailable", "tags": []},
|
||||
{"state": "current", "tags": ["a,b"]},
|
||||
{"state": "current", "tags": [], "locator": "/abs/path.png"},
|
||||
):
|
||||
with self.subTest(facet=facet):
|
||||
self.assertEqual(
|
||||
self.history_encoded(facet)["image_tags"],
|
||||
self.encoded(facet)["image_tags"],
|
||||
)
|
||||
self.assertNotIn("image_tags", self.history_encoded(None))
|
||||
|
||||
|
||||
class TestVideoProbeBoundsMatchPngBudgets(unittest.TestCase):
|
||||
"""ffprobe escapes tag text; the stdout bounds must never gate a record
|
||||
|
||||
@@ -345,8 +345,7 @@ class TestLibrary(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
(parent["title"], parent["notes"]), ("Portrait", "keep\nnotes")
|
||||
)
|
||||
# The parent's own archive state (#443), never a pair's.
|
||||
self.assertIs(parent["archived"], False)
|
||||
self.assertNotIn("archived", parent)
|
||||
self.assertRegex(parent["edit_token"], r"^[0-9a-f]{64}$")
|
||||
self.assertEqual(page["items"][1]["title"], " \t\u00a0\u3000")
|
||||
self.assertEqual(self.db.list_library(text="Portrait")["total"], 1)
|
||||
@@ -1521,7 +1520,6 @@ class TestLibrary(unittest.TestCase):
|
||||
"tags",
|
||||
"rating",
|
||||
"notes",
|
||||
"archived",
|
||||
"variants",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -1,353 +0,0 @@
|
||||
"""#443 (D152): the Library delete, archive and restore routes, and ingestion after."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
from tests.test_api_admin import AdminAPITestCase # first: it stubs ComfyUI
|
||||
from database.execution_history_schema import rotate_history_epoch # noqa: E402
|
||||
from database.prompt_archive import TABLE as ARCHIVE # noqa: E402
|
||||
from database.safety_backup import SafetyBackupFailed # noqa: E402
|
||||
from py.config import PromptManagerConfig # noqa: E402
|
||||
from utils.hashing import generate_prompt_hash # noqa: E402
|
||||
|
||||
PREVIEW = "/prompt_manager/library/delete/preview"
|
||||
DELETE = "/prompt_manager/library/delete"
|
||||
RESTORE = "/prompt_manager/library/restore"
|
||||
|
||||
|
||||
def graph(positive, negative="blur"):
|
||||
return {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": positive}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": negative}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["2", 0]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class LibraryDeleteRouteCase(AdminAPITestCase):
|
||||
def epoch(self):
|
||||
return self.api.db.get_library_options()["library_epoch"]
|
||||
|
||||
def save(self, text, **metadata):
|
||||
return self.api.db.save_prompt(
|
||||
text, prompt_hash=generate_prompt_hash(text), **metadata
|
||||
)
|
||||
|
||||
def paired(self, positive, negative="blur"):
|
||||
pair = self.api.db.save_prompt_pair(positive, negative)
|
||||
return self.api.db.get_prompt_pair(pair)["positive_prompt_id"]
|
||||
|
||||
def rows(self, sql, params=()):
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
return [tuple(row) for row in conn.execute(sql, params).fetchall()]
|
||||
|
||||
def ids(self):
|
||||
return sorted(row[0] for row in self.rows("SELECT id FROM prompts"))
|
||||
|
||||
def archived(self):
|
||||
return sorted(row[0] for row in self.rows(f"SELECT prompt_id FROM {ARCHIVE}"))
|
||||
|
||||
def backups(self):
|
||||
folder = os.path.dirname(self.api.db.model.db_path)
|
||||
return sorted(name for name in os.listdir(folder) if ".backup_" in name)
|
||||
|
||||
async def post(self, url, body):
|
||||
response = await self.client.request("POST", url, json=body)
|
||||
return response.status, await response.json()
|
||||
|
||||
async def review(self, ids):
|
||||
status, body = await self.post(
|
||||
PREVIEW, {"prompt_ids": ids, "expected_epoch": self.epoch()}
|
||||
)
|
||||
self.assertEqual(status, 200, body)
|
||||
return body
|
||||
|
||||
async def confirm(self, review, backup=None):
|
||||
return await self.post(
|
||||
DELETE,
|
||||
{
|
||||
"archive_ids": review["archive"],
|
||||
"delete_ids": review["delete"],
|
||||
"expected_epoch": review["library_epoch"],
|
||||
"backup": backup or review["backup"],
|
||||
},
|
||||
)
|
||||
|
||||
def policy(self, on):
|
||||
previous = PromptManagerConfig.SAFETY_BACKUP
|
||||
PromptManagerConfig.SAFETY_BACKUP = on
|
||||
self.addCleanup(setattr, PromptManagerConfig, "SAFETY_BACKUP", previous)
|
||||
|
||||
|
||||
class TestLibraryDeleteRoutes(LibraryDeleteRouteCase):
|
||||
async def test_preview_then_delete_archives_deletes_and_backs_up(self):
|
||||
self.policy(True)
|
||||
plain = self.save("Plain")
|
||||
guarded = self.save("Guarded", tags=["__protected__"])
|
||||
positive = self.paired("Cat")
|
||||
review = await self.review([plain, guarded, positive, 99999])
|
||||
self.assertEqual(
|
||||
{
|
||||
key: review[key]
|
||||
for key in ("delete", "archive", "protected", "not_found")
|
||||
},
|
||||
{
|
||||
"delete": [plain],
|
||||
"archive": [positive],
|
||||
"protected": [guarded],
|
||||
"not_found": 1,
|
||||
},
|
||||
)
|
||||
self.assertEqual(review["backup"], "on")
|
||||
self.assertTrue(review["disk"]["sufficient"])
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(
|
||||
(body["deleted"], body["archived"], body["changed"]), (1, 1, 0)
|
||||
)
|
||||
self.assertEqual(self.backups(), [body["backup"]])
|
||||
self.assertNotIn(os.sep, body["backup"])
|
||||
self.assertNotIn(plain, self.ids())
|
||||
self.assertEqual(self.archived(), [positive])
|
||||
# The backup still holds the deleted prompt.
|
||||
import sqlite3
|
||||
|
||||
path = os.path.join(os.path.dirname(self.api.db.model.db_path), body["backup"])
|
||||
with sqlite3.connect(path) as copy:
|
||||
self.assertEqual(
|
||||
copy.execute(
|
||||
"SELECT COUNT(*) FROM prompts WHERE id=?", (plain,)
|
||||
).fetchone()[0],
|
||||
1,
|
||||
)
|
||||
copy.close()
|
||||
# The hold ended: nothing is listed as running.
|
||||
self.assertEqual(self.api._jobs.running(), [])
|
||||
|
||||
async def test_archive_only_takes_no_backup(self):
|
||||
self.policy(True)
|
||||
positive = self.paired("Cat")
|
||||
review = await self.review([positive])
|
||||
self.assertIsNone(review["disk"])
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual((status, body["archived"], body["backup"]), (200, 1, None))
|
||||
self.assertEqual(self.backups(), [])
|
||||
|
||||
async def test_policy_off_takes_none_unless_the_review_promised_one(self):
|
||||
plain = self.save("Plain")
|
||||
other = self.save("Other")
|
||||
self.policy(False)
|
||||
review = await self.review([plain])
|
||||
self.assertEqual((review["backup"], review["disk"]), ("off", None))
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual((status, body["deleted"], body["backup"]), (200, 1, None))
|
||||
# Reviewed with the policy on, turned off before the confirm: kept.
|
||||
review = await self.review([other])
|
||||
status, body = await self.confirm(review, backup="on")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertIsNotNone(body["backup"])
|
||||
|
||||
async def test_a_failed_backup_changes_nothing(self):
|
||||
self.policy(True)
|
||||
plain = self.save("Plain")
|
||||
positive = self.paired("Cat")
|
||||
review = await self.review([plain, positive])
|
||||
with patch(
|
||||
"py.api.maintenance.take_safety_backup",
|
||||
side_effect=SafetyBackupFailed("x"),
|
||||
):
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual((status, body["code"]), (500, "backup-failed"))
|
||||
self.assertIn("nothing was changed", body["error"])
|
||||
self.assertIn(plain, self.ids())
|
||||
self.assertEqual(self.archived(), [])
|
||||
|
||||
async def test_a_short_disk_refuses_before_any_change(self):
|
||||
self.policy(True)
|
||||
plain = self.save("Plain")
|
||||
from py.api import maintenance
|
||||
|
||||
with patch.object(maintenance, "_free_bytes", return_value=0):
|
||||
review = await self.review([plain])
|
||||
self.assertFalse(review["disk"]["sufficient"])
|
||||
self.assertIn("Nothing was changed", review["disk"]["message"])
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual((status, body["code"]), (507, "insufficient-space"))
|
||||
self.assertIn(plain, self.ids())
|
||||
|
||||
async def test_busy_beside_a_record_changing_job(self):
|
||||
plain = self.save("Plain")
|
||||
review = await self.review([plain])
|
||||
async with self.api._jobs.hold("scan", "Scan", admission="records"):
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual((status, body["code"]), (409, "busy"))
|
||||
self.assertEqual(body["running"]["title"], "Scan")
|
||||
self.assertIn(plain, self.ids())
|
||||
|
||||
async def test_restored_storage_answers_409_on_every_route(self):
|
||||
plain = self.save("Plain")
|
||||
review = await self.review([plain])
|
||||
rotate_history_epoch(self.api.db.model.get_connection())
|
||||
status, body = await self.confirm(review)
|
||||
self.assertEqual((status, body["code"]), (409, "storage-changed"))
|
||||
for url in (PREVIEW, RESTORE):
|
||||
status, body = await self.post(
|
||||
url, {"prompt_ids": [plain], "expected_epoch": review["library_epoch"]}
|
||||
)
|
||||
self.assertEqual((status, body["code"]), (409, "storage-changed"))
|
||||
self.assertIn(plain, self.ids())
|
||||
|
||||
async def test_restore_route(self):
|
||||
positive = self.paired("Cat")
|
||||
await self.confirm(await self.review([positive]))
|
||||
status, body = await self.post(
|
||||
RESTORE, {"prompt_ids": [positive, 99999], "expected_epoch": self.epoch()}
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual((body["restored"], body["not_found"]), (1, 1))
|
||||
self.assertEqual(self.archived(), [])
|
||||
|
||||
async def test_bodies_are_validated(self):
|
||||
plain = self.save("Plain")
|
||||
epoch = self.epoch()
|
||||
selection = {"prompt_ids": [plain], "expected_epoch": epoch}
|
||||
for url, body in (
|
||||
(PREVIEW, {**selection, "extra": 1}),
|
||||
(PREVIEW, {"prompt_ids": [], "expected_epoch": epoch}),
|
||||
(PREVIEW, {"prompt_ids": [True], "expected_epoch": epoch}),
|
||||
(PREVIEW, {"prompt_ids": list(range(1, 1002)), "expected_epoch": epoch}),
|
||||
(PREVIEW, {"prompt_ids": [plain], "expected_epoch": "nope"}),
|
||||
(RESTORE, {"prompt_ids": [plain]}),
|
||||
(DELETE, {**selection}),
|
||||
(
|
||||
DELETE,
|
||||
{
|
||||
"archive_ids": [],
|
||||
"delete_ids": [],
|
||||
"expected_epoch": epoch,
|
||||
"backup": "on",
|
||||
},
|
||||
),
|
||||
(
|
||||
DELETE,
|
||||
{
|
||||
"archive_ids": [plain],
|
||||
"delete_ids": [plain],
|
||||
"expected_epoch": epoch,
|
||||
"backup": "on",
|
||||
},
|
||||
),
|
||||
(
|
||||
DELETE,
|
||||
{
|
||||
"archive_ids": [],
|
||||
"delete_ids": [plain],
|
||||
"expected_epoch": epoch,
|
||||
"backup": "maybe",
|
||||
},
|
||||
),
|
||||
(
|
||||
DELETE,
|
||||
{
|
||||
"archive_ids": list(range(1, 600)),
|
||||
"delete_ids": list(range(600, 1100)),
|
||||
"expected_epoch": epoch,
|
||||
"backup": "off",
|
||||
},
|
||||
),
|
||||
):
|
||||
with self.subTest(url=url, body=body):
|
||||
status, answer = await self.post(url, body)
|
||||
self.assertEqual(status, 400, answer)
|
||||
self.assertFalse(answer["success"])
|
||||
for url in (PREVIEW, DELETE, RESTORE):
|
||||
response = await self.client.request(
|
||||
"POST", url, data="[1]", headers={"Content-Type": "application/json"}
|
||||
)
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertIn(plain, self.ids())
|
||||
|
||||
async def test_the_legacy_bulk_delete_is_unchanged(self):
|
||||
plain = self.save("Plain")
|
||||
positive = self.paired("Cat")
|
||||
response = await self.client.request(
|
||||
"POST", "/prompt_manager/bulk/delete", json={"prompt_ids": [positive]}
|
||||
)
|
||||
self.assertEqual(response.status, 409)
|
||||
response = await self.client.request(
|
||||
"POST", "/prompt_manager/bulk/delete", json={"prompt_ids": [plain]}
|
||||
)
|
||||
self.assertEqual((await response.json())["deleted_count"], 1)
|
||||
self.assertEqual(self.archived(), [])
|
||||
|
||||
async def test_shelf_parameter_and_cursor_scope(self):
|
||||
kept = [self.paired(f"Kept {i}") for i in range(3)]
|
||||
gone = [self.paired(f"Gone {i}") for i in range(3)]
|
||||
await self.confirm(await self.review(gone))
|
||||
response = await self.client.request(
|
||||
"GET", "/prompt_manager/library?limit=2&shelf=active"
|
||||
)
|
||||
page = await response.json()
|
||||
self.assertEqual(page["archived_total"], 3)
|
||||
self.assertTrue(set(item["id"] for item in page["items"]) <= set(kept))
|
||||
response = await self.client.request(
|
||||
"GET",
|
||||
"/prompt_manager/library?limit=2&shelf=archived"
|
||||
f"&cursor={page['next_cursor']}",
|
||||
)
|
||||
self.assertEqual(response.status, 400)
|
||||
response = await self.client.request(
|
||||
"GET", "/prompt_manager/library?shelf=archived"
|
||||
)
|
||||
archived = await response.json()
|
||||
self.assertEqual(sorted(item["id"] for item in archived["items"]), gone)
|
||||
self.assertTrue(all(item["archived"] for item in archived["items"]))
|
||||
response = await self.client.request("GET", "/prompt_manager/library?shelf=all")
|
||||
self.assertEqual(response.status, 400)
|
||||
|
||||
|
||||
class TestIngestionAfterArchive(LibraryDeleteRouteCase):
|
||||
def scan(self, name, positive, negative="blur"):
|
||||
meta = {"prompt": json.dumps(graph(positive, negative))}
|
||||
path = self.output_dir / name
|
||||
path.write_bytes(b"png")
|
||||
return self.api._ingest_scanned_file(path, meta)
|
||||
|
||||
async def test_a_scan_after_archive_creates_no_duplicate_and_keeps_it_archived(
|
||||
self,
|
||||
):
|
||||
self.assertEqual(self.scan("first.png", "Sunset"), "added")
|
||||
positive = self.api.db.get_prompt_by_hash(generate_prompt_hash("Sunset"))["id"]
|
||||
await self.confirm(await self.review([positive]))
|
||||
before = self.ids()
|
||||
self.scan("second.png", "Sunset")
|
||||
self.scan("third.png", " sunset ", "other negative")
|
||||
self.assertEqual(
|
||||
[i for i in self.ids() if i not in before],
|
||||
[
|
||||
self.api.db.get_prompt_by_hash(generate_prompt_hash("other negative"))[
|
||||
"id"
|
||||
]
|
||||
],
|
||||
)
|
||||
self.assertEqual(self.archived(), [positive])
|
||||
linked = self.rows(
|
||||
"SELECT COUNT(*) FROM generated_images WHERE prompt_id=?", (positive,)
|
||||
)
|
||||
self.assertEqual(linked, [(3,)])
|
||||
response = await self.client.request("GET", "/prompt_manager/library")
|
||||
self.assertNotIn(
|
||||
positive, [item["id"] for item in (await response.json())["items"]]
|
||||
)
|
||||
|
||||
async def test_a_scan_after_delete_creates_the_prompt_again(self):
|
||||
plain = self.save("Lonely sunset")
|
||||
await self.confirm(await self.review([plain]))
|
||||
self.assertEqual(self.scan("again.png", "Lonely sunset"), "added")
|
||||
again = self.api.db.get_prompt_by_hash(generate_prompt_hash("Lonely sunset"))
|
||||
self.assertIsNotNone(again)
|
||||
self.assertNotEqual(again["id"], plain)
|
||||
self.assertEqual(self.archived(), [])
|
||||
@@ -14,7 +14,6 @@ import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
@@ -754,17 +753,6 @@ class LocalMediaTests(LayoutTestCase):
|
||||
self.assertEqual(layout_module.VIDEO_SUFFIXES, frozenset(VIDEO_EXTENSIONS))
|
||||
|
||||
|
||||
# Windows path rules without the filesystem. On Windows ntpath.realpath asks
|
||||
# the network for \\server\share and waits seconds for a host that does not
|
||||
# exist (#487); elsewhere ntpath.realpath already is ntpath.abspath.
|
||||
LEXICAL_NT = SimpleNamespace(
|
||||
normcase=ntpath.normcase,
|
||||
realpath=ntpath.abspath,
|
||||
dirname=ntpath.dirname,
|
||||
sep=ntpath.sep,
|
||||
)
|
||||
|
||||
|
||||
class FolderGuardTests(unittest.TestCase):
|
||||
"""Portable path-flavor controls (TESTING.md Windows portability preflight)."""
|
||||
|
||||
@@ -774,7 +762,7 @@ class FolderGuardTests(unittest.TestCase):
|
||||
|
||||
def guard_nt(self, path, home="C:\\Users\\Vito"):
|
||||
return layout_module._guard_folder(
|
||||
path, home, pathmod=LEXICAL_NT, isdir=lambda _p: True
|
||||
path, home, pathmod=ntpath, isdir=lambda _p: True
|
||||
)
|
||||
|
||||
def test_filesystem_roots_and_home_are_rejected(self):
|
||||
@@ -844,12 +832,8 @@ class FolderGuardTests(unittest.TestCase):
|
||||
|
||||
def test_missing_folders_add_nothing(self):
|
||||
self.assertIsNone(layout_module._guard_folder(str(self.tmp / "x"), "/home"))
|
||||
# A missing share on this machine: an unknown server name made
|
||||
# Windows wait about 2.8 s for the network (#487).
|
||||
self.assertIsNone(
|
||||
layout_module._guard_folder(
|
||||
"\\\\localhost\\pm-no-such-share\\loras", str(self.tmp)
|
||||
)
|
||||
layout_module._guard_folder("\\\\server\\share\\loras", str(self.tmp))
|
||||
)
|
||||
self.assertEqual(
|
||||
layout_module._guard_folder(str(self.tmp), "/nonexistent-home"),
|
||||
@@ -859,17 +843,17 @@ class FolderGuardTests(unittest.TestCase):
|
||||
def test_windows_model_path_matches_a_lower_case_root(self):
|
||||
self.assertTrue(
|
||||
layout_module._inside(
|
||||
"C:/Models/Loras/A.safetensors", ("c:\\models\\loras",), LEXICAL_NT
|
||||
"C:/Models/Loras/A.safetensors", ("c:\\models\\loras",), ntpath
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
layout_module._inside(
|
||||
"C:/Models/LorasX/A.safetensors", ("c:\\models\\loras",), LEXICAL_NT
|
||||
"C:/Models/LorasX/A.safetensors", ("c:\\models\\loras",), ntpath
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
layout_module._inside(
|
||||
"\\\\other\\share\\A.safetensors", ("c:\\models\\loras",), LEXICAL_NT
|
||||
"\\\\other\\share\\A.safetensors", ("c:\\models\\loras",), ntpath
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ import email
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import ssl
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
@@ -670,15 +669,6 @@ class TestStaleDownloadSweep(unittest.TestCase):
|
||||
self.assertTrue(old.exists())
|
||||
|
||||
|
||||
def _unused_tls_context():
|
||||
"""A TLS context for the fake handlers, which never open a connection.
|
||||
|
||||
HTTPSHandler() builds a default context, and on Windows that reads the
|
||||
system certificate store: about 0.14 s per handler (#487).
|
||||
"""
|
||||
return ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
|
||||
|
||||
class _FakeHTTPSHandler(urllib.request.HTTPSHandler):
|
||||
"""Serves canned responses through a real urllib opener, recording requests.
|
||||
|
||||
@@ -687,7 +677,7 @@ class _FakeHTTPSHandler(urllib.request.HTTPSHandler):
|
||||
"""
|
||||
|
||||
def __init__(self, redirect_to):
|
||||
super().__init__(context=_unused_tls_context())
|
||||
super().__init__()
|
||||
self.redirect_to = redirect_to
|
||||
self.requests = []
|
||||
|
||||
@@ -755,7 +745,7 @@ class _ChainHTTPSHandler(urllib.request.HTTPSHandler):
|
||||
urllib opener, recording every request; an unknown URL answers 404."""
|
||||
|
||||
def __init__(self, routes):
|
||||
super().__init__(context=_unused_tls_context())
|
||||
super().__init__()
|
||||
self.routes = routes
|
||||
self.requests = []
|
||||
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
"""The shared folder skip every media walker uses (#501)."""
|
||||
|
||||
import unittest
|
||||
|
||||
from utils.media_folders import in_skipped_folder, is_skipped_folder, is_trash_folder
|
||||
|
||||
|
||||
class TrashFolderNameTests(unittest.TestCase):
|
||||
def test_trash_folder_names_in_any_case(self):
|
||||
for name in (
|
||||
".Trash",
|
||||
".trash",
|
||||
".Trash-1000",
|
||||
".trash-0",
|
||||
".Trashes",
|
||||
"$RECYCLE.BIN",
|
||||
"$Recycle.Bin",
|
||||
):
|
||||
with self.subTest(name=name):
|
||||
self.assertTrue(is_trash_folder(name))
|
||||
self.assertTrue(is_skipped_folder(name))
|
||||
|
||||
def test_other_names_are_not_trash(self):
|
||||
for name in (".Trash-abc", ".Trash-", "trash", "Recycle", "RECYCLE.BIN", "x"):
|
||||
with self.subTest(name=name):
|
||||
self.assertFalse(is_trash_folder(name))
|
||||
|
||||
def test_thumbnails_is_skipped_exactly(self):
|
||||
self.assertTrue(is_skipped_folder("thumbnails"))
|
||||
self.assertFalse(is_skipped_folder("Thumbnails"))
|
||||
self.assertFalse(is_trash_folder("thumbnails"))
|
||||
|
||||
def test_folder_parts(self):
|
||||
self.assertTrue(in_skipped_folder(("a", ".Trash-1000", "files")))
|
||||
self.assertTrue(in_skipped_folder(("thumbnails",)))
|
||||
self.assertFalse(in_skipped_folder(("a", "b")))
|
||||
self.assertFalse(in_skipped_folder(()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,196 +0,0 @@
|
||||
"""#443: the Library parent archive table, its migration and its validation."""
|
||||
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from database.models import PromptModel
|
||||
from database.operations import PromptDatabase
|
||||
from database.prompt_archive import (
|
||||
TABLE,
|
||||
migrate_prompt_archive,
|
||||
prompt_archive_schema_error,
|
||||
)
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
|
||||
def _tables(conn):
|
||||
return {
|
||||
row[0]
|
||||
for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||
}
|
||||
|
||||
|
||||
class TestPromptArchiveSchema(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.path = str(Path(self.temp.name) / "archive.db")
|
||||
self.db = PromptDatabase(self.path)
|
||||
|
||||
def tearDown(self):
|
||||
self.db.close_all()
|
||||
self.temp.cleanup()
|
||||
|
||||
def conn(self):
|
||||
return self.db.model.get_connection()
|
||||
|
||||
def save(self, text):
|
||||
return self.db.save_prompt(text, prompt_hash=generate_prompt_hash(text))
|
||||
|
||||
def test_startup_creates_the_table_and_it_validates(self):
|
||||
self.assertIn(TABLE, _tables(self.conn()))
|
||||
self.assertIsNone(prompt_archive_schema_error(self.conn(), allow_missing=False))
|
||||
|
||||
def test_migration_is_idempotent(self):
|
||||
conn = self.conn()
|
||||
migrate_prompt_archive(conn)
|
||||
migrate_prompt_archive(conn)
|
||||
self.assertIsNone(prompt_archive_schema_error(conn, allow_missing=False))
|
||||
|
||||
def test_a_missing_table_is_accepted_only_when_allowed(self):
|
||||
conn = sqlite3.connect(":memory:")
|
||||
self.addCleanup(conn.close)
|
||||
self.assertIsNone(prompt_archive_schema_error(conn))
|
||||
self.assertEqual(
|
||||
prompt_archive_schema_error(conn, allow_missing=False),
|
||||
"Missing prompt archive schema",
|
||||
)
|
||||
|
||||
def test_deleting_a_prompt_drops_its_archive_row(self):
|
||||
pid = self.save("Archived then deleted")
|
||||
conn = self.conn()
|
||||
conn.execute(
|
||||
f"INSERT INTO {TABLE}(prompt_id, archived_at) VALUES (?, ?)",
|
||||
(pid, "2026-10-28T00:00:00+00:00"),
|
||||
)
|
||||
conn.commit()
|
||||
self.db.delete_prompt(pid)
|
||||
self.assertEqual(conn.execute(f"SELECT COUNT(*) FROM {TABLE}").fetchone()[0], 0)
|
||||
|
||||
def _broken(self, *statements):
|
||||
conn = sqlite3.connect(":memory:")
|
||||
self.addCleanup(conn.close)
|
||||
conn.execute("CREATE TABLE prompts (id INTEGER PRIMARY KEY)")
|
||||
for statement in statements:
|
||||
conn.execute(statement)
|
||||
return conn
|
||||
|
||||
def test_unsupported_structures_are_refused(self):
|
||||
good = (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
|
||||
"archived_at TEXT NOT NULL, "
|
||||
"FOREIGN KEY(prompt_id) REFERENCES prompts(id) ON DELETE CASCADE)"
|
||||
)
|
||||
cases = {
|
||||
"other definition": (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
|
||||
"archived_at TEXT)",
|
||||
),
|
||||
"no cascade": (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
|
||||
"archived_at TEXT NOT NULL, "
|
||||
"FOREIGN KEY(prompt_id) REFERENCES prompts(id))",
|
||||
),
|
||||
"extra index": (good, f"CREATE INDEX x ON {TABLE}(archived_at)"),
|
||||
"trigger": (
|
||||
good,
|
||||
f"CREATE TRIGGER t AFTER INSERT ON {TABLE} BEGIN SELECT 1; END",
|
||||
),
|
||||
"view name": (f"CREATE VIEW {TABLE} AS SELECT 1",),
|
||||
}
|
||||
for name, statements in cases.items():
|
||||
with self.subTest(name):
|
||||
self.assertIsNotNone(
|
||||
prompt_archive_schema_error(self._broken(*statements))
|
||||
)
|
||||
|
||||
def test_invalid_values_and_dangling_rows_are_refused(self):
|
||||
good = (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
|
||||
"archived_at TEXT NOT NULL, "
|
||||
"FOREIGN KEY(prompt_id) REFERENCES prompts(id) ON DELETE CASCADE)"
|
||||
)
|
||||
for name, row in {
|
||||
"number": "(1, 5)",
|
||||
"blob": "(1, x'00')",
|
||||
"long": f"(1, '{'x' * 65}')",
|
||||
"dangling": "(99, '2026-10-28T00:00:00+00:00')",
|
||||
}.items():
|
||||
with self.subTest(name):
|
||||
conn = self._broken(good, "INSERT INTO prompts(id) VALUES (1)")
|
||||
conn.execute(f"INSERT INTO {TABLE} VALUES {row}")
|
||||
self.assertIsNotNone(prompt_archive_schema_error(conn))
|
||||
|
||||
def test_a_refused_structure_stops_the_migration_without_changes(self):
|
||||
conn = self._broken(
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, archived_at TEXT)"
|
||||
)
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
migrate_prompt_archive(conn)
|
||||
sql = conn.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE name=?", (TABLE,)
|
||||
).fetchone()[0]
|
||||
self.assertNotIn("NOT NULL", sql)
|
||||
|
||||
|
||||
class TestPromptArchiveBackups(unittest.TestCase):
|
||||
"""Restore validation accepts an older backup and refuses a damaged table."""
|
||||
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temp.name)
|
||||
self.db = PromptDatabase(str(self.root / "live.db"))
|
||||
|
||||
def tearDown(self):
|
||||
self.db.close_all()
|
||||
self.temp.cleanup()
|
||||
|
||||
def copy(self, name):
|
||||
target = self.root / name
|
||||
self.db.model.backup_database(str(target))
|
||||
return str(target)
|
||||
|
||||
def test_a_backup_without_the_table_verifies_and_migrates(self):
|
||||
path = self.copy("old.db")
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute(f"DROP TABLE {TABLE}")
|
||||
conn.close()
|
||||
ok, reason = self.db.model.verify_database_file(path)
|
||||
self.assertTrue(ok, reason)
|
||||
model = PromptModel(path)
|
||||
try:
|
||||
self.assertIn(TABLE, _tables(model.get_connection()))
|
||||
finally:
|
||||
model.close_all()
|
||||
|
||||
def test_a_backup_with_a_damaged_table_is_refused(self):
|
||||
path = self.copy("bad.db")
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute(f"DROP TABLE {TABLE}")
|
||||
conn.execute(f"CREATE TABLE {TABLE} (prompt_id INTEGER, archived_at TEXT)")
|
||||
conn.close()
|
||||
ok, reason = self.db.model.verify_database_file(path)
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("archive", reason.lower())
|
||||
|
||||
def test_an_archived_prompt_survives_a_backup_copy(self):
|
||||
pid = self.db.save_prompt("Kept", prompt_hash=generate_prompt_hash("Kept"))
|
||||
conn = self.db.model.get_connection()
|
||||
conn.execute(
|
||||
f"INSERT INTO {TABLE}(prompt_id, archived_at) VALUES (?, ?)",
|
||||
(pid, "2026-10-28T00:00:00+00:00"),
|
||||
)
|
||||
conn.commit()
|
||||
path = self.copy("copy.db")
|
||||
shutil.copy(path, path + ".2")
|
||||
with sqlite3.connect(path + ".2") as other:
|
||||
self.assertEqual(
|
||||
other.execute(f"SELECT prompt_id FROM {TABLE}").fetchall(), [(pid,)]
|
||||
)
|
||||
other.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,951 +0,0 @@
|
||||
"""tools/repair_foreign_links.py: the reviewed repair of the #247 dataset links.
|
||||
|
||||
The fixture is a library whose foreign (no PromptManager node) dataset run
|
||||
wrote shared-prefix files that the old scan linked to the wrong branch text.
|
||||
Recorded history names each file's saver, so the ladder (``recorded-run``)
|
||||
names the right branch.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
import unittest.mock
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.models import PromptModel
|
||||
from database.operations import PromptDatabase
|
||||
from tests import saver_anchor_graphs as shapes
|
||||
from tests.test_run_attribution import RecordedRuns
|
||||
from tools import repair_foreign_links as tool
|
||||
from utils.file_prompt import Decision
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
TEXTS = shapes.BRANCH_TEXTS
|
||||
NAME = "IMG_GEN_{:05}_.png"
|
||||
|
||||
|
||||
def png(path, graph):
|
||||
info = PngInfo()
|
||||
info.add_text("prompt", json.dumps(graph))
|
||||
Image.new("RGB", (4, 4)).save(path, pnginfo=info)
|
||||
|
||||
|
||||
def sha(text):
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def file_sha(path):
|
||||
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def dump(path):
|
||||
"""The database's logical content (independent of WAL checkpoints)."""
|
||||
conn = sqlite3.connect(path)
|
||||
try:
|
||||
return list(conn.iterdump())
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def not_running(_comfyui_dir, _port):
|
||||
return []
|
||||
|
||||
|
||||
def no_handles(_database):
|
||||
return []
|
||||
|
||||
|
||||
class RepairFixture(RecordedRuns, unittest.TestCase):
|
||||
"""Rows: A relink to a new prompt, B relink to a stored prompt (with a
|
||||
pair link), C keep, D file gone, E two savers (skip), F the UNIQUE case
|
||||
(a wrong row beside a correct row), H relink from a rated prompt, G a
|
||||
PromptManager run (not a candidate)."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.tmp, True)
|
||||
self.addCleanup(assert_closed, self, self.tmp)
|
||||
self.addCleanup(PromptModel.reset_schema_cache)
|
||||
self.output = os.path.join(self.tmp, "output")
|
||||
os.makedirs(os.path.join(self.output, "dataset"))
|
||||
self.db_path = os.path.join(self.tmp, "prompts.db")
|
||||
self.plan_path = os.path.join(self.tmp, "plan.json")
|
||||
self.backups = os.path.join(self.tmp, "backups")
|
||||
os.makedirs(self.backups)
|
||||
self.comfy = os.path.join(self.tmp, "comfy")
|
||||
os.makedirs(self.comfy)
|
||||
Path(self.comfy, "main.py").write_text("# ComfyUI\n", encoding="utf-8")
|
||||
self.files = {}
|
||||
for key, number in zip("ABCDEFHG", range(1, 9)):
|
||||
path = os.path.join(self.output, "dataset", NAME.format(number))
|
||||
png(path, shapes.shared_prefix())
|
||||
self.files[key] = path
|
||||
db = PromptDatabase(self.db_path)
|
||||
try:
|
||||
self.build(db)
|
||||
finally:
|
||||
db.close_all()
|
||||
os.remove(self.files["D"])
|
||||
|
||||
def build(self, db):
|
||||
save = lambda text, **kw: db.save_prompt( # noqa: E731
|
||||
text=text, prompt_hash=generate_prompt_hash(text), **kw
|
||||
)
|
||||
self.p0 = save(TEXTS[0], category="scanned", tags=["auto-scanned"])
|
||||
self.p2 = save(TEXTS[2], category="scanned", tags=["auto-scanned"])
|
||||
self.orphan = save(
|
||||
"an old guess",
|
||||
category="scanned",
|
||||
tags=["auto-scanned"],
|
||||
notes="Auto-scanned from IMG_GEN_00001_.png",
|
||||
)
|
||||
self.rated = save("a rated guess", rating=5)
|
||||
link = lambda prompt, key: db.link_image_to_prompt( # noqa: E731
|
||||
prompt, self.files[key], metadata={}
|
||||
)
|
||||
self.rows = {
|
||||
"A": link(self.orphan, "A"),
|
||||
"B": link(self.p0, "B"),
|
||||
"C": link(self.p0, "C"),
|
||||
"D": link(self.p0, "D"),
|
||||
"E": link(self.p0, "E"),
|
||||
"F": link(self.p0, "F"),
|
||||
"F_ok": link(self.p2, "F"),
|
||||
"H": link(self.rated, "H"),
|
||||
"G": link(self.p0, "G"),
|
||||
}
|
||||
conn = db.model.get_connection()
|
||||
with conn:
|
||||
pair = conn.execute(
|
||||
"INSERT INTO prompt_pairs(positive_prompt_id) VALUES(?)", (self.p0,)
|
||||
).lastrowid
|
||||
conn.execute(
|
||||
"INSERT INTO image_prompt_pairs(image_id, pair_id, provenance) "
|
||||
"VALUES(?,?,'{}')",
|
||||
(self.rows["B"], pair),
|
||||
)
|
||||
name = lambda key: ("dataset", os.path.basename(self.files[key])) # noqa
|
||||
# Savers of the shared-prefix graph: "13", "23", "33", "43" (branches 1-4)
|
||||
self.record_run(
|
||||
db,
|
||||
self.output,
|
||||
[name(k) for k in "CDEAEBFH"],
|
||||
[],
|
||||
nodes=["13", "13", "13", "23", "23", "33", "33", "43"],
|
||||
)
|
||||
self.record_run(db, self.output, [name("G")], [TEXTS[0]], item=1, nodes=["23"])
|
||||
|
||||
def dry_run(self, *args):
|
||||
out = io.StringIO()
|
||||
code = tool.main(
|
||||
[
|
||||
self.db_path,
|
||||
"--output-dir",
|
||||
self.output,
|
||||
"--plan",
|
||||
self.plan_path,
|
||||
*args,
|
||||
],
|
||||
out=out,
|
||||
)
|
||||
return code, out.getvalue()
|
||||
|
||||
def apply(self, *args, probe=not_running, handles=no_handles):
|
||||
out = io.StringIO()
|
||||
code = tool.main(
|
||||
[
|
||||
self.db_path,
|
||||
"--apply",
|
||||
self.plan_path,
|
||||
"--comfyui-dir",
|
||||
os.path.join(self.tmp, "comfy"),
|
||||
"--backup-dir",
|
||||
self.backups,
|
||||
*args,
|
||||
],
|
||||
out=out,
|
||||
probe=probe,
|
||||
handles=handles,
|
||||
)
|
||||
return code, out.getvalue()
|
||||
|
||||
def plan(self):
|
||||
with open(self.plan_path, encoding="utf-8") as stream:
|
||||
return json.load(stream)
|
||||
|
||||
def write_plan(self, plan):
|
||||
with open(self.plan_path, "w", encoding="utf-8") as stream:
|
||||
json.dump(plan, stream)
|
||||
|
||||
def query(self, sql, args=()):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
return conn.execute(sql, args).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def query_write(self, sql, args):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
with conn:
|
||||
conn.execute(sql, args)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def links(self):
|
||||
return dict(self.query("SELECT id, prompt_id FROM generated_images"))
|
||||
|
||||
|
||||
class DryRunTests(RepairFixture):
|
||||
def test_plan_names_relink_keep_and_skip_rows(self):
|
||||
code, out = self.dry_run()
|
||||
self.assertEqual(code, 0, out)
|
||||
items = {item["row_id"]: item for item in self.plan()["rows"]}
|
||||
rows = self.rows
|
||||
self.assertNotIn(rows["G"], items, "a PromptManager run is not foreign")
|
||||
actions = {key: items[rows[key]]["action"] for key in rows if key != "G"}
|
||||
self.assertEqual(
|
||||
actions,
|
||||
{
|
||||
"A": "relink",
|
||||
"B": "relink",
|
||||
"C": "keep",
|
||||
"D": "skip",
|
||||
"E": "skip",
|
||||
"F": "relink",
|
||||
"F_ok": "keep",
|
||||
"H": "relink",
|
||||
},
|
||||
)
|
||||
a = items[rows["A"]]
|
||||
self.assertEqual(a["current_prompt_id"], self.orphan)
|
||||
self.assertEqual(a["current_text_sha256"], sha("an old guess"))
|
||||
self.assertEqual(a["target_text"], TEXTS[1])
|
||||
self.assertEqual(a["target_hash"], generate_prompt_hash(TEXTS[1]))
|
||||
self.assertEqual(a["target_prompt_id"], "new")
|
||||
self.assertEqual(a["file"], self.files["A"])
|
||||
self.assertEqual(items[rows["B"]]["target_prompt_id"], self.p2)
|
||||
self.assertEqual(items[rows["H"]]["target_prompt_id"], "new")
|
||||
self.assertEqual(items[rows["D"]]["reason"], "file gone")
|
||||
self.assertEqual(
|
||||
items[rows["E"]]["reason"], "not decided by the recorded saver"
|
||||
)
|
||||
self.assertIn("relink: 4", out)
|
||||
self.assertIn("keep: 2", out)
|
||||
self.assertIn("skip: 2", out)
|
||||
|
||||
def test_orphan_audit_lists_the_prompts_losing_every_link(self):
|
||||
self.dry_run()
|
||||
plan = self.plan()
|
||||
orphans = {o["id"]: o for o in plan["orphans"]}
|
||||
self.assertEqual(set(orphans), {self.orphan, self.rated})
|
||||
self.assertEqual(orphans[self.orphan]["blockers"], [])
|
||||
self.assertTrue(orphans[self.orphan]["safe"])
|
||||
self.assertEqual(orphans[self.rated]["blockers"], ["usage or rating"])
|
||||
self.assertFalse(orphans[self.rated]["safe"])
|
||||
self.assertFalse(plan["delete_orphans"], "off by default")
|
||||
self.dry_run("--delete-orphans")
|
||||
self.assertTrue(self.plan()["delete_orphans"])
|
||||
|
||||
def test_dry_run_writes_nothing_and_writes_the_markdown_list(self):
|
||||
before = file_sha(self.db_path)
|
||||
listing = sorted(os.listdir(self.tmp))
|
||||
code, out = self.dry_run()
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(file_sha(self.db_path), before)
|
||||
self.assertIn("Dry run: nothing was changed.", out)
|
||||
# A read-only reader of a WAL library may leave its empty side files
|
||||
sidecars = {"prompts.db-shm", "prompts.db-wal"}
|
||||
self.assertEqual(
|
||||
sorted(set(os.listdir(self.tmp)) - sidecars),
|
||||
sorted(set(listing + ["plan.json", "plan.list.md"]) - sidecars),
|
||||
)
|
||||
wal = Path(self.tmp, "prompts.db-wal")
|
||||
self.assertEqual(wal.stat().st_size if wal.exists() else 0, 0)
|
||||
markdown = Path(self.tmp, "plan.list.md").read_text(encoding="utf-8")
|
||||
self.assertIn(f"| {self.rows['A']} |", markdown)
|
||||
long_path = os.path.join("dataset", NAME.format(1))
|
||||
self.assertIn(f"| {long_path} |", markdown, "file paths are never cut")
|
||||
self.assertIn(generate_prompt_hash(TEXTS[1])[:12], markdown)
|
||||
self.assertIn("new", markdown)
|
||||
self.assertIn("would lose every link", markdown)
|
||||
|
||||
|
||||
class DryRunReviewTests(RepairFixture):
|
||||
def test_plan_records_its_library_and_fingerprint(self):
|
||||
self.dry_run()
|
||||
plan = self.plan()
|
||||
self.assertEqual(
|
||||
plan["source_db"], os.path.normcase(os.path.realpath(self.db_path))
|
||||
)
|
||||
self.assertEqual(plan["max_row_id"], max(self.links()))
|
||||
|
||||
def test_a_markdown_plan_path_is_refused(self):
|
||||
out = io.StringIO()
|
||||
with (
|
||||
self.assertRaises(SystemExit),
|
||||
contextlib.redirect_stderr(io.StringIO()),
|
||||
):
|
||||
tool.main(
|
||||
[
|
||||
self.db_path,
|
||||
"--output-dir",
|
||||
self.output,
|
||||
"--plan",
|
||||
os.path.join(self.tmp, "plan.md"),
|
||||
],
|
||||
out=out,
|
||||
)
|
||||
self.assertFalse(os.path.exists(os.path.join(self.tmp, "plan.md")))
|
||||
|
||||
def test_keep_compares_the_linked_text_not_the_stored_hash(self):
|
||||
self.query_write("UPDATE prompts SET hash='stale' WHERE id=?", (self.p0,))
|
||||
self.dry_run()
|
||||
items = {item["row_id"]: item for item in self.plan()["rows"]}
|
||||
self.assertEqual(items[self.rows["C"]]["action"], "keep")
|
||||
|
||||
def test_delete_orphans_is_rehearsed_with_the_deletes(self):
|
||||
code, out = self.dry_run("--delete-orphans")
|
||||
self.assertEqual(code, 0, out)
|
||||
simulated = self.plan()["simulated"]
|
||||
self.assertEqual(simulated["result"]["orphans_deleted"], 1)
|
||||
self.assertEqual(
|
||||
simulated["after"]["prompts"], simulated["before"]["prompts"] + 2 - 1
|
||||
)
|
||||
|
||||
def test_a_file_on_another_drive_is_listed_by_its_full_path(self):
|
||||
self.dry_run()
|
||||
plan = self.plan()
|
||||
with unittest.mock.patch.object(
|
||||
tool.os.path, "relpath", side_effect=ValueError("other drive")
|
||||
):
|
||||
listing = tool.markdown(plan, {})
|
||||
self.assertIn(self.files["A"], listing)
|
||||
|
||||
|
||||
class ApplyTests(RepairFixture):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
code, out = self.dry_run()
|
||||
self.assertEqual(code, 0, out)
|
||||
|
||||
def test_apply_relinks_creates_targets_and_drops_pair_links(self):
|
||||
code, out = self.apply()
|
||||
self.assertEqual(code, 0, out)
|
||||
links = self.links()
|
||||
new = dict(self.query("SELECT hash, id FROM prompts"))
|
||||
target_a = new[generate_prompt_hash(TEXTS[1])]
|
||||
target_h = new[generate_prompt_hash(TEXTS[3])]
|
||||
self.assertEqual(links[self.rows["A"]], target_a)
|
||||
self.assertEqual(links[self.rows["B"]], self.p2)
|
||||
self.assertEqual(links[self.rows["H"]], target_h)
|
||||
self.assertEqual(links[self.rows["C"]], self.p0, "keep")
|
||||
self.assertEqual(links[self.rows["E"]], self.p0, "skip")
|
||||
self.assertEqual(links[self.rows["G"]], self.p0, "not foreign")
|
||||
self.assertEqual(
|
||||
self.query(
|
||||
"SELECT COUNT(*) FROM image_prompt_pairs WHERE image_id=?",
|
||||
(self.rows["B"],),
|
||||
),
|
||||
[(0,)],
|
||||
)
|
||||
category, notes = self.query(
|
||||
"SELECT category, notes FROM prompts WHERE id=?", (target_a,)
|
||||
)[0]
|
||||
self.assertEqual(category, "scanned")
|
||||
self.assertEqual(
|
||||
notes, f"Auto-scanned from {os.path.basename(self.files['A'])}"
|
||||
)
|
||||
tags = self.query(
|
||||
"SELECT t.name FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.prompt_id=?",
|
||||
(target_a,),
|
||||
)
|
||||
self.assertEqual(tags, [("auto-scanned",)])
|
||||
self.assertIn("integrity_check", out)
|
||||
self.assertIn("foreign_key_check", out)
|
||||
self.assertRegex(out, r"integrity_check\s+ok\s+ok")
|
||||
# Orphans are kept unless the plan chose to delete them
|
||||
self.assertTrue(self.query("SELECT 1 FROM prompts WHERE id=?", (self.orphan,)))
|
||||
|
||||
def test_unique_case_deletes_the_wrong_row(self):
|
||||
code, out = self.apply()
|
||||
self.assertEqual(code, 0, out)
|
||||
links = self.links()
|
||||
self.assertNotIn(self.rows["F"], links)
|
||||
self.assertEqual(links[self.rows["F_ok"]], self.p2)
|
||||
rows = self.query(
|
||||
"SELECT prompt_id FROM generated_images WHERE image_path=?",
|
||||
(self.files["F"],),
|
||||
)
|
||||
self.assertEqual(rows, [(self.p2,)])
|
||||
|
||||
def test_backup_is_taken_before_writing(self):
|
||||
before = self.links()
|
||||
code, out = self.apply()
|
||||
self.assertEqual(code, 0, out)
|
||||
names = os.listdir(self.backups)
|
||||
self.assertEqual(len(names), 1)
|
||||
self.assertRegex(names[0], r"^prompts\.db\.pre-247-repair_\d{8}_\d{6}$")
|
||||
backup = os.path.join(self.backups, names[0])
|
||||
conn = sqlite3.connect(backup)
|
||||
try:
|
||||
saved = dict(conn.execute("SELECT id, prompt_id FROM generated_images"))
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(saved, before)
|
||||
self.assertIn(f"Backup: {backup}", out)
|
||||
|
||||
def test_backup_beside_the_database_by_default(self):
|
||||
out = io.StringIO()
|
||||
code = tool.main(
|
||||
[
|
||||
self.db_path,
|
||||
"--apply",
|
||||
self.plan_path,
|
||||
"--comfyui-dir",
|
||||
os.path.join(self.tmp, "comfy"),
|
||||
],
|
||||
out=out,
|
||||
probe=not_running,
|
||||
handles=no_handles,
|
||||
)
|
||||
self.assertEqual(code, 0, out.getvalue())
|
||||
self.assertTrue(
|
||||
any(
|
||||
n.startswith("prompts.db.pre-247-repair_") for n in os.listdir(self.tmp)
|
||||
)
|
||||
)
|
||||
|
||||
def test_refuses_while_comfyui_runs(self):
|
||||
before = file_sha(self.db_path)
|
||||
code, out = self.apply(probe=lambda _d, _p: ["process 42 (main.py)"])
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("REFUSED", out)
|
||||
self.assertIn("process 42", out)
|
||||
self.assertEqual(os.listdir(self.backups), [], "no backup either")
|
||||
self.assertEqual(file_sha(self.db_path), before)
|
||||
|
||||
def test_rehearsal_skips_the_running_check_on_a_copy_only(self):
|
||||
code, out = self.apply("--rehearsal", probe=lambda _d, _p: ["port 8188"])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("rehearsal", out)
|
||||
# The live library of that ComfyUI folder: --rehearsal is refused
|
||||
live_dir = os.path.join(self.tmp, "comfy", "user", "__promptmanager")
|
||||
os.makedirs(live_dir)
|
||||
live = os.path.join(live_dir, "prompts.db")
|
||||
shutil.copy(self.db_path, live)
|
||||
out = io.StringIO()
|
||||
code = tool.main(
|
||||
[
|
||||
live,
|
||||
"--apply",
|
||||
self.plan_path,
|
||||
"--comfyui-dir",
|
||||
os.path.join(self.tmp, "comfy"),
|
||||
"--rehearsal",
|
||||
],
|
||||
out=out,
|
||||
probe=lambda _d, _p: [],
|
||||
handles=no_handles,
|
||||
)
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("--rehearsal", out.getvalue())
|
||||
|
||||
def test_drift_refuses_and_changes_nothing(self):
|
||||
cases = {
|
||||
"row moved": lambda: self.query_write(
|
||||
"UPDATE generated_images SET prompt_id=? WHERE id=?",
|
||||
(self.p2, self.rows["H"]),
|
||||
),
|
||||
"text changed": lambda: self.query_write(
|
||||
"UPDATE prompts SET text='an older guess' WHERE id=?", (self.orphan,)
|
||||
),
|
||||
"file changed": lambda: png(self.files["B"], shapes.branch_bench()),
|
||||
"target stored since": lambda: self.query_write(
|
||||
"INSERT INTO prompts(text, hash) VALUES(?,?)",
|
||||
(TEXTS[1], generate_prompt_hash(TEXTS[1])),
|
||||
),
|
||||
}
|
||||
pristine = os.path.join(self.tmp, "pristine.db")
|
||||
shutil.copy(self.db_path, pristine)
|
||||
image = Path(self.files["B"]).read_bytes()
|
||||
for name, drift in cases.items():
|
||||
with self.subTest(name):
|
||||
shutil.copy(pristine, self.db_path)
|
||||
Path(self.files["B"]).write_bytes(image)
|
||||
shutil.rmtree(self.backups)
|
||||
os.makedirs(self.backups)
|
||||
drift()
|
||||
before = dump(self.db_path)
|
||||
code, out = self.apply()
|
||||
self.assertEqual(code, 2, out)
|
||||
self.assertIn("REFUSED (rolled back, nothing changed)", out)
|
||||
self.assertEqual(dump(self.db_path), before)
|
||||
|
||||
def test_late_failure_rolls_back_every_write(self):
|
||||
plan = self.plan()
|
||||
plan["delete_orphans"] = True
|
||||
self.write_plan(plan)
|
||||
# The safe orphan gained a user tag after the dry run
|
||||
self.query_write("INSERT INTO tags(name) VALUES('mine')", ())
|
||||
self.query_write(
|
||||
"INSERT INTO prompt_tags(prompt_id, tag_id) "
|
||||
"SELECT ?, id FROM tags WHERE name='mine'",
|
||||
(self.orphan,),
|
||||
)
|
||||
before = self.links()
|
||||
prompts = self.query("SELECT COUNT(*) FROM prompts")
|
||||
code, out = self.apply()
|
||||
self.assertEqual(code, 2, out)
|
||||
self.assertIn(f"prompt {self.orphan}", out)
|
||||
self.assertIn("user tags", out)
|
||||
self.assertEqual(self.links(), before, "the relinks were rolled back")
|
||||
self.assertEqual(self.query("SELECT COUNT(*) FROM prompts"), prompts)
|
||||
|
||||
def test_orphans_are_deleted_only_when_chosen_and_unreferenced(self):
|
||||
plan = self.plan()
|
||||
plan["delete_orphans"] = True
|
||||
self.write_plan(plan)
|
||||
code, out = self.apply()
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertFalse(self.query("SELECT 1 FROM prompts WHERE id=?", (self.orphan,)))
|
||||
self.assertTrue(
|
||||
self.query("SELECT 1 FROM prompts WHERE id=?", (self.rated,)),
|
||||
"a prompt with a rating is never deleted",
|
||||
)
|
||||
self.assertIn("orphans_deleted", out)
|
||||
|
||||
def test_an_applied_plan_refuses_a_second_time(self):
|
||||
self.assertEqual(self.apply()[0], 0)
|
||||
before = dump(self.db_path)
|
||||
code, out = self.apply()
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("REFUSED", out)
|
||||
self.assertEqual(dump(self.db_path), before)
|
||||
|
||||
|
||||
class ApplySafetyTests(RepairFixture):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
code, out = self.dry_run()
|
||||
self.assertEqual(code, 0, out)
|
||||
|
||||
def assert_unchanged(self, before, code, out):
|
||||
self.assertEqual(code, 2, out)
|
||||
self.assertEqual(dump(self.db_path), before)
|
||||
|
||||
def test_refuses_when_another_process_has_the_library_open(self):
|
||||
before = dump(self.db_path)
|
||||
held = lambda _db: ["process 7 has prompts.db-wal open"] # noqa: E731
|
||||
for args in ((), ("--rehearsal",)):
|
||||
with self.subTest(args):
|
||||
code, out = self.apply(*args, handles=held)
|
||||
self.assert_unchanged(before, code, out)
|
||||
self.assertIn("process 7", out)
|
||||
self.assertEqual(os.listdir(self.backups), [])
|
||||
|
||||
def test_the_running_check_runs_again_just_before_writing(self):
|
||||
calls = []
|
||||
|
||||
def late(_d, _p):
|
||||
calls.append(1)
|
||||
return [] if len(calls) == 1 else ["something listens on port 8188"]
|
||||
|
||||
before = dump(self.db_path)
|
||||
code, out = self.apply(probe=late)
|
||||
self.assert_unchanged(before, code, out)
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertIn("rolled back", out)
|
||||
self.assertIn("Backup:", out)
|
||||
|
||||
def test_without_proc_only_an_explicit_stop_lets_it_through(self):
|
||||
blind = lambda _db: [tool.NO_PROC] # noqa: E731
|
||||
before = dump(self.db_path)
|
||||
code, out = self.apply(handles=blind)
|
||||
self.assert_unchanged(before, code, out)
|
||||
self.assertIn("--i-stopped-comfyui", out)
|
||||
code, out = self.apply("--i-stopped-comfyui", handles=blind)
|
||||
self.assertEqual(code, 0, out)
|
||||
|
||||
def test_a_folder_without_main_py_is_refused(self):
|
||||
os.remove(os.path.join(self.comfy, "main.py"))
|
||||
before = dump(self.db_path)
|
||||
code, out = self.apply()
|
||||
self.assert_unchanged(before, code, out)
|
||||
self.assertIn("main.py", out)
|
||||
|
||||
def test_a_plan_for_another_library_is_refused(self):
|
||||
other = os.path.join(self.tmp, "other.db")
|
||||
shutil.copy(self.db_path, other)
|
||||
out = io.StringIO()
|
||||
code = tool.main(
|
||||
[other, "--apply", self.plan_path, "--comfyui-dir", self.comfy],
|
||||
out=out,
|
||||
probe=not_running,
|
||||
handles=no_handles,
|
||||
)
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("made for", out.getvalue())
|
||||
|
||||
def test_opening_the_library_must_not_change_its_schema(self):
|
||||
self.query_write("DROP INDEX idx_prompt_pairs_positive_only", ())
|
||||
PromptModel.reset_schema_cache()
|
||||
code, out = self.apply()
|
||||
self.assertEqual(code, 2, out)
|
||||
self.assertIn("schema", out)
|
||||
self.assertIn("live code version", out)
|
||||
|
||||
def test_an_existing_backup_file_is_refused(self):
|
||||
stamp = "20261028_000000"
|
||||
Path(self.backups, f"prompts.db.pre-247-repair_{stamp}").write_bytes(b"x")
|
||||
before = dump(self.db_path)
|
||||
with unittest.mock.patch.object(tool, "_stamp", return_value=stamp):
|
||||
code, out = self.apply()
|
||||
self.assert_unchanged(before, code, out)
|
||||
self.assertIn("already exists", out)
|
||||
|
||||
def test_a_backup_that_does_not_match_is_refused(self):
|
||||
real = tool.backup_database
|
||||
|
||||
def short(source, target):
|
||||
real(source, target)
|
||||
conn = sqlite3.connect(target)
|
||||
try:
|
||||
with conn:
|
||||
conn.execute("DELETE FROM image_prompt_pairs")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
before = dump(self.db_path)
|
||||
with unittest.mock.patch.object(tool, "backup_database", short):
|
||||
code, out = self.apply()
|
||||
self.assert_unchanged(before, code, out)
|
||||
self.assertIn("backup", out)
|
||||
|
||||
def test_a_changed_target_text_is_refused(self):
|
||||
plan = self.plan()
|
||||
for item in plan["rows"]:
|
||||
if item["row_id"] == self.rows["A"]:
|
||||
item["target_text"] = "something else"
|
||||
self.write_plan(plan)
|
||||
before = dump(self.db_path)
|
||||
code, out = self.apply()
|
||||
self.assert_unchanged(before, code, out)
|
||||
self.assertIn("target_text", out)
|
||||
|
||||
def test_an_orphan_gone_or_changed_at_apply_is_refused(self):
|
||||
plan = self.plan()
|
||||
plan["delete_orphans"] = True
|
||||
plan["orphans"][0]["text_sha256"] = "0" * 64
|
||||
self.write_plan(plan)
|
||||
before = dump(self.db_path)
|
||||
code, out = self.apply()
|
||||
self.assert_unchanged(before, code, out)
|
||||
self.assertIn("gone or text changed", out)
|
||||
|
||||
def test_any_error_rolls_back_and_names_the_backup(self):
|
||||
before = dump(self.db_path)
|
||||
with unittest.mock.patch.object(
|
||||
tool, "_relink", side_effect=RuntimeError("disk on fire")
|
||||
):
|
||||
code, out = self.apply()
|
||||
self.assert_unchanged(before, code, out)
|
||||
self.assertIn("rolled back", out)
|
||||
self.assertIn("disk on fire", out)
|
||||
self.assertIn(self.backups, out)
|
||||
|
||||
def test_a_failed_integrity_check_after_writing_exits_1(self):
|
||||
real = tool.counts
|
||||
seen = []
|
||||
|
||||
def counts(path, readonly=True):
|
||||
found = real(path, readonly)
|
||||
if path == self.db_path:
|
||||
seen.append(path)
|
||||
if len(seen) > 1:
|
||||
return {**found, "integrity_check": "corrupt"}
|
||||
return found
|
||||
|
||||
with unittest.mock.patch.object(tool, "counts", counts):
|
||||
code, out = self.apply()
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("restore", out)
|
||||
|
||||
|
||||
class BlockerTests(RepairFixture):
|
||||
"""Each reference that keeps a prompt from being deleted."""
|
||||
|
||||
def test_every_blocker(self):
|
||||
pid = self.orphan
|
||||
cases = {
|
||||
"linked images": None,
|
||||
"saved pairs": "INSERT INTO prompt_pairs(positive_prompt_id) VALUES(:id)",
|
||||
"recorded runs": "UPDATE execution_run_contexts "
|
||||
"SET positive_prompt_id=:id",
|
||||
"library state": "INSERT INTO prompt_library_state(prompt_id) "
|
||||
"VALUES(:id)",
|
||||
"user tags": "INSERT INTO prompt_tags(prompt_id, tag_id) "
|
||||
"SELECT :id, id FROM tags WHERE name='mine'",
|
||||
"usage or rating": "UPDATE prompts SET run_count=3 WHERE id=:id",
|
||||
"category": "UPDATE prompts SET category='portraits' WHERE id=:id",
|
||||
"notes": "UPDATE prompts SET notes='mine' WHERE id=:id",
|
||||
}
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
conn.execute("INSERT INTO tags(name) VALUES('mine')")
|
||||
conn.commit()
|
||||
for name, sql in cases.items():
|
||||
with self.subTest(name):
|
||||
conn.execute("SAVEPOINT blocker")
|
||||
if sql is None:
|
||||
unused = {pid}
|
||||
else:
|
||||
conn.execute(
|
||||
"DELETE FROM generated_images WHERE prompt_id=?", (pid,)
|
||||
)
|
||||
conn.execute(sql, {"id": pid})
|
||||
unused = {pid}
|
||||
found = tool.blockers(conn, pid, unused)
|
||||
self.assertTrue(any(item.startswith(name) for item in found), found)
|
||||
conn.execute("ROLLBACK TO blocker")
|
||||
conn.execute("RELEASE blocker")
|
||||
conn.execute("DELETE FROM generated_images WHERE prompt_id=?", (pid,))
|
||||
self.assertEqual(tool.blockers(conn, pid, {pid}), [])
|
||||
self.assertEqual(
|
||||
tool.blockers(conn, pid, set()), ["not unused by the app's rule"]
|
||||
)
|
||||
conn.rollback()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _off_text_node(text, save=False):
|
||||
return {
|
||||
"class_type": "PromptManagerText",
|
||||
"inputs": {"text": text, "save_to_library": save},
|
||||
}
|
||||
|
||||
|
||||
def _encode_from(source):
|
||||
return {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": source, "clip": ["1", 1]},
|
||||
}
|
||||
|
||||
|
||||
def off_behind_pass_through(branch=2):
|
||||
"""The dataset graph with ``branch``'s text coming from a switched-off
|
||||
PromptManagerText through a pass-through the #90 walk does not follow."""
|
||||
graph = shapes.shared_prefix()
|
||||
graph["50"] = _off_text_node(TEXTS[branch - 1])
|
||||
graph["51"] = {"class_type": "MyNode", "inputs": {"value": ["50", 0]}}
|
||||
graph[str(branch * 10)] = _encode_from(["51", 0])
|
||||
return graph
|
||||
|
||||
|
||||
def off_target(branch=2, save=False):
|
||||
"""``branch``'s text straight from a PromptManagerText whose switch is
|
||||
``save`` (False, or a link to a node the graph cannot read)."""
|
||||
graph = shapes.shared_prefix()
|
||||
graph["50"] = _off_text_node(TEXTS[branch - 1], save)
|
||||
graph[str(branch * 10)] = _encode_from(["50", 0])
|
||||
if isinstance(save, list):
|
||||
graph["52"] = {"class_type": "SomeBoolNode", "inputs": {}}
|
||||
return graph
|
||||
|
||||
|
||||
def off_negative_only():
|
||||
"""Every branch's negative from a switched-off PromptManagerText."""
|
||||
graph = shapes.shared_prefix()
|
||||
graph["50"] = _off_text_node(shapes.NEGATIVE)
|
||||
graph["3"] = _encode_from(["50", 0])
|
||||
return graph
|
||||
|
||||
|
||||
class SaveToLibraryTests(RecordedRuns, unittest.TestCase):
|
||||
"""#90: the repair never relinks a file to a switched-off node's text.
|
||||
|
||||
The plan decides each file through ``decide_file_prompt``; every answer
|
||||
but a positive from the recorded saver or the inspector is a skip, so a
|
||||
``not_saved`` or ``not_saved_wired`` file is never written. Each file is
|
||||
reported by a run without contexts (as a cached or unrecorded run is),
|
||||
so only the ladder can stop it. Saver "23" is branch 2.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.tmp, True)
|
||||
self.addCleanup(assert_closed, self, self.tmp)
|
||||
self.addCleanup(PromptModel.reset_schema_cache)
|
||||
self.output = os.path.join(self.tmp, "output")
|
||||
os.makedirs(os.path.join(self.output, "dataset"))
|
||||
self.db_path = os.path.join(self.tmp, "prompts.db")
|
||||
|
||||
def plan_for(self, graph):
|
||||
path = os.path.join(self.output, "dataset", NAME.format(1))
|
||||
png(path, graph)
|
||||
db = PromptDatabase(self.db_path)
|
||||
try:
|
||||
wrong = db.save_prompt(
|
||||
text=TEXTS[0], prompt_hash=generate_prompt_hash(TEXTS[0])
|
||||
)
|
||||
db.link_image_to_prompt(wrong, path, metadata={})
|
||||
self.record_run(
|
||||
db, self.output, [("dataset", NAME.format(1))], [], nodes=["23"]
|
||||
)
|
||||
plan = tool.build_plan(db, self.output, self.db_path)
|
||||
finally:
|
||||
db.close_all()
|
||||
(item,) = plan["rows"]
|
||||
return item
|
||||
|
||||
def test_an_off_node_behind_a_pass_through_is_skipped(self):
|
||||
item = self.plan_for(off_behind_pass_through())
|
||||
self.assertEqual(item["action"], "skip")
|
||||
self.assertEqual(item["decision"], "not_saved/none")
|
||||
self.assertNotIn("target_text", item)
|
||||
|
||||
def test_an_off_node_in_the_recorded_run_target_is_skipped(self):
|
||||
cases = {
|
||||
"off": (False, "not_saved/none"),
|
||||
"wired": (["52", 0], "not_saved_wired/none"),
|
||||
}
|
||||
for name, (save, decision) in cases.items():
|
||||
with self.subTest(name):
|
||||
if os.path.exists(self.db_path):
|
||||
os.remove(self.db_path)
|
||||
item = self.plan_for(off_target(save=save))
|
||||
self.assertEqual(item["action"], "skip")
|
||||
self.assertEqual(item["decision"], decision)
|
||||
|
||||
def test_an_off_node_on_another_branch_still_skips_the_file(self):
|
||||
"""The ladder judges the file's whole graph: any switched-off text
|
||||
upstream of a positive makes the file not saved."""
|
||||
item = self.plan_for(off_target(branch=3))
|
||||
self.assertEqual(item["action"], "skip")
|
||||
self.assertEqual(item["decision"], "not_saved/none")
|
||||
|
||||
def test_an_off_node_wired_only_to_the_negative_still_repairs(self):
|
||||
item = self.plan_for(off_negative_only())
|
||||
self.assertEqual(item["action"], "relink")
|
||||
self.assertEqual(item["source"], "recorded-run")
|
||||
self.assertEqual(item["target_text"], TEXTS[1])
|
||||
|
||||
def test_a_switched_on_node_in_the_target_still_repairs(self):
|
||||
"""Regression guard: a switch that is on is no reason to skip."""
|
||||
item = self.plan_for(off_target(save=True))
|
||||
self.assertEqual(item["action"], "relink")
|
||||
self.assertEqual(item["target_text"], TEXTS[1])
|
||||
|
||||
|
||||
class ActionTests(unittest.TestCase):
|
||||
"""Only the recorded saver relinks; a certain step naming the linked
|
||||
prompt keeps it; the legacy floor and the rest are skipped."""
|
||||
|
||||
def test_action_by_decision(self):
|
||||
linked = generate_prompt_hash("linked")
|
||||
cases = [
|
||||
(Decision("positive", "other", source="recorded-run"), "relink"),
|
||||
(Decision("positive", "linked", source="recorded-run"), "keep"),
|
||||
(Decision("positive", "linked", source="inspector"), "keep"),
|
||||
(Decision("positive", "other", source="inspector"), "skip"),
|
||||
(Decision("positive", "linked", source="legacy-parser"), "skip"),
|
||||
(Decision("pair", "linked", source="resolver"), "skip"),
|
||||
(Decision("never"), "skip"),
|
||||
(Decision("not_saved"), "skip"),
|
||||
(Decision("not_saved_wired"), "skip"),
|
||||
(Decision("positive", "other", source="opted-out-negative"), "skip"),
|
||||
]
|
||||
for decision, action in cases:
|
||||
with self.subTest(decision.source, text=decision.text):
|
||||
self.assertEqual(tool.action_of(decision, linked), action)
|
||||
|
||||
|
||||
class CommandLineTests(unittest.TestCase):
|
||||
def test_running_probe_reports_a_listening_port(self):
|
||||
import socket
|
||||
|
||||
with socket.socket() as server:
|
||||
server.bind(("127.0.0.1", 0))
|
||||
server.listen(1)
|
||||
port = server.getsockname()[1]
|
||||
found = tool.comfyui_running("/nonexistent/comfy", port)
|
||||
self.assertTrue(any(str(port) in item for item in found))
|
||||
|
||||
def test_running_probe_is_quiet_when_nothing_runs(self):
|
||||
import socket
|
||||
|
||||
with socket.socket() as probe:
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
port = probe.getsockname()[1]
|
||||
self.assertEqual(tool.comfyui_running("/nonexistent/comfy", port), [])
|
||||
|
||||
def fake_proc(self, tmp):
|
||||
proc = Path(tmp, "proc")
|
||||
db = Path(tmp, "prompts.db")
|
||||
db.write_bytes(b"")
|
||||
Path(tmp, "prompts.db-wal").write_bytes(b"")
|
||||
comfy = Path(tmp, "comfy")
|
||||
comfy.mkdir()
|
||||
holder = proc / "125" / "fd"
|
||||
holder.mkdir(parents=True)
|
||||
os.symlink(Path(tmp, "prompts.db-wal"), holder / "3")
|
||||
(proc / "125" / "cmdline").write_bytes(b"python\0main.py\0")
|
||||
os.symlink(comfy, proc / "125" / "cwd")
|
||||
(proc / "123").mkdir() # unreadable: no fd, cmdline or cwd
|
||||
(proc / "self").mkdir() # not a pid
|
||||
return proc, db, comfy
|
||||
|
||||
@unittest.skipIf(os.name == "nt", "symlinks need privileges on Windows")
|
||||
def test_proc_scan_finds_holders_and_skips_unreadable_entries(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
proc, db, comfy = self.fake_proc(tmp)
|
||||
found = tool.open_by_others(str(db), proc=str(proc), own_pid=1)
|
||||
self.assertEqual(len(found), 1)
|
||||
self.assertIn("125", found[0])
|
||||
self.assertEqual(
|
||||
tool.open_by_others(str(db), proc=str(proc), own_pid=125), []
|
||||
)
|
||||
running = tool.comfyui_running(str(comfy), 1, proc=str(proc))
|
||||
self.assertTrue(any("125" in item for item in running), running)
|
||||
|
||||
def test_without_proc_the_scan_says_it_cannot_tell(self):
|
||||
self.assertEqual(
|
||||
tool.open_by_others("/x/prompts.db", proc="/nonexistent/proc"),
|
||||
[tool.NO_PROC],
|
||||
)
|
||||
|
||||
def test_missing_database_is_an_argument_error(self):
|
||||
with self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()):
|
||||
tool.main(
|
||||
["/nonexistent/prompts.db", "--output-dir", "/tmp"], out=io.StringIO()
|
||||
)
|
||||
|
||||
def test_apply_needs_the_comfyui_folder(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db = os.path.join(tmp, "prompts.db")
|
||||
Path(db).write_bytes(b"")
|
||||
with (
|
||||
self.assertRaises(SystemExit),
|
||||
contextlib.redirect_stderr(io.StringIO()),
|
||||
):
|
||||
tool.main([db, "--apply", "plan.json"], out=io.StringIO())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -33,7 +33,6 @@ from utils.execution_outputs import (
|
||||
snapshot_output_roots,
|
||||
)
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from py.config import GalleryConfig
|
||||
from utils.image_monitor import ImageGenerationHandler
|
||||
|
||||
STYLE = "windows" if os.name == "nt" else "posix"
|
||||
@@ -772,487 +771,6 @@ class TestMonitorHistoryRecheck(RecordedRuns, unittest.TestCase):
|
||||
run_attribution.notify_outputs_sealed("run", []) # no dead callback raises
|
||||
|
||||
|
||||
def folder_is_case_sensitive(folder):
|
||||
"""True when ``folder``'s volume tells names apart by case: the existing
|
||||
folder spelled with swapped case is not found (#476)."""
|
||||
head, name = os.path.split(folder)
|
||||
return not os.path.exists(os.path.join(head, name.swapcase()))
|
||||
|
||||
|
||||
class TestMonitorForeignRuns(RecordedRuns, unittest.TestCase):
|
||||
"""#247: outputs of workflows without a PromptManager node, in the monitor."""
|
||||
|
||||
def setUp(self):
|
||||
history_tests.HistoryDatabaseTests.setUp(self)
|
||||
self.db = history_tests.HistoryDatabaseTests.database(self)
|
||||
self.output = str(Path(self.temp.name) / "output")
|
||||
os.makedirs(os.path.join(self.output, "dataset"))
|
||||
self.tracker = make_tracker(self.db)
|
||||
self.handler = ImageGenerationHandler(self.db, self.tracker)
|
||||
self.handler.processing_delay = 0
|
||||
self.handler.settle_interval = 0.01
|
||||
self.addCleanup(close_handler, self, self.handler)
|
||||
self.metadata = {"prompt": shapes.shared_prefix()}
|
||||
saved = GalleryConfig.SAVE_FOREIGN_RUNS
|
||||
self.addCleanup(setattr, GalleryConfig, "SAVE_FOREIGN_RUNS", saved)
|
||||
GalleryConfig.SAVE_FOREIGN_RUNS = True
|
||||
patches = (
|
||||
mock.patch(
|
||||
"utils.image_monitor.host_output_directory", return_value=self.output
|
||||
),
|
||||
mock.patch(
|
||||
"database.run_attribution.host_output_directory",
|
||||
return_value=self.output,
|
||||
),
|
||||
mock.patch.object(
|
||||
self.handler.metadata_extractor,
|
||||
"extract_metadata",
|
||||
side_effect=lambda path: self.metadata,
|
||||
),
|
||||
)
|
||||
for patch in patches:
|
||||
patch.start()
|
||||
self.addCleanup(patch.stop)
|
||||
|
||||
close_databases = history_tests.HistoryDatabaseTests.close_databases
|
||||
wait_for = TestMonitorHistoryRecheck.wait_for
|
||||
|
||||
def image(self, name, subfolder="dataset"):
|
||||
path = os.path.join(self.output, subfolder, name)
|
||||
Path(path).write_bytes(b"png")
|
||||
return path
|
||||
|
||||
def linked_text(self, path):
|
||||
row = (
|
||||
self.db.model.get_connection()
|
||||
.execute(
|
||||
"SELECT p.text FROM generated_images g "
|
||||
"JOIN prompts p ON p.id = g.prompt_id WHERE g.image_path = ?",
|
||||
(str(path),),
|
||||
)
|
||||
.fetchone()
|
||||
)
|
||||
return row[0] if row else None
|
||||
|
||||
def key(self, path):
|
||||
return self.handler._awaiting_key(path)
|
||||
|
||||
def prompt_count(self):
|
||||
connection = self.db.model.get_connection()
|
||||
return connection.execute("SELECT COUNT(*) FROM prompts").fetchone()[0]
|
||||
|
||||
def settled(self):
|
||||
return self.wait_for(lambda: self.handler.pending_count() == 0)
|
||||
|
||||
def test_each_dataset_file_links_to_its_branch_once_the_run_ends(self):
|
||||
"""CUJ 1: unlinked while the run is open, its own branch after the seal."""
|
||||
paths = [self.image(f"IMG_GEN_0000{n}_.png") for n in (1, 2)]
|
||||
for path in paths:
|
||||
self.handler.process_new_image(path)
|
||||
self.assertEqual([self.linked_text(path) for path in paths], [None, None])
|
||||
self.assertEqual(
|
||||
self.handler.awaiting_foreign(), [self.key(path) for path in paths]
|
||||
)
|
||||
self.assertEqual(self.handler.awaiting_history(), [])
|
||||
self.record_run(
|
||||
self.db,
|
||||
self.output,
|
||||
[("dataset", os.path.basename(path)) for path in paths],
|
||||
[],
|
||||
nodes=["13", "23"],
|
||||
)
|
||||
self.assertTrue(self.wait_for(lambda: all(map(self.linked_text, paths))))
|
||||
self.assertTrue(self.settled())
|
||||
self.assertEqual(
|
||||
[self.linked_text(path) for path in paths], list(shapes.BRANCH_TEXTS[:2])
|
||||
)
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [])
|
||||
prompt = self.db.get_prompt_by_hash(
|
||||
generate_prompt_hash(shapes.BRANCH_TEXTS[0])
|
||||
)
|
||||
self.assertEqual(prompt["category"], "scanned")
|
||||
self.assertIn("Saved from a workflow run: IMG_GEN_00001_.png", prompt["notes"])
|
||||
|
||||
def test_a_run_sealed_before_the_file_settles_links_it_at_once(self):
|
||||
path = self.image("IMG_GEN_00003_.png")
|
||||
self.record_run(
|
||||
self.db, self.output, [("dataset", "IMG_GEN_00003_.png")], [], nodes=["33"]
|
||||
)
|
||||
self.handler.process_new_image(path)
|
||||
self.assertEqual(self.linked_text(path), shapes.BRANCH_TEXTS[2])
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [])
|
||||
|
||||
def test_a_name_two_runs_reported_stays_unlinked(self):
|
||||
"""CUJ 3: the file was overwritten; no run proves its saver."""
|
||||
path = self.image("IMG_GEN_00004_.png")
|
||||
reported = [("dataset", "IMG_GEN_00004_.png")]
|
||||
self.record_run(self.db, self.output, reported, [], nodes=["13"])
|
||||
self.record_run(self.db, self.output, reported, [], item=1, nodes=["23"])
|
||||
self.handler.process_new_image(path)
|
||||
self.assertIsNone(self.linked_text(path))
|
||||
reference = {"folder_type": "output", "subfolder": "dataset"}
|
||||
reference["filename"] = "IMG_GEN_00004_.png"
|
||||
with self.assertLogs("prompt_manager.image_monitor", "INFO") as logs:
|
||||
self.handler._recheck_foreign(path, reference)
|
||||
self.assertIn(
|
||||
"No certain prompt for IMG_GEN_00004_.png after its run ended; "
|
||||
"left unlinked",
|
||||
"\n".join(logs.output),
|
||||
)
|
||||
self.assertIsNone(self.linked_text(path))
|
||||
self.assertEqual(self.prompt_count(), 0)
|
||||
|
||||
def test_the_setting_off_stores_nothing_at_settle_or_after_the_run(self):
|
||||
"""CUJ 4: read for every file, so it applies without a restart."""
|
||||
GalleryConfig.SAVE_FOREIGN_RUNS = False
|
||||
skipped = self.image("IMG_GEN_00005_.png")
|
||||
with self.assertLogs("prompt_manager.image_monitor", "INFO") as logs:
|
||||
self.handler.process_new_image(skipped)
|
||||
self.assertIn(
|
||||
"No PromptManager prompt in workflow, skipping image",
|
||||
"\n".join(logs.output),
|
||||
)
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [])
|
||||
GalleryConfig.SAVE_FOREIGN_RUNS = True
|
||||
waiting = self.image("IMG_GEN_00006_.png")
|
||||
self.handler.process_new_image(waiting)
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [self.key(waiting)])
|
||||
GalleryConfig.SAVE_FOREIGN_RUNS = False
|
||||
self.record_run(
|
||||
self.db, self.output, [("dataset", "IMG_GEN_00006_.png")], [], nodes=["13"]
|
||||
)
|
||||
self.assertTrue(self.wait_for(lambda: not self.handler.awaiting_foreign()))
|
||||
self.assertTrue(self.settled())
|
||||
self.assertEqual(
|
||||
[self.linked_text(p) for p in (skipped, waiting)], [None, None]
|
||||
)
|
||||
self.assertEqual(self.prompt_count(), 0)
|
||||
|
||||
def test_a_plain_foreign_workflow_links_at_settle_and_counts_no_run(self):
|
||||
"""CUJ 2: the default profile decides; the tracker queue is untouched."""
|
||||
text = "a lighthouse on a cliff at dusk"
|
||||
self.metadata = {"prompt": sampled_graph(text, "blurry")}
|
||||
queued = self.db.save_prompt(
|
||||
text="queued item", prompt_hash=generate_prompt_hash("queued item")
|
||||
)
|
||||
self.tracker.set_current_prompt("queued item", {"prompt_id": queued})
|
||||
first, second = self.image("plain_00001_.png"), self.image("plain_00002_.png")
|
||||
self.handler.process_new_image(first)
|
||||
self.handler.process_new_image(second)
|
||||
self.assertEqual(
|
||||
[self.linked_text(first), self.linked_text(second)], [text, text]
|
||||
)
|
||||
prompt = self.db.get_prompt_by_hash(generate_prompt_hash(text))
|
||||
self.assertEqual(prompt["run_count"] or 0, 0)
|
||||
self.assertIsNone(prompt.get("last_used_at"))
|
||||
self.assertEqual(self.tracker.pop_next_prompt()["id"], queued)
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [])
|
||||
|
||||
def test_the_sealed_reference_spells_the_file_as_history_recorded_it(self):
|
||||
"""Windows: the watcher's spelling differs in case from the run's report.
|
||||
|
||||
On a case-sensitive folder the two spellings are two files and the
|
||||
folded key joins them; on a case-insensitive one (Windows, default
|
||||
macOS) they already name one file, so only history's spelling and
|
||||
the watcher's differ (#476)."""
|
||||
seen = os.path.join(self.output, "dataset", "img_gen_00007_.png")
|
||||
Path(seen).write_bytes(b"png")
|
||||
recorded = os.path.join(self.output, "Dataset", "IMG_GEN_00007_.png")
|
||||
self.assertNotEqual(os.path.basename(recorded), os.path.basename(seen))
|
||||
if folder_is_case_sensitive(os.path.join(self.output, "dataset")):
|
||||
Path(self.output, "Dataset").mkdir()
|
||||
self.image("IMG_GEN_00007_.png", subfolder="Dataset")
|
||||
else:
|
||||
self.assertTrue(os.path.exists(recorded))
|
||||
folded = staticmethod(lambda path: os.path.realpath(path).lower())
|
||||
with mock.patch.object(ImageGenerationHandler, "_awaiting_key", folded):
|
||||
self.handler.process_new_image(seen)
|
||||
self.assertIsNone(self.linked_text(seen))
|
||||
self.record_run(
|
||||
self.db,
|
||||
self.output,
|
||||
[("Dataset", os.path.basename(recorded))],
|
||||
[],
|
||||
nodes=["43"],
|
||||
)
|
||||
self.assertTrue(self.wait_for(lambda: self.linked_text(seen)))
|
||||
self.assertTrue(self.settled())
|
||||
self.assertEqual(self.linked_text(seen), shapes.BRANCH_TEXTS[3])
|
||||
|
||||
def test_a_full_foreign_set_never_evicts_a_waiting_keyless_file(self):
|
||||
from utils import image_monitor
|
||||
|
||||
keyless = self.image("bare_00001_.png", subfolder="")
|
||||
with mock.patch.object(
|
||||
self.handler.metadata_extractor, "extract_metadata", return_value=None
|
||||
):
|
||||
self.handler.process_new_image(keyless)
|
||||
paths = [self.image(f"IMG_GEN_1000{n}_.png") for n in range(3)]
|
||||
with (
|
||||
mock.patch.object(image_monitor, "MAX_AWAITING_HISTORY", 2),
|
||||
self.assertLogs("prompt_manager.image_monitor", "INFO") as logs,
|
||||
):
|
||||
for path in paths:
|
||||
self.handler.process_new_image(path)
|
||||
self.assertEqual(
|
||||
self.handler.awaiting_foreign(), [self.key(p) for p in paths[1:]]
|
||||
)
|
||||
self.assertEqual(self.handler.awaiting_history(), [self.key(keyless)])
|
||||
self.assertIn(
|
||||
"Too many files wait for their run; IMG_GEN_10000_.png left unlinked",
|
||||
"\n".join(logs.output),
|
||||
)
|
||||
|
||||
def test_an_expired_foreign_entry_is_logged_and_left_unlinked(self):
|
||||
from utils import image_monitor
|
||||
|
||||
path = self.image("IMG_GEN_00008_.png")
|
||||
self.handler.process_new_image(path)
|
||||
later = time.monotonic() + image_monitor.AWAIT_HISTORY_SECONDS + 1
|
||||
with (
|
||||
mock.patch.object(image_monitor.time, "monotonic", return_value=later),
|
||||
self.assertLogs("prompt_manager.image_monitor", "INFO") as logs,
|
||||
):
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [])
|
||||
self.assertIn(
|
||||
"No recorded run for IMG_GEN_00008_.png; left unlinked",
|
||||
"\n".join(logs.output),
|
||||
)
|
||||
self.assertIsNone(self.linked_text(path))
|
||||
|
||||
def test_a_seal_while_the_pending_set_is_full_is_rechecked_later(self):
|
||||
from utils import image_monitor
|
||||
|
||||
path = self.image("IMG_GEN_00009_.png")
|
||||
self.handler.process_new_image(path)
|
||||
blocker = self.image("blocker.png", subfolder="")
|
||||
release = threading.Event()
|
||||
original = self.handler.process_new_image
|
||||
|
||||
def slow(image_path, prompt_snapshot=None):
|
||||
if image_path == blocker:
|
||||
release.wait(5)
|
||||
return None
|
||||
return original(image_path, prompt_snapshot=prompt_snapshot)
|
||||
|
||||
self.handler.process_new_image = slow
|
||||
with mock.patch.object(image_monitor, "MAX_PENDING_IMAGES", 1):
|
||||
self.assertTrue(self.handler.schedule(blocker))
|
||||
self.assertTrue(self.wait_for(lambda: self.handler.pending_count() == 1))
|
||||
self.record_run(
|
||||
self.db,
|
||||
self.output,
|
||||
[("dataset", "IMG_GEN_00009_.png")],
|
||||
[],
|
||||
nodes=["13"],
|
||||
)
|
||||
self.assertEqual(list(self.handler._foreign_due), [path])
|
||||
release.set()
|
||||
self.assertTrue(self.wait_for(lambda: self.linked_text(path)))
|
||||
self.assertTrue(self.settled())
|
||||
self.assertEqual(self.linked_text(path), shapes.BRANCH_TEXTS[0])
|
||||
|
||||
def test_shutdown_forgets_waiting_and_due_foreign_files(self):
|
||||
path = self.image("IMG_GEN_00010_.png")
|
||||
self.handler.process_new_image(path)
|
||||
self.handler._foreign_due[path] = {"filename": "IMG_GEN_00010_.png"}
|
||||
self.handler.shutdown()
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [])
|
||||
self.assertEqual(list(self.handler._foreign_due), [])
|
||||
|
||||
def link_count(self, path):
|
||||
connection = self.db.model.get_connection()
|
||||
return connection.execute(
|
||||
"SELECT COUNT(*) FROM generated_images WHERE image_path = ?", (str(path),)
|
||||
).fetchone()[0]
|
||||
|
||||
def test_a_file_linked_to_another_prompt_never_gets_a_second_link(self):
|
||||
"""One prompt per file: the pair path and the positive path both refuse."""
|
||||
other = self.db.save_prompt(
|
||||
"an earlier prompt", generate_prompt_hash("an earlier prompt")
|
||||
)
|
||||
plain = self.image("plain_00004_.png")
|
||||
self.db.link_image_to_prompt(other, plain)
|
||||
self.metadata = {"prompt": sampled_graph("a quiet harbor", "blurry")}
|
||||
with self.assertLogs("prompt_manager.image_monitor", "INFO") as logs:
|
||||
self.handler.process_new_image(plain)
|
||||
self.assertIn("Foreign run: conflict", "\n".join(logs.output))
|
||||
dataset = self.image("IMG_GEN_00011_.png")
|
||||
self.db.link_image_to_prompt(other, dataset)
|
||||
self.metadata = {"prompt": shapes.shared_prefix()}
|
||||
self.record_run(
|
||||
self.db, self.output, [("dataset", "IMG_GEN_00011_.png")], [], nodes=["13"]
|
||||
)
|
||||
self.handler.process_new_image(dataset)
|
||||
for path in (plain, dataset):
|
||||
self.assertEqual(self.linked_text(path), "an earlier prompt")
|
||||
self.assertEqual(self.link_count(path), 1)
|
||||
for text in ("a quiet harbor", shapes.BRANCH_TEXTS[0]):
|
||||
self.assertIsNone(self.db.get_prompt_by_hash(generate_prompt_hash(text)))
|
||||
|
||||
def test_the_file_size_is_read_only_when_the_ladder_needs_it(self):
|
||||
"""PR 1's lazy size: a foreign file decided without a store reads none."""
|
||||
# Foreign (a core encoder only) and no API graph: never, at step 3
|
||||
encoder = {"type": "CLIPTextEncode", "inputs": [], "widgets_values": ["x"]}
|
||||
self.metadata = {"text_encoder_nodes": [encoder]}
|
||||
path = self.image("bare_00002_.png", subfolder="")
|
||||
with mock.patch("utils.foreign_runs.os.path.getsize") as size:
|
||||
self.handler.process_new_image(path)
|
||||
size.assert_not_called()
|
||||
|
||||
def untraceable(self):
|
||||
"""Unresolved and only the legacy parser names a text (an unknown node)."""
|
||||
graph = sampled_graph("unused", "blurry")
|
||||
graph["1"]["inputs"]["text"] = ["30", 0]
|
||||
graph["30"] = {
|
||||
"class_type": "VendorPromptMaker",
|
||||
"inputs": {"prompt": "a misty valley at dawn"},
|
||||
}
|
||||
return {"prompt": graph}
|
||||
|
||||
def test_no_record_at_settle_keeps_the_file_waiting_for_the_legacy_floor(self):
|
||||
"""Review: a lookup that found no record is not the run's answer yet."""
|
||||
self.metadata = self.untraceable()
|
||||
path = self.image("vendor_00001_.png")
|
||||
self.handler.process_new_image(path)
|
||||
self.assertIsNone(self.linked_text(path))
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [self.key(path)])
|
||||
self.record_run(self.db, self.output, [("dataset", "vendor_00001_.png")], [])
|
||||
self.assertTrue(self.wait_for(lambda: self.linked_text(path)))
|
||||
self.assertTrue(self.settled())
|
||||
self.assertEqual(self.linked_text(path), "a misty valley at dawn")
|
||||
|
||||
def test_a_record_found_at_settle_without_a_certain_prompt_ends_the_wait(self):
|
||||
"""Review: sealed before settle, still ambiguous: no seal is coming."""
|
||||
path = self.image("IMG_GEN_00012_.png")
|
||||
# One run reported it from two savers: a record, but no saver
|
||||
reported = [("dataset", "IMG_GEN_00012_.png")] * 2
|
||||
self.record_run(self.db, self.output, reported, [], nodes=["13", "23"])
|
||||
with self.assertLogs("prompt_manager.image_monitor", "INFO") as logs:
|
||||
self.handler.process_new_image(path)
|
||||
self.assertIn(
|
||||
"No certain prompt for IMG_GEN_00012_.png; left unlinked",
|
||||
"\n".join(logs.output),
|
||||
)
|
||||
self.assertEqual(self.handler.awaiting_foreign(), [])
|
||||
self.assertIsNone(self.linked_text(path))
|
||||
|
||||
def test_monitoring_saved_off_stops_foreign_writes_at_settle_and_recheck(self):
|
||||
"""Review: the observer runs until a restart; nothing is written."""
|
||||
enabled = GalleryConfig.MONITORING_ENABLED
|
||||
self.addCleanup(setattr, GalleryConfig, "MONITORING_ENABLED", enabled)
|
||||
GalleryConfig.MONITORING_ENABLED = False
|
||||
self.metadata = {"prompt": sampled_graph("a quiet harbor", "blurry")}
|
||||
plain = self.image("plain_00005_.png")
|
||||
self.handler.process_new_image(plain)
|
||||
self.assertIsNone(self.linked_text(plain))
|
||||
GalleryConfig.MONITORING_ENABLED = True
|
||||
self.metadata = {"prompt": shapes.shared_prefix()}
|
||||
waiting = self.image("IMG_GEN_00013_.png")
|
||||
self.handler.process_new_image(waiting)
|
||||
GalleryConfig.MONITORING_ENABLED = False
|
||||
reference = {"folder_type": "output", "subfolder": "dataset"}
|
||||
reference["filename"] = "IMG_GEN_00013_.png"
|
||||
self.record_run(
|
||||
self.db, self.output, [("dataset", reference["filename"])], [], nodes=["13"]
|
||||
)
|
||||
self.assertTrue(self.wait_for(lambda: not self.handler.awaiting_foreign()))
|
||||
self.assertTrue(self.settled())
|
||||
self.handler._recheck_foreign(waiting, reference)
|
||||
self.assertIsNone(self.linked_text(waiting))
|
||||
self.assertEqual(self.prompt_count(), 0)
|
||||
|
||||
def test_the_setting_is_read_again_just_before_the_store(self):
|
||||
self.metadata = {"prompt": sampled_graph("a quiet harbor", "blurry")}
|
||||
path = self.image("plain_00006_.png")
|
||||
real = __import__("utils.foreign_runs", fromlist=["x"]).decide_foreign
|
||||
|
||||
def decide_then_switch_off(*args, **kwargs):
|
||||
decision = real(*args, **kwargs)
|
||||
GalleryConfig.SAVE_FOREIGN_RUNS = False
|
||||
return decision
|
||||
|
||||
with mock.patch(
|
||||
"utils.image_monitor.decide_foreign", side_effect=decide_then_switch_off
|
||||
):
|
||||
self.handler.process_new_image(path)
|
||||
self.assertIsNone(self.linked_text(path))
|
||||
self.assertEqual(self.prompt_count(), 0)
|
||||
|
||||
def test_a_seal_between_registering_and_the_settle_lookup_links_once(self):
|
||||
"""Regression guard (passes before the review fixes): one link only."""
|
||||
path = self.image("IMG_GEN_00014_.png")
|
||||
real = __import__("utils.foreign_runs", fromlist=["x"]).history_lookup
|
||||
|
||||
def sealed_first(db, image_path, reference=None):
|
||||
lookup = real(db, image_path, reference)
|
||||
|
||||
def run_then_look():
|
||||
if reference is None: # the settle lookup: the run seals now
|
||||
self.record_run(
|
||||
self.db,
|
||||
self.output,
|
||||
[("dataset", "IMG_GEN_00014_.png")],
|
||||
[],
|
||||
nodes=["13"],
|
||||
)
|
||||
return lookup()
|
||||
|
||||
return run_then_look
|
||||
|
||||
with mock.patch("utils.image_monitor.history_lookup", side_effect=sealed_first):
|
||||
self.assertTrue(self.handler.schedule(path))
|
||||
self.assertTrue(self.wait_for(lambda: self.linked_text(path)))
|
||||
self.assertTrue(self.settled())
|
||||
self.assertEqual(self.linked_text(path), shapes.BRANCH_TEXTS[0])
|
||||
self.assertEqual(self.link_count(path), 1)
|
||||
self.assertEqual(list(self.handler._foreign_due), [])
|
||||
|
||||
def test_a_drained_recheck_never_drops_a_newer_reference(self):
|
||||
path = self.image("IMG_GEN_00015_.png")
|
||||
older, newer = {"filename": "old"}, {"filename": "new"}
|
||||
self.handler._foreign_due[path] = older
|
||||
real = self.handler.schedule
|
||||
|
||||
def schedule(image_path, snapshot=None):
|
||||
# The run is sealed again while the older one is scheduled
|
||||
self.handler._foreign_due[image_path] = newer
|
||||
return real(image_path, snapshot)
|
||||
|
||||
self.handler.schedule = schedule
|
||||
with mock.patch.object(self.handler, "process_new_image"):
|
||||
self.handler._drain_history_due()
|
||||
self.assertIs(self.handler._foreign_due[path], newer)
|
||||
|
||||
def test_an_evicted_due_recheck_is_logged(self):
|
||||
from utils import image_monitor
|
||||
|
||||
self.handler.schedule = lambda *args, **kwargs: False
|
||||
with (
|
||||
mock.patch.object(image_monitor, "MAX_AWAITING_HISTORY", 1),
|
||||
self.assertLogs("prompt_manager.image_monitor", "INFO") as logs,
|
||||
):
|
||||
for name in ("a_00001_.png", "b_00001_.png"):
|
||||
self.handler._schedule_recheck(self.image(name), {"filename": name})
|
||||
self.assertIn(
|
||||
"Too many files wait for their run; a_00001_.png left unlinked",
|
||||
"\n".join(logs.output),
|
||||
)
|
||||
|
||||
def test_a_failed_store_is_logged_and_leaves_the_file_unlinked(self):
|
||||
self.metadata = {"prompt": sampled_graph("a quiet harbor", "blurry")}
|
||||
path = self.image("plain_00003_.png")
|
||||
with (
|
||||
mock.patch(
|
||||
"utils.foreign_runs.ingest_resolved_pair",
|
||||
side_effect=sqlite3.OperationalError("locked"),
|
||||
),
|
||||
self.assertLogs("prompt_manager.image_monitor", "ERROR"),
|
||||
):
|
||||
self.handler.process_new_image(path)
|
||||
self.assertIsNone(self.linked_text(path))
|
||||
|
||||
|
||||
class TestBoundedLookup(RecordedRuns, unittest.TestCase):
|
||||
"""M5: history lookups are batched, and the history schema is unchanged."""
|
||||
|
||||
|
||||
+2
-53
@@ -18,13 +18,11 @@ from pathlib import Path
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SUBPROCESS_TIMEOUT = 60
|
||||
WATCHDOG_ENV = ("PM_TEST_HANG_LIMIT", "PM_TEST_SHUTDOWN_LIMIT")
|
||||
# Integer settings the runner reads; a bad value is a usage error.
|
||||
RUNNER_ENV = (*WATCHDOG_ENV, "PM_TEST_SLOWEST")
|
||||
|
||||
|
||||
def run_suite(folder, extra_env=None, *args, verbosity=("-v",)):
|
||||
"""Run the runner on ``folder`` and return the completed process."""
|
||||
env = {k: v for k, v in os.environ.items() if k not in RUNNER_ENV}
|
||||
env = {k: v for k, v in os.environ.items() if k not in WATCHDOG_ENV}
|
||||
# Windows writes a piped stderr in the ANSI codepage; ask for the UTF-8
|
||||
# this helper decodes, so non-ASCII values and paths survive.
|
||||
env["PYTHONIOENCODING"] = "utf-8"
|
||||
@@ -271,57 +269,8 @@ class RunSuiteTests(unittest.TestCase):
|
||||
self.assertIn("ok", result.stderr, detail)
|
||||
self.assertIn("Timeout (", result.stderr, detail)
|
||||
|
||||
def test_slowest_tests_are_listed_when_asked(self):
|
||||
self.write(
|
||||
"test_probe_slowest.py",
|
||||
"""
|
||||
import time
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_fast(self):
|
||||
pass
|
||||
|
||||
def test_slow(self):
|
||||
time.sleep(0.3)
|
||||
|
||||
def test_slower(self):
|
||||
time.sleep(0.6)
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder, {"PM_TEST_SLOWEST": "2"})
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
report = result.stderr.split("slowest 2 of 3 tests:\n", 1)
|
||||
self.assertEqual(len(report), 2, result.stderr)
|
||||
lines = [line.split() for line in report[1].strip().splitlines()[:2]]
|
||||
self.assertEqual(
|
||||
[line[1] for line in lines],
|
||||
[
|
||||
"test_probe_slowest.Probe.test_slower",
|
||||
"test_probe_slowest.Probe.test_slow",
|
||||
],
|
||||
result.stderr,
|
||||
)
|
||||
self.assertGreaterEqual(float(lines[0][0].rstrip("s")), 0.6)
|
||||
self.assertNotIn("test_fast", report[1])
|
||||
|
||||
def test_slowest_tests_are_not_listed_by_default(self):
|
||||
self.write(
|
||||
"test_probe_quiet.py",
|
||||
"""
|
||||
import unittest
|
||||
|
||||
class Probe(unittest.TestCase):
|
||||
def test_ok(self):
|
||||
pass
|
||||
""",
|
||||
)
|
||||
result = run_suite(self.folder)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertNotIn("slowest", result.stderr)
|
||||
|
||||
def test_invalid_limit_is_a_usage_error(self):
|
||||
for name in RUNNER_ENV:
|
||||
for name in WATCHDOG_ENV:
|
||||
for value in ("abc", "0", "-3", "1.5", "\u00b2"):
|
||||
with self.subTest(name=name, value=value):
|
||||
result = run_suite(self.folder, {name: value})
|
||||
|
||||
@@ -323,43 +323,6 @@ class TestMonitoringSetting(SourcesTestCase):
|
||||
self.assertIs(GalleryConfig.MONITORING_ENABLED, False)
|
||||
|
||||
|
||||
class TestSaveForeignRunsSetting(SourcesTestCase):
|
||||
"""#247 D-Q4: "Save prompts from other workflows", applied live."""
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
saved = GalleryConfig.SAVE_FOREIGN_RUNS
|
||||
self.addCleanup(setattr, GalleryConfig, "SAVE_FOREIGN_RUNS", saved)
|
||||
GalleryConfig.SAVE_FOREIGN_RUNS = True
|
||||
|
||||
async def test_read_and_write_published_live_without_a_restart(self):
|
||||
self.assertIs((await self._get())["settings"]["save_foreign_runs"], True)
|
||||
self._install_monitor(FakeMonitor([self.output_dir]))
|
||||
status, data = await self._post(
|
||||
"/prompt_manager/settings", {"save_foreign_runs": False}
|
||||
)
|
||||
self.assertEqual(status, 200, data)
|
||||
self.assertTrue(data["persisted"])
|
||||
self.assertEqual(data["restart_required_fields"], [])
|
||||
saved = self._read_config()
|
||||
self.assertIs(saved["gallery"]["monitoring"]["save_foreign_runs"], False)
|
||||
self.assertIs(GalleryConfig.SAVE_FOREIGN_RUNS, False)
|
||||
self.assertIs((await self._get())["settings"]["save_foreign_runs"], False)
|
||||
|
||||
async def test_only_true_or_false_is_accepted(self):
|
||||
self._write_config({"custom": {"keep": True}})
|
||||
before = self._config_bytes()
|
||||
for value in ("false", 0, None, []):
|
||||
with self.subTest(value=value):
|
||||
status, data = await self._post(
|
||||
"/prompt_manager/settings", {"save_foreign_runs": value}
|
||||
)
|
||||
self.assertEqual(status, 400, data)
|
||||
self.assertIn("save_foreign_runs must be true or false", data["error"])
|
||||
self.assertEqual(self._config_bytes(), before)
|
||||
self.assertIs(GalleryConfig.SAVE_FOREIGN_RUNS, True)
|
||||
|
||||
|
||||
class TestSourceChangeHold(SourcesTestCase):
|
||||
async def test_root_save_refused_while_scan_job_runs(self):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.a)]
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user