Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Vito Sansevero
|
9579cd8bb6 | ||
|
Vito Sansevero
|
acdc7c3561 | ||
|
Vito Sansevero
|
93fbe5757c | ||
|
Vito Sansevero
|
6f7ae47924 | ||
|
Vito Sansevero
|
4f927549fd | ||
|
Vito Sansevero
|
a842a3c256 |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"required_status_checks": {
|
||||
"strict": false,
|
||||
"strict": true,
|
||||
"contexts": [
|
||||
"py3.12 on ubuntu-latest",
|
||||
"py3.13 on ubuntu-latest",
|
||||
|
||||
@@ -4,7 +4,6 @@ permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
+14
-38
@@ -4,7 +4,6 @@ permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
@@ -64,18 +63,16 @@ jobs:
|
||||
comfyui:
|
||||
name: real ComfyUI compatibility
|
||||
runs-on: ubuntu-latest
|
||||
# Chromium and its system libraries come from the pinned Playwright image (tag = @playwright/test in
|
||||
# tools/vendor/package-lock.json; tests/vendor/browser-image.test.mjs checks it). The runner install
|
||||
# pulled them from Ubuntu's apt mirror on every run, and three runs (#349, #378, #382) crawled at
|
||||
# ~26 kB/s for 11-12 minutes until the job hit its limit after every test had passed.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 24
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: npm
|
||||
cache-dependency-path: tools/vendor/package-lock.json
|
||||
- name: Read the reviewed runtime pins
|
||||
id: runtime
|
||||
run: |
|
||||
@@ -100,42 +97,21 @@ jobs:
|
||||
ref: ${{ steps.runtime.outputs.kikoRevision }}
|
||||
path: .ci-kikotools
|
||||
persist-credentials: false
|
||||
# The image's own Python and Node 24: setup-python/setup-node install the runner's toolcache builds,
|
||||
# whose entry points name host paths that do not exist in the container. The image has no ensurepip
|
||||
# and the job has no root, so a checksum-pinned pip wheel bootstraps a user-space venv. The image
|
||||
# Python must be the reviewed host Python, so a pin or image bump that disagrees fails here.
|
||||
- name: Python environment on the reviewed host Python
|
||||
env:
|
||||
PIP_WHEEL: https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl
|
||||
PIP_SHA256: 71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e
|
||||
PINNED_PYTHON: ${{ steps.runtime.outputs.pythonVersion }}
|
||||
run: |
|
||||
image="$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])')"
|
||||
[ "$image" = "$PINNED_PYTHON" ] || { echo "::error::image Python $image is not the pinned $PINNED_PYTHON"; exit 1; }
|
||||
venv="$RUNNER_TEMP/venv" wheel="$RUNNER_TEMP/pip-26.2.1-py3-none-any.whl"
|
||||
python3 -m venv --without-pip "$venv"
|
||||
curl -fsSL --retry 3 -o "$wheel" "$PIP_WHEEL"
|
||||
echo "$PIP_SHA256 $wheel" | sha256sum -c -
|
||||
"$venv/bin/python" "$wheel/pip" install --no-index --no-cache-dir "$wheel"
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
# pip's download cache (torch alone is ~200 MB); inside the container ~ is the job's HOME (/github/home).
|
||||
- name: Cache pip downloads
|
||||
uses: actions/cache@v6
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: pip-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('requirements-dev.txt', '.ci-comfyui/requirements.txt', 'tools/vendor/comfyui-runtime.json') }}
|
||||
python-version: ${{ steps.runtime.outputs.pythonVersion }}
|
||||
cache: pip
|
||||
cache-dependency-path: |
|
||||
requirements-dev.txt
|
||||
.ci-comfyui/requirements.txt
|
||||
tools/vendor/comfyui-runtime.json
|
||||
- name: Install CPU runtime and test dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}" --index-url https://download.pytorch.org/whl/cpu
|
||||
pip install -r .ci-comfyui/requirements.txt -r requirements-dev.txt "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}"
|
||||
# npm's download cache, keyed on the lockfile. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- run: npx --prefix tools/vendor playwright install --with-deps chromium
|
||||
- name: Classic and Nodes 2.0 startup and compatibility
|
||||
env:
|
||||
PM_COMFY_ROOT: ${{ github.workspace }}/.ci-comfyui
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
name: Viewer.js vendor and compatibility
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
|
||||
+27
-417
@@ -1245,8 +1245,7 @@ Recorded deviations on this page (spec §8):
|
||||
`#bulkbar` shows "N selected" and only the real actions in the concept's order (Add
|
||||
tags, then D85's Auto-tag selected, Set category, Export metadata, Clear). Add to
|
||||
collection waits for collections; "Remove links…" is dropped. (Amended by #249 PR D:
|
||||
Add tags ships, see "Bulk tagging from the selection bar"; #392 adds range, keyboard,
|
||||
touch and "all N matching" selection, see "Selecting many images", D135.) The selection survives Load more and
|
||||
Add tags ships, see "Bulk tagging from the selection bar".) The selection survives Load more and
|
||||
bounded re-mounting and is cleared whenever the collection is replaced (search,
|
||||
category, sort, Filters, source change, rescan, reload). A return to Images restores
|
||||
the saved collection unreplaced and keeps its selection; a full page reload starts
|
||||
@@ -1684,9 +1683,6 @@ targets of the longest legal path (1024 code points, even escaped as `\uXXXX`) f
|
||||
unchanged, `-` skipped and `!` error.
|
||||
- The progress line is the scan's: "Processing file N of M · about X min left", the
|
||||
estimate (D56's `_time_left`) shown once two files are done, each file one batch.
|
||||
An ended job never keeps it: its status line becomes "<Add tags | Auto-tag>
|
||||
<finished | cancelled> · N files processed" (or the storage-restored sentence),
|
||||
pinned by `test_an_ended_job_shows_no_time_left` (#379 live check).
|
||||
- Download report saves `prompt-manager-image-tags-<YYYYMMDD-HHMM>.json`.
|
||||
- The start toasts are "Tagging started." / "Auto-tagging started." (D85); the end
|
||||
toasts follow the design's pattern with the job's counts, as the scan's: "<title>
|
||||
@@ -1723,65 +1719,6 @@ busy". Units:
|
||||
| Copy without a concept line: no tags, start fallbacks, the panel heading | D118 |
|
||||
| Activity item and job view for image tagging jobs | D89 (plan D66) |
|
||||
|
||||
### Selecting many images (#392)
|
||||
|
||||
**D135** (beyond the concept, which draws only the per-card checkbox): Images selects
|
||||
like a file manager, in Grid, List (where the concept hides the checkbox) and Masonry,
|
||||
over every loaded batch. The selection is a set of image keys in display order, so
|
||||
ranges include cards that bounded rendering has unmounted.
|
||||
|
||||
- Clicking a checkbox toggles it, and that image becomes the anchor. Ctrl/Cmd+click
|
||||
anywhere on a card (image, star, Info, Tag) also toggles it and makes it the anchor;
|
||||
nothing opens, inspects or changes the favorite.
|
||||
- Shift+click on a card selects the range from the anchor to it. Shift+click on a
|
||||
checkbox gives its range the box's new state, so it can clear a range. The anchor
|
||||
stays where it is. Without an anchor, the clicked image becomes the anchor.
|
||||
- In the grid, Ctrl/Cmd+A selects every loaded image; in the search box it keeps its
|
||||
text meaning. Shift+Arrow, Shift+Home and Shift+End move focus as the plain keys do,
|
||||
then select from the anchor (the starting card when there is none) to the new card.
|
||||
Escape in the grid clears the selection. Escape in the bar clears it too, and focus
|
||||
moves to the heading, as after Clear.
|
||||
- Touch: a 500 ms long-press (at most 10 px of movement) selects the card and turns on
|
||||
touch selection mode. The context menu during the press and the click after it are
|
||||
swallowed, and the card has no image callout. Taps then toggle cards until the
|
||||
selection is empty. A keyboard Enter or Space keeps its plain meaning.
|
||||
- A plain click keeps its meaning: the image opens the viewer, and Info inspects.
|
||||
- When every loaded image is checked and the listing has more, the bar offers
|
||||
**Select all N matching** right after the count. Pressing it gives **All N matching
|
||||
selected**: later batches arrive checked, and unchecking any image goes back to the
|
||||
loaded images minus that one. Clear, Escape, and a new search, filter, sort or rescan
|
||||
end it. A refresh after a tag save keeps it. Return visits keep the loaded images,
|
||||
and the offer shows again.
|
||||
- In that mode, Add tags and Auto-tag selected start the `image-tags` job with
|
||||
`filter` (the listing's own query as an object: `q`, `subfolder`, `category`,
|
||||
`tags`, `root`, `media`, `sort`, `favorites`, and with a Source the loaded root's
|
||||
`root_scope`) instead of `targets`. The server resolves the filter at start, in
|
||||
listing order, using the listing's own admission code (`_admitted_entries`). That
|
||||
means "all matching" when the action starts: an output written since then that
|
||||
matches is included. A `root_scope` that no longer names its root (the sources were
|
||||
renumbered) answers 409 `source-changed`. The limit is 10,000 images (`MAX_LISTING_SELECTION`).
|
||||
Over the limit, the client toasts "Narrow the view to at most 10,000 images." and the
|
||||
server answers 400 `too-many-images`. No match is 400 `no-images`. Auto-tag in this
|
||||
mode does not offer Review each image, because the review walks images in the
|
||||
browser. Set category posts `{images_filter, category}` to `/prompt_manager/bulk/category`.
|
||||
The server sets the category on the distinct linked positives and answers
|
||||
`prompt_count` and `skipped`. A listing whose `positive_count` is 0 toasts the
|
||||
nothing-linked sentence and opens no dialog, and over 10,000 matches it toasts the
|
||||
limit sentence. Export metadata reads every image in
|
||||
the browser, so it is `aria-disabled` in this mode, and its `title` says why.
|
||||
|
||||
Tests: `tests/browser/workspace-images.spec.mjs` ("Shift+click selects a range across
|
||||
an appended batch in Grid, List and Masonry", "Ctrl/Cmd+click toggles a card without
|
||||
opening it, and a plain click still opens the viewer", "Ctrl/Cmd+A, Select all N
|
||||
matching and Add tags tag every matching image on the server", "Shift+arrow keys extend
|
||||
the selection from the focused card, and Escape in the bar clears it", "a touch
|
||||
long-press starts selection mode, taps then toggle, and an empty selection ends it");
|
||||
units `tests/js/workspace-images-multiselect.test.js`,
|
||||
`tests/js/workspace-images-tagging-bulk.test.js` ("all matching sends the listing
|
||||
filter instead of targets"), `tests/js/workspace-images-state.test.js`
|
||||
(`listingFilter`); Python `tests/test_image_tag_jobs.py` (`TestFilterStart`) and
|
||||
`tests/test_api_workspace_images.py` (`TestListingFilterSelection`).
|
||||
|
||||
Native image tagging (#249 PR B routes, PR C UI): `POST
|
||||
/prompt_manager/image-annotations/observe` (capture one listed file, its saved set and
|
||||
`tag_revision`), `GET`/`DELETE /prompt_manager/image-annotations/capture/<token>`
|
||||
@@ -2765,8 +2702,7 @@ the connection's `total_changes` is 0 at close and the data reads back unchanged
|
||||
`-shm` bytes are not compared: every WAL reader rewrites its read marks there, so no
|
||||
reader can keep it byte-identical (`test_health_check_never_writes`).
|
||||
|
||||
Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107, D109–D110 and D111–D114; D121–D134 assigned). D94,
|
||||
the per-section hand-off panel, was retired by S12 (#329): every section is native.
|
||||
Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107 and D111–D114; D121–D134 assigned):
|
||||
|
||||
- **D92** (shell): no "Sample configuration" chip. The Save status is truthful: "No
|
||||
unsaved changes.", "Unsaved changes.", "Saved in this browser." / "Saved to the server
|
||||
@@ -2798,6 +2734,10 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
"N output sources" with "Monitoring and enrichment share one source list.", prefixed
|
||||
by "M of N configured sources available." or "None configured is available; ComfyUI's
|
||||
output folder is used." when that is the case.
|
||||
- **D94** (all sections): a section not delivered yet shows the hand-off panel "Your
|
||||
current tools are ready" with a link to each legacy tool that still owns it (the
|
||||
plan's legacy coverage table); retired section by section, gone after S12. Since S11
|
||||
(#328) only Diagnostics & logs hands off (`SECTION_HANDOFFS`, to the admin page).
|
||||
- **D95** (health report): the concept's `health-check` dialog titled "Health report",
|
||||
with no sample note. It lists every check in the server's order: Integrity check,
|
||||
Foreign keys & pair references, Canonical hashes, Pair uniqueness (added), Missing
|
||||
@@ -2828,7 +2768,7 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
- **D123** (Maintenance): the concept's Maintenance notice
|
||||
("Check first. Review changes. Then apply." + Run health checks) was delivered ahead of
|
||||
the rest of Maintenance (S2), as the native place that runs the health check; since S7
|
||||
the section's two surfaces follow it and its hand-off panel is gone. A press shows "Checking…" (`aria-disabled`) and opens the report; a
|
||||
the section's two surfaces follow it and its D94 hand-off is gone. A press shows "Checking…" (`aria-disabled`) and opens the report; a
|
||||
failed check toasts "The health check could not finish. Nothing was changed."
|
||||
- **D97** (Appearance): added rows for the legacy gallery settings the concept leaves
|
||||
out, after its own rows: "Show image details on cards" (`showImageInfo`; off hides a
|
||||
@@ -2949,11 +2889,6 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
and the media hashes, `GET /__test__/system/digest`). `plan dialogs match the concept`
|
||||
(system-maintenance-concept.spec, both themes and sizes) compares each of the four
|
||||
operations' dialogs with the concept's, mapped by these values only.
|
||||
S12 (#329): the `rotated_logs` plan (Diagnostics & logs › Review log cleanup, `h3`
|
||||
"Rotated log copies") changes no library record: its Backup reads "Not needed: no
|
||||
library record changes" (`backup.policy: "none"`), it has no D55 backup box, and its
|
||||
`details` reads "Listed files (N)" (`a plan without a database change reads Not needed
|
||||
and shows no backup box`, `rotated_logs details read Listed files`).
|
||||
- **D101** (Maintenance, S7): Duplicate image files → Find duplicates starts the
|
||||
duplicate search (Activity, D111); its job view adds "Review groups" after Download
|
||||
report once it finished with groups. The concept's `duplicates` dialog ("Review
|
||||
@@ -3212,88 +3147,25 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
the persisted count the Tag image dialog's Suggestions field (D91) starts from. Pinned
|
||||
by the comparison's mapping (it inserts the row) and `defaults save through Save
|
||||
preferences and the Tag image dialog uses them`.
|
||||
- **D134** (Auto-tagging, S11, OQ2/OQ3/OQ7/OQ8; amended by #379): an `.operation` "Tag
|
||||
output images" with "Review" follows `.row-actions`: the per-image batch (#379), whose
|
||||
description reads "Tag each output image from its own pixels with the model above.
|
||||
Prompts are not changed. Review the scope first.". Review is
|
||||
- **D134** (Auto-tagging, S11, OQ2/OQ3/OQ7/OQ8): an `.operation` "Tag output images"
|
||||
with "Review" follows `.row-actions`: the legacy prompt-tag batch. Review is
|
||||
`aria-disabled` (the press explains) while the chosen engine is not downloaded or a
|
||||
value it uses is invalid; otherwise it posts `/autotag/batch/preview` with the drafted
|
||||
engine and policy (read-only) and opens "Review tag output images" in the concept's
|
||||
record-changing review (its `maintenancePlan` shell): `h3` "Tag output images", the
|
||||
`dl.meta-grid` Model, Existing tags, Images, Already auto-tagged, Images to tag
|
||||
("about N"), Time, Settings ("General <g> · Character <c>" or "Template of N
|
||||
characters", then " · up to L tags per image" with the preview's saved
|
||||
`suggestion_limit`) and Backup, the estimate sentence ("Nothing is written until you
|
||||
start. The number to tag is an estimate: images changed since they were tagged are
|
||||
tagged again, and missing files are skipped. Activity reports what was tagged.") in the
|
||||
`p.small`, the D55 backup box and Cancel / "Start tagging", which starts the
|
||||
cancellable `autotag` job and opens it. Time (DEC11) is `m = max(1,
|
||||
ceil(estimate_seconds / 60))` read "about m min" under 90 and "about H h M min" (or
|
||||
"about H h") from 90, followed by " on a processor; a graphics card can be faster."
|
||||
or, when the preview says WD14 runs on CUDA, " on the graphics card."; the language
|
||||
models read "Language-model engines take longer per image. Activity shows the time
|
||||
left once tagging starts.". The option label "Skip already tagged" is the concept's;
|
||||
for this batch it skips images already auto-tagged (hand tags do not count). Replace
|
||||
after review, a model not on disk, a short disk, no image and nothing to tag each show
|
||||
the rows with only Cancel and the reason ("Replacing tags needs a review of each
|
||||
image. Use Preview tagging review, or choose Skip or Merge for a batch run.", "The
|
||||
output folders contain no images to tag.", "Every output image is already
|
||||
auto-tagged.", "Not enough free disk space: the safety backup needs about X MB and Y
|
||||
MB is free. Nothing is written until there is room.") in the close-only review
|
||||
Integrations shares (`openCloseOnlyReview`). Pinned by
|
||||
`dl.meta-grid` Model, Existing tags, Images linked, Prompts, Already tagged, Prompts
|
||||
to tag ("up to N"), Settings ("General <g> · Character <c>" or "Template of N
|
||||
characters") and Backup, the upper-bound sentence in the `p.small`, the D55 backup box
|
||||
and Cancel / "Start tagging", which starts the cancellable `autotag` job and opens it.
|
||||
Replace after review, a model not on disk, a short disk, no linked image and nothing to
|
||||
tag each show the rows with only Cancel and the reason ("Replacing tags needs a review
|
||||
of each image. Use Preview tagging review, or choose Skip or Merge for a batch run.",
|
||||
"Every prompt with an output image is already tagged.", "Not enough free disk space:
|
||||
the safety backup needs about X MB and Y MB is free. Nothing is written until there is
|
||||
room.") in the close-only review Integrations shares (`openCloseOnlyReview`). Pinned by
|
||||
`tag output images review matches the concept`, its mutation check and `the
|
||||
Cancel-only review matches the concept review shell` (system-tagging-concept.spec) and
|
||||
the batch tests of workspace-system-tagging.spec; the mutation check renames each grid
|
||||
label in turn.
|
||||
- **D109** (Logs, S12, #329): the concept's Logs surface on the live server. The toolbar
|
||||
keeps "Level", Refresh and the download (renamed "Download log": the active log,
|
||||
`/logs/download/prompt_manager.log`, through the api client), with all six minimum
|
||||
levels (All levels, DEBUG, INFO, WARNING, ERROR, CRITICAL; the concept has three) and an
|
||||
added `label` "Lines" + `input#log-limit[type=number]` 1–500 (default 100): a value that
|
||||
is not a whole number 1–500 after trimming gets `aria-invalid` and sends nothing, a
|
||||
Level change included. The `pre` shows the in-memory buffer newest first as `HH:MM:SS
|
||||
LEVEL message` ("No log entries at this level." / "The log could not be read."); a
|
||||
newer read wins. Added after it, `p.small#log-stats[role=status]` "Log statistics":
|
||||
"N entries in memory · E errors · W warnings · F files · S on disk" (errors are ERROR +
|
||||
CRITICAL, singular for 1, links not counted). "Runtime log level" adds CRITICAL after
|
||||
the concept's INFO, WARNING, DEBUG, ERROR; it and "Console logging" are runtime drafts
|
||||
saved by Save preferences (`POST /logs/config`, only the changed keys) and read the
|
||||
server's values; their descriptions say "Applied until the server restarts." (the
|
||||
concept's say a gallery toggle is replaced and output stays quiet). Inside "Log files &
|
||||
retention", after the concept's line ("prompt_manager.log · <size> · rotated copies:
|
||||
N", N the `rotated` rows) and Review log cleanup (D100's `rotated_logs` plan): a
|
||||
`p.small` "Rotates at M MB · keeps K rotated copies · file logging on|off · memory
|
||||
buffer B entries" (read-only), an `.operation-list` of `/logs/files` (`h3` name, `p`
|
||||
"<size> · <YYYY-MM-DD HH:MM> · Active log | Rotated copy | Other file | Link, not
|
||||
downloaded", "Download" named "Download <name>" except for a link), and the danger
|
||||
`.operation` "Clear all logs" / "Empty the active log and delete its rotated copies." /
|
||||
"Clear all logs…", which confirms ("Empties prompt_manager.log, deletes N rotated
|
||||
copies (S) and clears the lines shown here. The other files in the log folder are
|
||||
kept. This cannot be undone."), runs `POST /logs/truncate`, toasts "Logs cleared · N
|
||||
rotated copies deleted." (plus " · K could not be deleted") and reads the viewer,
|
||||
statistics, files and configuration again. Pinned by workspace-system-diagnostics.spec's
|
||||
Task 5 and Task 6 tests.
|
||||
- **D110** (Diagnostics, S12, #329): the concept's Diagnostics surface with live values.
|
||||
The `div.meta-grid` `dd`s read one `GET /system/diagnostics`: Database "Readable ·
|
||||
<size> · N images" or "Not readable: <message>"; Sources "N sources · M watching", "N
|
||||
sources · K not available", or with none configured "ComfyUI output folder · watching
|
||||
| manual scan"; Viewer.js "<pinned version> · pinned vendor". Host renderers is D106's
|
||||
renderer line from the send bridge. Each reads "Loading…" first, "Busy: <job> is
|
||||
running." when an exclusive job refuses the read (D57's naming) and "Could not be
|
||||
read." on any other failure. Run checks opens the shared Health report (D95). The
|
||||
buttons drop the concept's "sample": "Download diagnostic report" saves a fresh report
|
||||
plus `host_renderers` as `prompt-manager-diagnostics-YYYYMMDD-HHMM.json` (local time)
|
||||
and toasts "Diagnostic report downloaded." ("Busy: <job> is running. Try again when it
|
||||
ends." / "The diagnostic report could not be downloaded. Nothing was saved to this
|
||||
device."); "Inspect" opens `#/inspect`, which reads a file and writes nothing. An added
|
||||
`.operation` "Test image link" / "Write a placeholder image record to check that
|
||||
linking works." / "Create test link…" keeps the legacy synthetic link (S12-11): it
|
||||
reads the most recent prompt on mount (`aria-disabled` with "Add a prompt first." or
|
||||
"The most recent prompt could not be read." added to the description), confirms in
|
||||
"Create test link" naming the prompt and how Missing-file links removes the record,
|
||||
and toasts "Test link created (image record N)." or "The test link could not be
|
||||
created. Nothing was changed.". Pinned by workspace-system-diagnostics.spec's Task 4
|
||||
tests.
|
||||
the batch tests of workspace-system-tagging.spec.
|
||||
- **D124** (inspector): with "Load metadata on selection" off, the Images panel and
|
||||
the viewer's details show "Metadata loads when you ask for it." and a "Load metadata"
|
||||
button instead of reading the file. The concept has no such state, so no concept
|
||||
@@ -3826,9 +3698,7 @@ threshold off that grid is refused ("General-tag confidence must be a multiple o
|
||||
0.05.", `validate_saved_threshold`; `draftProblem` holds back Preview tagging review and
|
||||
the batch Review with the same sentence) and falls back on load; an accepted one is
|
||||
stored exactly on the grid (#369). A per-request threshold keeps its own check: `/image-annotations/suggest` and
|
||||
`/jobs/autotag` take 0–1; the legacy `/autotag/start` passes its options through
|
||||
`validate_options` and starts the `autotag` job titled "Tag output images (legacy page)";
|
||||
the legacy `/autotag/single` only requires a number. `GET /autotag/models` reports the
|
||||
`/jobs/autotag` take 0–1; the legacy `/autotag/single` and `/autotag/start` only require a number. `GET /autotag/models` reports the
|
||||
saved values (`wd14_general_threshold`, `wd14_character_threshold`, `suggestion_limit`,
|
||||
`default_prompt`, `model_type`, `existing`), so the Tag image and Auto-tag selected
|
||||
dialogs start from them; a preset (the bulk review, Preview tagging review) wins.
|
||||
@@ -3854,96 +3724,19 @@ images", "Tag output images (legacy page)", "Tagging one image (legacy page)",
|
||||
`409 busy`; the legacy SSE route attaches to the same job and keeps its frames.
|
||||
- **Unload**: `POST /prompt_manager/autotag/unload` answers `{success, message}` or
|
||||
`409 model-busy`.
|
||||
- **Tag output images** (per image since #379; prompt tags and prompt text are never
|
||||
written): its targets are every still image the output folders list (the Images
|
||||
listing of each root, linked to a prompt or not; videos are not targets; a file under
|
||||
two overlapping roots is one target). `POST /prompt_manager/autotag/batch/preview
|
||||
{model_type, existing}` writes nothing and opens or stats no image file (it may wait
|
||||
for the shared listing walk); it answers `images`, `already_tagged` (images with any
|
||||
auto-tagged byte record), `to_tag` (`images − already_tagged` under Skip, `images`
|
||||
under Merge; both estimates), `estimate_seconds` (WD14: `to_tag` × 1.6 s on the
|
||||
processor or × `WD14_GPU_SECONDS_PER_IMAGE` when ONNX Runtime offers CUDA, see
|
||||
`py/tagging_defaults.py`; `null` for GGUF and HF), `estimate_device` (`cpu`, `gpu`,
|
||||
or `null`), `suggestion_limit` (the saved count the run uses), `model {type, label,
|
||||
size_label, downloaded}`, `existing`, `review_required`, `backup {policy}` and
|
||||
`disk`. `POST
|
||||
- **Tag output images**: `POST /prompt_manager/autotag/batch/preview {model_type,
|
||||
existing}` writes nothing and opens no file; it answers `images`, `prompts`,
|
||||
`already_tagged`, `to_tag` (an upper bound), `model {type, label, size_label,
|
||||
downloaded}`, `existing`, `review_required`, `backup {policy}` and `disk`. `POST
|
||||
/prompt_manager/jobs/autotag {model_type, existing, options}` (WD14: both thresholds;
|
||||
GGUF/HF: `instructions`, the template) starts the cancellable `autotag` job (202),
|
||||
title "Tag output images", admission `records`, a safety backup by the policy first.
|
||||
The job runs the `image-tags` loop (`_tag_each_file`) on each image: a capture, at
|
||||
most `suggestion_limit` model tags, a re-hash, then a union into that image's tags
|
||||
that also marks its bytes auto-tagged (`media_image_autotags`, one row per byte
|
||||
record; an empty result and an unsupported still are marked too; File changed, File
|
||||
not found, Too large to verify, read, inference and tag-limit errors are not). Phases,
|
||||
each checked for Cancel: backup, model load, listing ("Listing output images…"), a
|
||||
pre-pass that stats each file without opening it (gone: "File not found", over 256
|
||||
MiB: "Too large to verify", under Skip a marked current record: "Already
|
||||
auto-tagged"; counted with `record_counted`, rows up to the cap, no log line each),
|
||||
then the loop ("Processing file {k + n} of {T}", the time left from the model's own
|
||||
rate, D56 unchanged). It ends "Tagged N images", "Cancelled. Tags already added were
|
||||
kept.", or failed with "Prompt storage was restored; start the job again.", "The
|
||||
output folders could not be listed. Try again." or the model's refusal.
|
||||
Refusals: `400 review-required` (Replace after review never runs as a batch), `409
|
||||
model-unavailable` ("<label> is not downloaded. Download it in System ›
|
||||
Auto-tagging."), `507 insufficient-space`, `409 busy`, `409 model-busy`. The section
|
||||
toasts `model-busy` with its operation, the other three with the server's sentence,
|
||||
`busy` with the running job (D57), anything else "<what> could not start. Try again.".
|
||||
Outcomes: Tagged / Unchanged / Skipped / Errors (the `image-tags` descriptor, D111).
|
||||
`tests/test_autotag_batch.py` pins it, including two images of one prompt getting
|
||||
different tags with no `prompts` or `prompt_tags` row changed; the marks' schema group
|
||||
is pinned by `tests/test_autotag_schema.py` (forward-only: one empty `WITHOUT ROWID`
|
||||
table, optional in old backups, refused without the annotation group).
|
||||
|
||||
- **The legacy pages (#379, PR 2)** write image tags, never prompt tags.
|
||||
`POST /autotag/start` (the admin and gallery batch) reads `model_type`, `prompt`,
|
||||
`skip_tagged` (Skip or Merge), `keep_in_memory` and the thresholds (else the saved
|
||||
ones), checks them with `validate_options` (WD14: both thresholds; GGUF/HF:
|
||||
`instructions` = the non-blank `prompt`, else the saved template; the saved
|
||||
"Suggestions per image") and starts the `autotag` job "Tag output images (legacy
|
||||
page)" (gate owner `legacy-batch`) through the native start, so the safety backup and
|
||||
the disk check apply. A second legacy batch answers JSON 409 `{success: false, error:
|
||||
"Auto-tagging is already running"}` (also when another legacy start wins the race past
|
||||
that check); `SseStream` hands a non-2xx JSON `error` to `onerror` as `serverMessage`,
|
||||
which both pages show instead of "Connection lost during auto tagging". Any other refusal (a job in the way: "<title>
|
||||
already running", `model-unavailable`, `insufficient-space`, `model-busy`, a bad
|
||||
option) is one `{type: "error", status: "error", message}` frame. The stream relays
|
||||
the job: the phases as `{type: "progress", progress: 0 | 5 | 8, status: "Taking a
|
||||
safety backup…" | "Loading model…" | "Listing output images…"}`, each image as
|
||||
`{type: "progress", progress: 10 + progress × 85 / 100, status, processed, tagged,
|
||||
skipped}` (at most every 0.5 s; `processed` = Tagged + Unchanged + Errors), and the end
|
||||
as `complete` (`processed`, `tagged`, `skipped`, `errors`, `skipped_outside_roots: 0`,
|
||||
`status` "Complete" or "No still images found", `model_status`, `model_loaded`),
|
||||
`{type: "cancelled", message: "Cancelled. Tags already added were kept."}` or an
|
||||
`error` frame with the job's status line. A job quiet for 5 s repeats its last frame,
|
||||
and a closed stream cancels the job (the legacy pages have no other stop).
|
||||
`POST /autotag/single` takes `existing` (`"skip"` or `"merge"`, default merge). Under
|
||||
Skip, an image whose current bytes are auto-tagged answers `{success, already_tagged:
|
||||
true, image_tags, prompt_id, filename}` after one stat, without the model gate.
|
||||
Otherwise the image is captured and the model runs on the capture under the gate with
|
||||
the saved cap; the answer adds `tags`, `sha256` (the captured bytes) and `image_tags`,
|
||||
and `service.custom_prompt` is never set. Refusals: `409 model-unavailable`, `413
|
||||
too-large`, `404 file-not-found`, `409 file-changed`, `503 busy`, `415 not-a-still`
|
||||
(the image is marked), `409 model-busy`. `POST /autotag/apply` takes exactly
|
||||
`{image_id, sha256, tags}` or `{path, root_index?, sha256, tags}` (64 KiB), re-hashes
|
||||
the file and unions the tags into that image (`add_image_tags`, marked): `{success,
|
||||
added_tags, total_tags}`; an empty list only marks the image. A body with `prompt_id`
|
||||
is `400` "This page is out of date. Reload it."; a changed file `409 file-changed`
|
||||
("The image changed since its tags were generated."); a full set `409 tag-limit`; a
|
||||
restore `409 storage-changed`. The admin review asks each image with Skip first: an
|
||||
auto-tagged one shows "Image Already Auto-tagged" / "A tagging model already tagged
|
||||
this image. Generate new tags?" with its image tags (Skip, Skip All Tagged, Re-tag asks
|
||||
again with Merge); the gallery review always merges. Apply & Next sends the image and
|
||||
its `sha256` (linked or not). The batch copy: "Skip images already auto-tagged",
|
||||
"Auto tagging complete! Tagged N images." ("1 image"), and under both progress bars
|
||||
"Closing this window stops tagging. Tags already added are kept. Prompt Manager's
|
||||
Activity page shows the time left.". Pinned by `tests/test_autotag_legacy.py`,
|
||||
`tests/js/image-helpers.test.js` and `legacy-autotag.spec`.
|
||||
- **Tag image dialog marks (DEC4).** A save after Generate suggestions ran on that
|
||||
capture marks the image auto-tagged (also a replayed save, and a saved retry after
|
||||
`tag-conflict`); a save without Generate does not. The API keeps the suggested capture
|
||||
tokens; each new one drops the tokens whose capture is gone (`expires_in`, which never
|
||||
resets a capture's idle clock), a release or a saved receipt drops its own, and
|
||||
closing the captures clears them (`TestDialogMarks`).
|
||||
|
||||
No browser test downloads or runs a model: the injected `TestModel` answers, its
|
||||
`download_model` raises unless `/__test__/tagging/download {mode: hold | succeed |
|
||||
@@ -3952,180 +3745,6 @@ it. Every workspace-system-tagging test asserts no `/autotag/download/` request.
|
||||
Changing the routes or `py/config.py` needs a ComfyUI restart; the section itself is
|
||||
HTML, CSS and JS only.
|
||||
|
||||
### Logs and diagnostics contract (#249 System S12)
|
||||
|
||||
System › Diagnostics & logs (`web/js/workspace/system-diagnostics.mjs`, D109–D110)
|
||||
reads and deletes only through the routes below; the legacy admin Logs and Diagnostics
|
||||
modals use the same routes and keep working.
|
||||
|
||||
- **Rule R** (`utils/logging_config.py`): a file may be deleted as a rotated copy only
|
||||
when it is a direct entry of the log folder named `prompt_manager.log.<n>` (`<n>` a
|
||||
decimal integer ≥ 1 with no leading zero, compared after `normcase`) and a regular
|
||||
file that is not a link or junction (`lstat`). Every other entry is `active`, `link`
|
||||
or `other` and is never deleted. `/logs/files` rows carry that `kind`; links are listed
|
||||
with their own `lstat` size and are not counted or sized by `/logs/stats`. A download
|
||||
is served only for a regular, non-link file of the folder (a link, a folder or a name
|
||||
not listed is 404), read off the event loop.
|
||||
- **Reviewed cleanup**: `POST /maintenance/preview {operation: "rotated_logs", options:
|
||||
{}}` lists every rotated copy (basenames, size and time) and writes nothing; `POST
|
||||
/maintenance/apply {token}` starts the `maintenance` job "Rotated log copies"
|
||||
(admission `read`, not cancellable). Under the manager's files lock and every file
|
||||
handler's lock it re-scans; when any listed file is missing, no longer rotated or
|
||||
differs in size, `mtime_ns` or a non-zero inode, it deletes nothing and fails
|
||||
`plan-changed` ("The log files changed since the review; nothing was deleted. Review
|
||||
again.", Activity offers Review again). Otherwise it deletes exactly the listed files;
|
||||
each is a Changed row ("Deleted · <size>"), a file that cannot be removed an Errors
|
||||
row. Two Prompt Manager processes sharing one log folder are not locked against each
|
||||
other: each file's identity is checked just before its unlink.
|
||||
- **Clear all logs** (`POST /logs/truncate`, also the admin page's Clear) empties the
|
||||
active log through its handler, deletes the rotated copies (Rule R) and clears the
|
||||
buffer; other files and links are kept. Response keys are unchanged.
|
||||
- **Runtime settings** (`GET/POST /logs/config`): the level (DEBUG…CRITICAL) and console
|
||||
logging apply until the server restarts; the retention values are shown read-only in
|
||||
the workspace and the API keeps accepting them.
|
||||
- **Report** (`GET /system/diagnostics`): built on the server from an explicit list of
|
||||
fields (version, Python, platform, Viewer.js version, the four legacy checks, database
|
||||
info, source counts, log configuration and statistics, the session's health summary
|
||||
and the non-path configuration with `has_civitai_api_key`). Every path is relative to
|
||||
a ComfyUI anchor or reads "outside ComfyUI"; a backstop scrub replaces any absolute
|
||||
path or the saved key and counts it in `redacted`, which the tests require to be 0. It
|
||||
takes a registry lease (409 `busy` while an exclusive job such as a restore runs).
|
||||
- **Compressed downloads**: the gzip middleware drops a handler's declared
|
||||
`Content-Length` before it compresses a body, so the response declares the compressed
|
||||
length. A stale length made a browser wait for bytes that never came: the log download
|
||||
of any file over 1 KB hung (`test_compressed_body_declares_its_own_length`).
|
||||
|
||||
The browser server writes its logs into `<server root>/logs`
|
||||
(`tests/browser/workspace_diagnostics_fixture.py`), whose hooks seed records with fixed
|
||||
times, rotated copies, a link (POSIX) and a user's `.bak`, and roll the copies over; no
|
||||
test reads or deletes a real log folder.
|
||||
|
||||
`tests/browser/system-diagnostics-concept.spec.mjs` compares the section and its log
|
||||
cleanup review with the concept (dark and light, 1600x1100 and 390x844, no page scroll;
|
||||
both selects show their whole option at 320 wide). The fixture's `{mode: "concept"}` seed
|
||||
reproduces the concept's Logs sample: only its four buffer lines, a 1.4 MB active log with
|
||||
three rotated copies and console logging off, so those values are compared as the concept
|
||||
prints them. `mapDiagnosticsConcept` applies only D109 and D110, and `mapCleanupConcept`
|
||||
D100 (the shared `mapPlanConcept`) with D109's rule for this plan (no backup box, "Not
|
||||
needed: no library record changes", "Listed files"). The mutation check asserts that the
|
||||
unmapped difference is exactly the deviated paths. The comparison found that the Database
|
||||
line read `database_info.size_bytes`, a key the report does not have, so it always said
|
||||
"0 bytes"; it reads `database_size_bytes`, the key `get_database_info()` returns.
|
||||
|
||||
### Legacy System capability coverage
|
||||
|
||||
Every System capability of the legacy admin and gallery pages has an owner (#249 System
|
||||
S12, #329). Status is `covered` (a native control replaces it), `kept in legacy` (it stays
|
||||
on the legacy page, which keeps working) or `approved removal` (a ruling retired it). Native
|
||||
control is `` `<section>` · <role> · "<accessible name>" ``, `<section>` a System hash id
|
||||
and `<role>` one of button, combobox, switch, slider, spinbutton, textbox, group, status or
|
||||
term, followed by ` · in "<row title>"` when several rows of the section share the name.
|
||||
Every `covered` row names one, except the rows in `CONTROL_EXEMPTIONS`
|
||||
(`tools/vendor/legacy-coverage.mjs`), each with its reason: today only "Gallery per-image
|
||||
auto-tag / review", which the Images selection bar owns, not a System section. A row that
|
||||
is not `covered` names none (`—`). Test is the exact title of a test that runs, then its
|
||||
file. Anchors are line numbers on `main` at this PR.
|
||||
|
||||
`tests/vendor/legacy-coverage.test.mjs` fails on a row with any of these problems:
|
||||
- no owner, or a status outside the three;
|
||||
- a malformed Native control cell;
|
||||
- a covered row without a control (unless exempt), or a control on a row that is not
|
||||
covered;
|
||||
- a test title its file does not declare: `test(` or `it(` (`.only` and `.slow` count),
|
||||
or a `def test_` at function or method indentation;
|
||||
- a title whose test is only declared with `.skip`, `.fixme` or `.fail`.
|
||||
|
||||
A title that appears only in a comment or a Python docstring does not count. CI running
|
||||
those tests green is what proves them. `every covered legacy System capability has its
|
||||
native control` (workspace-system.spec) opens each covered row's section and finds the
|
||||
control by role and accessible name, inside its row when the cell names one. The shared
|
||||
"Review plan" of the per-operation review row and the Sources "Refresh" of the monitoring
|
||||
list stay unscoped: any row's control proves them.
|
||||
|
||||
| Capability | Legacy anchor | Owner | Status | Native control | Test |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| Image scan directories add/remove | `admin.html:377–390`; `admin.js:200,1273–1323` | S4 Sources | covered | `sources` · button · "+ Add source" | Add source validates on the server, then marks the host draft dirty; Save persists (workspace-system-sources.spec.mjs) |
|
||||
| Currently monitoring list + Refresh | `admin.html:392–400`; `admin.js:1032` | S4 Sources | covered | `sources` · button · "Refresh" | watching only with a live observer; restart pending otherwise (workspace-system-sources.spec.mjs) |
|
||||
| Monitoring enabled | `py/config.py:212,248` | S4 Sources | covered | `sources` · switch · "Watch for new outputs" | Watch for new outputs saves monitoring_enabled; a restart applies it (workspace-system-sources.spec.mjs) |
|
||||
| Search results auto-hide `result_timeout` | `admin.html:407–412` | S3 Appearance | covered | `appearance` · spinbutton · "Node search auto-hide (seconds)" | host fields round-trip through /settings with revision (workspace-system.spec.mjs) |
|
||||
| Web UI display mode | `admin.html:414–422` | S3 Appearance | covered | `appearance` · combobox · "Open from ComfyUI" | host fields round-trip through /settings with revision (workspace-system.spec.mjs) |
|
||||
| Infinite scrolling default | `admin.html:424–430`; `admin.js:689–727` | Legacy prompt list (README continuous browsing); read-only in the report | kept in legacy | — | shouldLoadMore asks for the next page only near the bottom with pages left (list-state.test.js) |
|
||||
| Worker threads | `admin.html:437–445` | S4 Sources | covered | `sources` · slider · "Worker threads" | worker range max is cpu_count (workspace-system-sources.spec.mjs) |
|
||||
| LoraManager badge, enable, path, trigger words, CivitAI key, clear key | `admin.html:448–490`; `admin.js:1052–1156` | S10 Integrations | covered | `integrations` · switch · "LoraManager integration" | LoRA rows show detected / enabled / cache-loaded separately (workspace-system-integrations.spec.mjs) |
|
||||
| LoRA data import with progress | `admin.html:492–498,517–541`; `admin.js:1157` | S10 Integrations | covered | `integrations` · button · "Review import" | Review import shows replacement, blocked items and network effects with the backup box; Start opens the job (D107) (workspace-system-integrations.spec.mjs) |
|
||||
| Statistics | `admin.html:159`; `admin.js:1627` | S2 Overview | covered | `overview` · button · "Explore" | statistics dialog lists runs by category and library totals (workspace-system.spec.mjs) |
|
||||
| Export components JSON `/export` | `admin.html:163`; `admin.js:1607`; `py/api/prompts.py:238` | S9 Backup & export | covered | `backups` · button · "Export all prompts JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Saved pairs export | `py/api/pairs.py:84` | S9 Backup & export | covered | `backups` · button · "Export saved pairs JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Library export | `py/api/library.py:223` | S9 Backup & export | covered | `backups` · button · "Export JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Diagnostics run | `admin.html:171,659`; `admin.js:1984` | S12 Diagnostics | covered | `diagnostics` · button · "Run checks" | Run checks opens the shared Health report (workspace-system-diagnostics.spec.mjs) |
|
||||
| Synthetic image-link test | `admin.html:662`; `admin.js:2101` | S12 Diagnostics | covered | `diagnostics` · button · "Create test link…" | Create test link confirms first and names the prompt (workspace-system-diagnostics.spec.mjs) |
|
||||
| Maintenance cleanup_duplicates, vacuum, cleanup_orphaned_images, prune_orphaned_prompts | `admin.html:692–748`; `admin.js:1834` | S6/S7 Maintenance | covered | `maintenance` · button · "Review plan" · in "Duplicate prompt text" | Apply reviewed operation closes, toasts, opens the job in Activity (workspace-system-maintenance.spec.mjs) |
|
||||
| Maintenance check_hash_duplicates, check_consistency | `admin.html:692–748`; `admin.js:1834` | S2 Health report (Maintenance notice, D123) | covered | `maintenance` · button · "Run health checks" | Run health checks opens Health report and updates metric 4 (workspace-system.spec.mjs) |
|
||||
| Maintenance statistics (database info) | `admin.html:692–748`; `admin.js:1834` | S12 Diagnostics | covered | `diagnostics` · term · "Database" | meta-grid shows real database and source status, the pinned Viewer.js version and the host renderers (workspace-system-diagnostics.spec.mjs) |
|
||||
| Maintenance select all / clear all / run | `admin.html:754–762`; `admin.js:1822–1833` | S7 per-operation review | covered | `maintenance` · button · "Review plan" | plan dialog shows the server's scope, protected, backup policy and media lines with the backup box (D100); Cancel writes nothing (workspace-system-maintenance.spec.mjs) |
|
||||
| Prompt cleanup `/cleanup` | `py/api/admin.py:620` | S6/S7 Maintenance | covered | `maintenance` · button · "Review plan" · in "Duplicate prompt text" | test_duplicate_prompts_plan_counts_the_legacy_groups (test_maintenance_plans.py) |
|
||||
| Database backup | `admin.html:179,824`; `admin.js:2139,2454` | S9 Backup & export | covered | `backups` · button · "Back up" | Back up downloads through the api client and revokes the object URL (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Restore from file | `admin.html:183,616–642`; `admin.js:2177–2248` | S8/S9 Backup & export | covered | `backups` · button · "Review restore…" | Review restore stages the chosen file and shows the real summary with the backup box (D105) (workspace-system-backups.spec.mjs) |
|
||||
| Metadata scan with progress, cancel, pill | `admin.html:187,805–865`; `admin.js:2336–2500` | S4 Sources | covered | `sources` · button · "Review scan" | Review scan opens the shared Review metadata scan with all sources (workspace-system-sources.spec.mjs) |
|
||||
| Rescan-required notice + dismiss | `admin.html:1367–1410`; `admin.js:991–1011` | S4 Sources | covered | `sources` · button · "Later" | notice Later persists the 3.2.0 dismissal (pm_update_notice_dismissed) without scanning (workspace-system-sources.spec.mjs) |
|
||||
| View logs with level and limit | `admin.html:195,885–907,974`; `admin.js:2579–2641` | S12 Logs | covered | `diagnostics` · combobox · "Level" | level filter is minimum-level over all six choices (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log statistics | `admin.html:921–941`; `admin.js:2512` | S12 Logs | covered | `diagnostics` · status · "Log statistics" | log statistics show entries, errors, warnings, files and size (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log files list and download | `admin.html:910,982–985`; `admin.js:2545,2642,2679` | S12 Logs | covered | `diagnostics` · button · "Download log" | each listed file downloads by its returned name and a link has no Download (workspace-system-diagnostics.spec.mjs) |
|
||||
| Clear/truncate logs | `admin.html:915`; `admin.js:2704` | S12 Logs | covered | `diagnostics` · button · "Clear all logs…" | Clear all logs states both counts, confirms first and reads everything again (workspace-system-diagnostics.spec.mjs) |
|
||||
| Rotated-log cleanup (new, #329) | — | S12 Logs | covered | `diagnostics` · button · "Review log cleanup" | Review log cleanup lists only rotated copies and keeps the rest (workspace-system-diagnostics.spec.mjs) |
|
||||
| Set log level (incl. CRITICAL), console logging | `admin.html:948–966`; `admin.js:2530,2729` | S12 Logs | covered | `diagnostics` · combobox · "Runtime log level" | runtime level (including CRITICAL) and console logging save and say they apply until the server restarts (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log file_logging / max_file_size / backup_count / buffer_size | `py/api/logging_routes.py:56,285–299` | S12 Logs (read-only) | covered | `diagnostics` · group · "Log files & retention" | each listed file downloads by its returned name and a link has no Download (workspace-system-diagnostics.spec.mjs) |
|
||||
| Auto-tag model list with status | `admin.html:1095–1148`; `admin.js:3641` | S11 Auto-tagging | covered | `tagging` · combobox · "Tagging model" | model options are the four real engines with size and the description shows live status (workspace-system-tagging.spec.mjs) |
|
||||
| Auto-tag model download | `admin.html:1413–1438`; `admin.js:3680–3747`; `gallery.js:3412` | S11 Auto-tagging | covered | `tagging` · button · "Download" | Download starts the job and Activity shows it without Cancel (workspace-system-tagging.spec.mjs) |
|
||||
| Unload model | `gallery.js:3480` | S11 Auto-tagging | covered | `tagging` · button · "Unload" | Unload is enabled only while a model is loaded, and a held gate's refusal names the operation (workspace-system-tagging.spec.mjs) |
|
||||
| Tag template, WD14 thresholds, already-tagged mode | `admin.html:1150–1198`; `admin.js:3748` | S11 Auto-tagging | covered | `tagging` · combobox · "Existing tags" | defaults save through Save preferences and the Tag image dialog uses them (workspace-system-tagging.spec.mjs) |
|
||||
| Batch auto-tag output images | `admin.html:191,1210,1279–1318`; `admin.js:3748–3840` | S11 Auto-tagging | covered | `tagging` · button · "Review" | Tag output images reviews the images, tags each one and never a prompt (#379) (workspace-system-tagging.spec.mjs) |
|
||||
| Batch review loop | `admin.html:1206,1218–1364`; `admin.js:3841–4172` | S11 + image-tagging plan | covered | `tagging` · button · "Preview tagging review" | Preview tagging review opens the Tag image dialog for the selected image (workspace-system-tagging.spec.mjs) |
|
||||
| Gallery per-image auto-tag / review | `gallery.html:601,605`; `gallery.js:3506,3674–3753` | Image-tagging plan (selection bar) | covered | — | Auto-tag selected with Apply automatically runs a job with the chosen model, thresholds and suggestion count (workspace-tagging-bulk.spec.mjs) |
|
||||
| Gallery thumbnail generation | `gallery.js:1852,1967` | S5/S7 | covered | `sources` · button · "Review options" · in "Thumbnail generation" | Start closes the dialog, toasts, and opens the job in Activity (workspace-system-sources.spec.mjs) |
|
||||
| Clear server thumbnails | `gallery.js:2175–2192` | S7 Maintenance | covered | `maintenance` · button · "Review options" · in "Thumbnail cache" | Thumbnail cache opens the S5 dialog (workspace-system-maintenance.spec.mjs) |
|
||||
| Browser cache clear | `gallery.js:2194–2207` | S7 Maintenance | covered | `maintenance` · button · "Review plan" · in "Metadata cache" | Metadata cache reviews and clears only owned browser entries when there are some (D102) (workspace-system-maintenance.spec.mjs) |
|
||||
| Refresh file listing | `gallery.js:2209` | S4 Sources | covered | `sources` · button · "Refresh" · in "Refresh files" | Refresh files calls /gallery/rescan and toasts its counts (workspace-system-sources.spec.mjs) |
|
||||
| Duplicate image scan, review, delete | `gallery.js:2234–2577` | S6/S7 Maintenance | covered | `maintenance` · button · "Find duplicates" | Find duplicates starts a job; its view links Review groups; Keep radios build a duplicate_files plan (D101) (workspace-system-maintenance.spec.mjs) |
|
||||
| Default view mode, image quality | `gallery-settings.js:26–27` | S3 Appearance | covered | `appearance` · combobox · "Default layout" | defaults apply to a new visit but not to the current view or a saved view (workspace-system.spec.mjs) |
|
||||
| Auto-load metadata, cache metadata, show file paths | `gallery-settings.js:31–33` | S3 Appearance | covered | `appearance` · switch · "Load metadata on selection" | metadata off shows an explicit load and loads once on request (workspace-system.spec.mjs) |
|
||||
| Video autoplay, mute, loop | `gallery-settings.js:40–42` | S3 Appearance | covered | `appearance` · switch · "Autoplay videos" | video defaults reach the viewer (workspace-system.spec.mjs) |
|
||||
| Image info on cards; sidebar collapsed | `gallery-settings.js:30,34` | S3 Appearance | covered | `appearance` · switch · "Show image details on cards" | added rows reach their consumers (workspace-system.spec.mjs) |
|
||||
| Reset gallery settings | `gallery.js:1813` | S3 Appearance | covered | `appearance` · button · "Reset" | reset restores defaults without touching other keys (workspace-system.spec.mjs) |
|
||||
| Lazy loading | `gallery-settings.js:25` | Legacy gallery (native always lazy) | kept in legacy | — | a legacy gallery Save keeps the workspace keys and every other stored key (gallery-settings.test.js) |
|
||||
| Results per page / grid columns / paged mode | `gallery-settings.js:28–29,35` | README continuous browsing | approved removal | — | numeric strings from <select> values are accepted and columns are clamped (gallery-settings.test.js) |
|
||||
| Startup thumbnail prompt | `gallery-settings.js:38–39`; `gallery.js:2578` | README; Overview thumbnail row | approved removal | — | thumbnail row reflects the status endpoint (workspace-system.spec.mjs) |
|
||||
| Client debug output `debugMode` | `gallery-settings.js:36` | Legacy gallery (S12-10) | kept in legacy | — | a legacy gallery Save keeps the workspace keys and every other stored key (gallery-settings.test.js) |
|
||||
| API timeout `apiTimeout` | `gallery-settings.js:37` | Legacy gallery (S12-10; read by no request) | kept in legacy | — | defaults are a complete, sane settings object (gallery-settings.test.js) |
|
||||
| config.json-only keys | `py/config.py:212–234,722–739`; `py/api/diagnostics_report.py:185` | S12 report (read-only) | covered | `diagnostics` · button · "Download diagnostic report" | test_report_lists_config_only_keys (test_api_diagnostics_report.py) |
|
||||
|
||||
### Legacy System UI removed by S13
|
||||
|
||||
System S13 (#330) points the legacy pages' System buttons at native System (a link that
|
||||
opens `workspace#/system/<id>` in a new tab) and removes the legacy copy of a capability
|
||||
only when its row above is `covered`. One row per removed capability: Capability is the
|
||||
exact Capability cell of its coverage row, Page is `admin` or `gallery`, Removed ids and
|
||||
Removed methods list what went (`—` for none), and Native test copies that row's Test
|
||||
cell. An id that stays on a System link is not listed, only its handler.
|
||||
|
||||
`tests/vendor/legacy-coverage.test.mjs` fails when a row cites a capability that is not
|
||||
`covered` or copies another test, when an id or method is listed twice for one page, or
|
||||
when a listed id is still on its page (`id="…"` in the markup or the script, a
|
||||
`getElementById`/`querySelector('#…')` lookup, an entry of a modal-id list) or a listed
|
||||
method is still defined or called (`.name(`). A deletion listed nowhere is caught by the
|
||||
kept-flow tests (`kept admin flows still work` in `legacy-system-links.spec.mjs` and the
|
||||
legacy page workflows), not by this check.
|
||||
|
||||
| Capability | Page | Removed ids | Removed methods | Native test |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Diagnostics run | admin | `#diagnosticsModal`, `#runDiagnosticsBtn`, `#diagnosticsContent` | `showDiagnosticsModal()`, `closeDiagnostics()`, `runDiagnostics()`, `renderDiagnostics()` | Run checks opens the shared Health report (workspace-system-diagnostics.spec.mjs) |
|
||||
| Synthetic image-link test | admin | `#testImageLinkBtn` | `testImageLink()` | Create test link confirms first and names the prompt (workspace-system-diagnostics.spec.mjs) |
|
||||
| View logs with level and limit | admin | `#logsModal`, `#logLevel`, `#logLimit`, `#refreshLogsBtn`, `#logsContainer` | `showLogsModal()`, `closeLogs()`, `refreshLogs()`, `displayLogs()` | level filter is minimum-level over all six choices (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log statistics | admin | `#logStatsTotal`, `#logStatsErrors`, `#logStatsWarnings`, `#logStatsLevel`, `#logStatsSize` | `loadLogStats()`, `formatBytes()` | log statistics show entries, errors, warnings, files and size (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log files list and download | admin | `#downloadLogsBtn`, `#logFilesList` | `loadLogFiles()`, `downloadLogs()`, `downloadLogFile()` | each listed file downloads by its returned name and a link has no Download (workspace-system-diagnostics.spec.mjs) |
|
||||
| Clear/truncate logs | admin | `#clearLogsBtn` | `clearLogs()` | Clear all logs states both counts, confirms first and reads everything again (workspace-system-diagnostics.spec.mjs) |
|
||||
| Set log level (incl. CRITICAL), console logging | admin | `#updateLogConfigBtn`, `#setLogLevel`, `#consoleLogging` | `loadLogConfig()`, `updateLogConfig()` | runtime level (including CRITICAL) and console logging save and say they apply until the server restarts (workspace-system-diagnostics.spec.mjs) |
|
||||
|
||||
### Browser cache ownership (#222)
|
||||
|
||||
The legacy gallery's Clear cache used to delete every localStorage key starting `pm_` or
|
||||
@@ -4520,17 +4139,8 @@ impact and command receipts report `image_count` / `images` /
|
||||
`(size, mtime_ns)` equals its verified fingerprint; an in-place rewrite that
|
||||
preserves both keeps the old set matching until the file is next observed. The
|
||||
Images Tags filter (a file matches when every tag is on its linked positive or
|
||||
its current image tag set) and `q` search find their candidates through the tag
|
||||
index (#379), never by streaming every image tag row: `image_tag_ids` reads the
|
||||
tag names that tag at least one image (an `EXISTS` on `idx_media_image_tags_tag`)
|
||||
and keeps a name when it equals a Tags filter tag exactly or contains `q`
|
||||
case-folded; `image_tag_locators` reads the files carrying those ids (`tag_id IN`,
|
||||
500 ids a chunk). Only the listed ones among them are stat'ed, and only their
|
||||
current records count; a linked positive's prompt tags still combine with the
|
||||
image tags under a Tags filter. `test_indexed_candidates_equal_the_full_stream`
|
||||
compares the lookup with the old full stream (kept in the test as the reference)
|
||||
and `test_tag_search_reads_image_tags_only_by_tag_id` traces every statement; the
|
||||
|
||||
its current image tag set) and `q` search stream the tagged records once and
|
||||
stat only the listed files whose records carry a filter tag or a `q` hit; the
|
||||
inspector's `image_tags` facet (`current` / `none` / `unavailable`) uses the
|
||||
inspection's own stable stat (`tests/test_api_workspace_images.py`
|
||||
`TestWorkspaceImageTags`, `tests/test_api_prompt_inspector.py`). The
|
||||
|
||||
+45
-135
@@ -19,11 +19,6 @@ links (``PromptDatabase._move_tag_links``).
|
||||
Cheap reads (spec amendment S1) treat a byte record as *current* only while
|
||||
the file's present ``(size, mtime_ns)`` equals the record's verified
|
||||
fingerprint; a file without a current record shows no image tags.
|
||||
|
||||
A write made from a tagging model's result passes ``mark_autotagged`` (#379):
|
||||
the byte record is then marked auto-tagged (``media_image_autotags``) in the
|
||||
same transaction, so a mark never exists without the write it records. Hand
|
||||
tags never mark.
|
||||
"""
|
||||
|
||||
import re
|
||||
@@ -39,7 +34,6 @@ from .annotation_schema import (
|
||||
MAX_LOCATOR_LENGTH,
|
||||
METADATA_TABLE,
|
||||
)
|
||||
from .autotag_schema import AUTOTAG_TABLE
|
||||
from .library_commands import _UUID, LibraryCommandError, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .vocabulary import validate_vocabulary_tag_name
|
||||
@@ -102,24 +96,12 @@ _TAGS_OF = (
|
||||
"JOIN tags t ON t.id=mit.tag_id WHERE mit.byte_id IN ({marks}) "
|
||||
"ORDER BY mit.byte_id,mit.position"
|
||||
)
|
||||
_MARK = f"INSERT OR IGNORE INTO {AUTOTAG_TABLE}(byte_id) VALUES(?)" # nosec B608
|
||||
_MARKED_OF = (
|
||||
f"SELECT byte_id FROM {AUTOTAG_TABLE} WHERE byte_id IN ({{marks}})" # nosec B608
|
||||
)
|
||||
_MARKED_LOCATORS = (
|
||||
f"SELECT DISTINCT f.locator FROM {AUTOTAG_TABLE} m " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.id=m.byte_id JOIN {FILES_TABLE} f ON f.id=b.file_id"
|
||||
)
|
||||
# Tag search (#379): the names that tag at least one image (the tag index
|
||||
# answers the EXISTS), then the locators carrying chosen names (by tag_id).
|
||||
_IMAGE_TAG_NAMES = (
|
||||
"SELECT t.id,t.name FROM tags t WHERE EXISTS " # nosec B608
|
||||
f"(SELECT 1 FROM {IMAGE_TAGS_TABLE} mit WHERE mit.tag_id=t.id)"
|
||||
)
|
||||
_TAG_LOCATORS = (
|
||||
f"SELECT DISTINCT f.locator FROM {IMAGE_TAGS_TABLE} mit " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.id=mit.byte_id "
|
||||
f"JOIN {FILES_TABLE} f ON f.id=b.file_id WHERE mit.tag_id IN ({{marks}})"
|
||||
_RECORDS = (
|
||||
"SELECT f.locator,b.id,b.size,b.mtime_ns,t.name " # nosec B608
|
||||
f"FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"LEFT JOIN {IMAGE_TAGS_TABLE} mit ON mit.byte_id=b.id "
|
||||
"LEFT JOIN tags t ON t.id=mit.tag_id "
|
||||
"ORDER BY f.locator,b.first_seen_at,b.id,mit.position"
|
||||
)
|
||||
|
||||
|
||||
@@ -319,17 +301,14 @@ class AnnotationOperations:
|
||||
expected_epoch: str,
|
||||
expected_tag_revision: int,
|
||||
tags: list,
|
||||
mark_autotagged: bool = False,
|
||||
) -> dict:
|
||||
"""Replace the set of these exact bytes, guarded by epoch and revision.
|
||||
|
||||
Creates the owner and byte record when the bytes are new (an explicit
|
||||
empty set is a real save). The identical set at the current revision,
|
||||
or one behind it (a lost-response retry), is ``replayed`` and writes
|
||||
no tag; any other revision mismatch is ``tag-conflict`` with the
|
||||
current set and no writes. ``mark_autotagged`` marks the bytes
|
||||
auto-tagged on a saved or replayed receipt (the Tag image dialog's
|
||||
save after Generate suggestions).
|
||||
nothing; any other revision mismatch is ``tag-conflict`` with the
|
||||
current set and no writes.
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
@@ -350,8 +329,6 @@ class AnnotationOperations:
|
||||
):
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
if mark_autotagged:
|
||||
conn.execute(_MARK, (byte_id,))
|
||||
return _receipt(epoch, sha256, current, revision, True)
|
||||
if expected_tag_revision != revision:
|
||||
raise AnnotationCommandError(
|
||||
@@ -360,8 +337,6 @@ class AnnotationOperations:
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
revision = self._write_image_tags(conn, byte_id, tags)
|
||||
if mark_autotagged:
|
||||
conn.execute(_MARK, (byte_id,))
|
||||
return _receipt(epoch, sha256, tags, revision, False)
|
||||
|
||||
def add_image_tags(
|
||||
@@ -373,22 +348,16 @@ class AnnotationOperations:
|
||||
mtime_ns: int,
|
||||
expected_epoch: str,
|
||||
tags: list,
|
||||
mark_autotagged: bool = False,
|
||||
) -> dict:
|
||||
"""Union ``tags`` into the set of these exact bytes, keeping stored spelling.
|
||||
|
||||
``added`` counts the names that were new (case-insensitively). A union
|
||||
over ``MAX_IMAGE_TAGS`` is ``tag-limit`` with the current set and no
|
||||
writes; a union that adds nothing to an existing record is ``replayed``.
|
||||
``mark_autotagged`` (a tagging model's result) also marks the bytes
|
||||
auto-tagged, replayed or not; only such a call may pass no tags (the
|
||||
model found none: the image is marked and its set is unchanged).
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
tags = normalize_image_tags(tags)
|
||||
if not tags and not mark_autotagged:
|
||||
raise ValueError("tags must name at least one tag")
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
now = utc_now_iso()
|
||||
@@ -406,8 +375,6 @@ class AnnotationOperations:
|
||||
epoch,
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
if mark_autotagged:
|
||||
conn.execute(_MARK, (byte_id,))
|
||||
if not created and not added:
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
@@ -436,41 +403,6 @@ class AnnotationOperations:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
@staticmethod
|
||||
def _validated_stats(stats: object) -> dict:
|
||||
if not isinstance(stats, dict):
|
||||
raise ValueError(
|
||||
"stats must map locators to (size, mtime_ns) integer pairs"
|
||||
)
|
||||
for locator, fingerprint in stats.items():
|
||||
_validate_fingerprint(locator, fingerprint)
|
||||
return stats
|
||||
|
||||
@staticmethod
|
||||
def _current_byte_records(conn: sqlite3.Connection, stats: dict) -> dict:
|
||||
"""``{locator: byte_id}`` of each locator's current byte record.
|
||||
|
||||
The one rule every cheap read uses: a record is current while its
|
||||
verified ``(size, mtime_ns)`` equals the file's present fingerprint
|
||||
in ``stats``; the most recently verified wins (ties on ``(verified_at,
|
||||
byte_id)``). A locator without a current record is left out. Lookups
|
||||
go in chunks of 500 locators.
|
||||
"""
|
||||
locators = list(stats)
|
||||
chosen: dict[str, tuple[tuple[str, str], str]] = {}
|
||||
for start in range(0, len(locators), _LOOKUP_CHUNK):
|
||||
chunk = locators[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
|
||||
_FINGERPRINTS.format(marks=marks), chunk
|
||||
):
|
||||
if (size, mtime_ns) != stats[locator]:
|
||||
continue
|
||||
key = (verified_at, byte_id)
|
||||
if locator not in chosen or key > chosen[locator][0]:
|
||||
chosen[locator] = (key, byte_id)
|
||||
return {locator: byte_id for locator, (_key, byte_id) in chosen.items()}
|
||||
|
||||
def current_image_tags(self, stats: dict) -> dict:
|
||||
"""Image tags of the files whose current ``(size, mtime_ns)`` is verified.
|
||||
|
||||
@@ -479,11 +411,29 @@ class AnnotationOperations:
|
||||
exactly that verified fingerprint; the most recently verified record
|
||||
wins when several do. Lookups go in chunks of 500 locators.
|
||||
"""
|
||||
stats = self._validated_stats(stats)
|
||||
if not isinstance(stats, dict):
|
||||
raise ValueError(
|
||||
"stats must map locators to (size, mtime_ns) integer pairs"
|
||||
)
|
||||
for locator, fingerprint in stats.items():
|
||||
_validate_fingerprint(locator, fingerprint)
|
||||
locators = list(stats)
|
||||
chosen: dict[str, tuple[tuple[str, str], str]] = {}
|
||||
result: dict[str, list[str]] = {}
|
||||
with self._annotation_read() as conn:
|
||||
current = self._current_byte_records(conn, stats)
|
||||
owners = {byte_id: locator for locator, byte_id in current.items()}
|
||||
result: dict[str, list[str]] = {locator: [] for locator in current}
|
||||
for start in range(0, len(locators), _LOOKUP_CHUNK):
|
||||
chunk = locators[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
|
||||
_FINGERPRINTS.format(marks=marks), chunk
|
||||
):
|
||||
if (size, mtime_ns) != stats[locator]:
|
||||
continue
|
||||
key = (verified_at, byte_id)
|
||||
if locator not in chosen or key > chosen[locator][0]:
|
||||
chosen[locator] = (key, byte_id)
|
||||
owners = {byte_id: locator for locator, (_key, byte_id) in chosen.items()}
|
||||
result = {locator: [] for locator in chosen}
|
||||
byte_ids = list(owners)
|
||||
for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
|
||||
chunk = byte_ids[start : start + _LOOKUP_CHUNK]
|
||||
@@ -492,60 +442,20 @@ class AnnotationOperations:
|
||||
result[owners[byte_id]].append(name)
|
||||
return result
|
||||
|
||||
def autotag_state(self, stats: dict) -> dict:
|
||||
"""``{locator: bool}``: is each file's current byte record auto-tagged?
|
||||
def iter_image_tag_records(self) -> Iterator[tuple[str, int, int, list[str]]]:
|
||||
"""Stream every byte record as ``(locator, size, mtime_ns, tags)``.
|
||||
|
||||
``stats`` as for :meth:`current_image_tags`, whose rule picks the
|
||||
current record; a locator without one is False. One read snapshot.
|
||||
One SELECT ordered by locator, then by the bytes' first sighting, with
|
||||
each set's names in position order.
|
||||
"""
|
||||
stats = self._validated_stats(stats)
|
||||
with self._annotation_read() as conn:
|
||||
current = self._current_byte_records(conn, stats)
|
||||
byte_ids = list(dict.fromkeys(current.values()))
|
||||
marked: set[str] = set()
|
||||
for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
|
||||
chunk = byte_ids[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
marked.update(
|
||||
row[0]
|
||||
for row in conn.execute(_MARKED_OF.format(marks=marks), chunk)
|
||||
)
|
||||
return {locator: current.get(locator) in marked for locator in stats}
|
||||
|
||||
def autotagged_locators(self) -> set:
|
||||
"""Locators with at least one auto-tagged byte record, current or not.
|
||||
|
||||
No fingerprint is checked (the Tag output images review's estimate).
|
||||
"""
|
||||
with self._annotation_read() as conn:
|
||||
return {row[0] for row in conn.execute(_MARKED_LOCATORS)}
|
||||
|
||||
def image_tag_ids(self, wanted, query: str) -> list[int]:
|
||||
"""Ids of the tag names on at least one image that a search wants.
|
||||
|
||||
A name counts when it is in ``wanted`` exactly (the Tags filter) or
|
||||
contains ``query`` case-folded (the text search; ``query`` is
|
||||
already folded). One read of the names that tag an image.
|
||||
"""
|
||||
wanted = set(wanted)
|
||||
with self._annotation_read() as conn:
|
||||
return [
|
||||
tag_id
|
||||
for tag_id, name in conn.execute(_IMAGE_TAG_NAMES)
|
||||
if name in wanted or (query and query in name.casefold())
|
||||
]
|
||||
|
||||
def image_tag_locators(self, tag_ids) -> set[str]:
|
||||
"""Locators with any byte record (current or not) carrying one of
|
||||
``tag_ids``, through the tag index, in chunks of 500 ids."""
|
||||
ids = list(dict.fromkeys(tag_ids))
|
||||
found: set[str] = set()
|
||||
with self._annotation_read() as conn:
|
||||
for start in range(0, len(ids), _LOOKUP_CHUNK):
|
||||
chunk = ids[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
found.update(
|
||||
row[0]
|
||||
for row in conn.execute(_TAG_LOCATORS.format(marks=marks), chunk)
|
||||
)
|
||||
return found
|
||||
conn = self.model.get_connection()
|
||||
current: tuple | None = None
|
||||
for locator, byte_id, size, mtime_ns, name in conn.execute(_RECORDS):
|
||||
if current is None or current[0] != byte_id:
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
current = (byte_id, (locator, size, mtime_ns, []))
|
||||
if name is not None:
|
||||
current[1][3].append(name)
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
"""Auto-tagged marks (#379): which image bytes a tagging model has handled.
|
||||
|
||||
One table, ``media_image_autotags``: one row per byte record of the image
|
||||
annotation group (``media_annotation_bytes``) means "a tagging model looked at
|
||||
these exact bytes and its result was stored", or that the bytes were found to
|
||||
be untaggable. "Skip already tagged" skips an image whose current byte record
|
||||
is marked; hand tags never mark. Rows are written only by the image tag store
|
||||
(``database/annotations.py``), in the transaction of the tag write they
|
||||
record.
|
||||
|
||||
The group installs like ``image_favorites``: optional in old backups,
|
||||
required once a database is current, and never backfilled. Like the Needs
|
||||
review group it is valid only beside a valid image annotation group, because
|
||||
every mark cites one byte record (``ON DELETE CASCADE``). Validation reads
|
||||
only: the catalog compared after whitespace compaction, the columns and the
|
||||
foreign key through the PRAGMAs, then one bounded value pass and the foreign
|
||||
key check.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
from .annotation_schema import BYTES_TABLE
|
||||
from .annotation_schema import TABLES as ANNOTATION_TABLES
|
||||
from .annotation_schema import _catalog as _annotation_catalog
|
||||
from .annotation_schema import _uuid, annotation_schema_error
|
||||
|
||||
AUTOTAG_TABLE = "media_image_autotags"
|
||||
_DDL = (
|
||||
f"CREATE TABLE {AUTOTAG_TABLE} (byte_id TEXT PRIMARY KEY "
|
||||
f"REFERENCES {BYTES_TABLE}(id) ON DELETE CASCADE) WITHOUT ROWID"
|
||||
)
|
||||
# (cid, name, declared type, not null, default, primary key position): the
|
||||
# key of a WITHOUT ROWID table is implicitly NOT NULL.
|
||||
_COLUMNS = [(0, "byte_id", "TEXT", 1, None, 1)]
|
||||
# (column, parent table, parent column, on delete)
|
||||
_FOREIGN_KEYS = {("byte_id", BYTES_TABLE, "id", "CASCADE")}
|
||||
|
||||
NEEDS_ANNOTATIONS = "Image auto-tag marks need the image annotation tables"
|
||||
INVALID_SCHEMA = "Image auto-tag marks have an invalid schema"
|
||||
INVALID_VALUES = "Image auto-tag marks contain invalid values"
|
||||
MISSING = "Missing image auto-tag marks schema"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql).lower()
|
||||
|
||||
|
||||
def _catalog(conn: sqlite3.Connection) -> list:
|
||||
return conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name=? OR tbl_name=?",
|
||||
(AUTOTAG_TABLE, AUTOTAG_TABLE),
|
||||
).fetchall()
|
||||
|
||||
|
||||
def autotag_without_annotations_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
"""Catalog only: the marks table present without its annotation tables.
|
||||
|
||||
Startup asks this before the annotation group installs, which would make
|
||||
a marks table copied without it look valid. It reads no rows.
|
||||
"""
|
||||
if not _catalog(conn):
|
||||
return None
|
||||
present = {
|
||||
name for kind, name, _sql in _annotation_catalog(conn) if kind == "table"
|
||||
}
|
||||
if present != set(ANNOTATION_TABLES):
|
||||
return NEEDS_ANNOTATIONS
|
||||
return None
|
||||
|
||||
|
||||
def autotag_schema_error(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
allow_missing: bool = True,
|
||||
annotations_valid: bool = False,
|
||||
) -> Optional[str]:
|
||||
"""Reject an unsupported marks table or invalid marks; reads only.
|
||||
|
||||
A database without the table is acceptable for backups that predate it
|
||||
unless ``allow_missing`` is false. A present table requires a present,
|
||||
valid image annotation group; ``annotations_valid`` says the caller has
|
||||
just validated it (startup's annotation install), so only its presence is
|
||||
checked again.
|
||||
"""
|
||||
objects = _catalog(conn)
|
||||
if not objects:
|
||||
return None if allow_missing else MISSING
|
||||
if annotations_valid:
|
||||
if autotag_without_annotations_error(conn):
|
||||
return NEEDS_ANNOTATIONS
|
||||
elif annotation_schema_error(conn, allow_missing=False):
|
||||
return NEEDS_ANNOTATIONS
|
||||
actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
|
||||
if actual != {("table", AUTOTAG_TABLE, _compact(_DDL))}:
|
||||
return INVALID_SCHEMA
|
||||
columns = [
|
||||
(row[0], row[1], row[2].upper(), row[3], row[4], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({AUTOTAG_TABLE})")
|
||||
]
|
||||
if columns != _COLUMNS:
|
||||
return INVALID_SCHEMA
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({AUTOTAG_TABLE})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS:
|
||||
return INVALID_SCHEMA
|
||||
# Only this module's constants are interpolated into the SQL below.
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {AUTOTAG_TABLE} " # nosec B608
|
||||
f"WHERE NOT ({_uuid('byte_id')}) LIMIT 1"
|
||||
).fetchone():
|
||||
return INVALID_VALUES
|
||||
if conn.execute(f"PRAGMA foreign_key_check({AUTOTAG_TABLE})").fetchone():
|
||||
return INVALID_VALUES
|
||||
return None
|
||||
|
||||
|
||||
def migrate_autotag_state(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table atomically; validate before and after creating it.
|
||||
|
||||
Runs after ``migrate_annotation_state`` (which has just validated the
|
||||
annotation group). Creates one empty table: no row is read or copied.
|
||||
"""
|
||||
conn.execute("SAVEPOINT media_autotag_schema")
|
||||
try:
|
||||
error = autotag_schema_error(conn, annotations_valid=True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
error = autotag_schema_error(conn, allow_missing=False, annotations_valid=True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE media_autotag_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO media_autotag_schema")
|
||||
conn.execute("RELEASE media_autotag_schema")
|
||||
raise
|
||||
@@ -23,11 +23,6 @@ from . import restore_stage
|
||||
from .library_schema import library_schema_error, migrate_library_state
|
||||
from .image_favorites import image_favorites_schema_error, migrate_image_favorites
|
||||
from .annotation_schema import annotation_schema_error, migrate_annotation_state
|
||||
from .autotag_schema import (
|
||||
autotag_schema_error,
|
||||
autotag_without_annotations_error,
|
||||
migrate_autotag_state,
|
||||
)
|
||||
from .review_schema import (
|
||||
migrate_review_state,
|
||||
review_schema_error,
|
||||
@@ -343,13 +338,8 @@ class PromptModel:
|
||||
review_error = review_without_annotations_error(conn)
|
||||
if review_error:
|
||||
raise sqlite3.DatabaseError(review_error)
|
||||
# The auto-tag marks cite annotation bytes the same way (#379).
|
||||
autotag_error = autotag_without_annotations_error(conn)
|
||||
if autotag_error:
|
||||
raise sqlite3.DatabaseError(autotag_error)
|
||||
migrate_annotation_state(conn)
|
||||
migrate_review_state(conn)
|
||||
migrate_autotag_state(conn)
|
||||
# Last: the older generated_images rebuilds run only before file_path.
|
||||
migrate_workflow_documents(conn)
|
||||
|
||||
@@ -1235,11 +1225,6 @@ class PromptModel:
|
||||
)
|
||||
if review_error:
|
||||
return False, review_error
|
||||
autotag_error = autotag_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if autotag_error:
|
||||
return False, autotag_error
|
||||
documents_error = workflow_documents_schema_error(
|
||||
conn, allow_missing=not require_library_state, documents=documents
|
||||
)
|
||||
|
||||
@@ -1808,6 +1808,37 @@ class PromptDatabase(
|
||||
reader = DocumentReader(conn)
|
||||
return [self._image_row_to_dict(row, reader) for row in cursor]
|
||||
|
||||
def autotag_batch_candidates(self) -> List[Dict[str, Any]]:
|
||||
"""The images a Tag output images batch looks at (#328).
|
||||
|
||||
``{id, image_path, prompt_id, prompt_tags}`` for every image with a
|
||||
stored path that is linked to a prompt, newest first (the
|
||||
``get_all_images`` order, ties newest row first). Never reads
|
||||
``workflow_data`` or the other documents: a batch needs none of them.
|
||||
"""
|
||||
sql = (
|
||||
"SELECT gi.id, gi.image_path, gi.prompt_id, "
|
||||
"(SELECT GROUP_CONCAT(t.name, '|||') FROM prompt_tags pt "
|
||||
"JOIN tags t ON pt.tag_id = t.id WHERE pt.prompt_id = gi.prompt_id) "
|
||||
"AS prompt_tags "
|
||||
"FROM generated_images gi "
|
||||
"INNER JOIN prompts p ON gi.prompt_id = p.id "
|
||||
"WHERE gi.image_path IS NOT NULL AND gi.image_path != '' "
|
||||
"ORDER BY gi.generation_time DESC, gi.id DESC"
|
||||
)
|
||||
with self.model.get_connection() as conn:
|
||||
return [
|
||||
{
|
||||
"id": row["id"],
|
||||
"image_path": row["image_path"],
|
||||
"prompt_id": row["prompt_id"],
|
||||
"prompt_tags": (
|
||||
row["prompt_tags"].split("|||") if row["prompt_tags"] else []
|
||||
),
|
||||
}
|
||||
for row in conn.execute(sql)
|
||||
]
|
||||
|
||||
def get_all_images(
|
||||
self,
|
||||
limit: int = 0,
|
||||
|
||||
+193
-7
@@ -295,10 +295,6 @@ async def _gzip_middleware(request, handler):
|
||||
if len(compressed) >= len(body):
|
||||
return response
|
||||
|
||||
# A length the handler declared is the uncompressed one: drop it so the
|
||||
# compressed body's own length is sent (#329: a stale length made the
|
||||
# browser wait for bytes that never came).
|
||||
response.headers.pop("Content-Length", None)
|
||||
response.body = compressed
|
||||
response.headers["Content-Encoding"] = "gzip"
|
||||
response.headers["Vary"] = "Accept-Encoding"
|
||||
@@ -363,9 +359,6 @@ class PromptManagerAPI(
|
||||
# Private media captures of the annotation routes (py/api/annotations.py),
|
||||
# created on first use with their restore subscription.
|
||||
self._captures = None
|
||||
# Capture tokens Generate suggestions ran on: a save of one marks the
|
||||
# image auto-tagged (#379).
|
||||
self._suggested_tokens = set()
|
||||
self._captures_unsubscribe = None
|
||||
# Set by close_annotation_captures (shutdown): no store is recreated.
|
||||
self._captures_closed = False
|
||||
@@ -1107,3 +1100,196 @@ class PromptManagerAPI(
|
||||
result["parameters"][field] = metadata[field]
|
||||
|
||||
return result
|
||||
|
||||
def _extract_readable_prompt(self, parsed_data):
|
||||
"""Extract human-readable prompt text from ComfyUI/A1111 data."""
|
||||
|
||||
def safe_to_string(value):
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
elif isinstance(value, list):
|
||||
return " ".join(str(item) for item in value if item)
|
||||
elif value is not None:
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
# First check if we already extracted a positive prompt (A1111 format)
|
||||
if parsed_data.get("positive_prompt"):
|
||||
return parsed_data["positive_prompt"]
|
||||
|
||||
# Check if prompt is already a string
|
||||
if isinstance(parsed_data.get("prompt"), str):
|
||||
return parsed_data["prompt"]
|
||||
|
||||
# Check if prompt is a simple value that can be converted
|
||||
if parsed_data.get("prompt") and not isinstance(
|
||||
parsed_data.get("prompt"), dict
|
||||
):
|
||||
return safe_to_string(parsed_data["prompt"])
|
||||
|
||||
prompt_data = parsed_data.get("prompt")
|
||||
if isinstance(prompt_data, dict):
|
||||
# Use the enhanced logic from parse-metadata.py
|
||||
positive_prompt = self._extract_positive_prompt_from_comfyui_data(
|
||||
prompt_data
|
||||
)
|
||||
if positive_prompt:
|
||||
return positive_prompt
|
||||
|
||||
# Check workflow data if available
|
||||
workflow_data = parsed_data.get("workflow")
|
||||
if isinstance(workflow_data, dict):
|
||||
positive_prompt = self._extract_positive_prompt_from_comfyui_data(
|
||||
workflow_data
|
||||
)
|
||||
if positive_prompt:
|
||||
return positive_prompt
|
||||
|
||||
# Check parameters for positive prompt
|
||||
if parsed_data.get("parameters", {}).get("positive"):
|
||||
return safe_to_string(parsed_data["parameters"]["positive"])
|
||||
|
||||
return None
|
||||
|
||||
def _get_node_inputs(self, node):
|
||||
"""Safely get inputs from a node, handling both dict and list formats.
|
||||
|
||||
Old format: inputs is a dict with direct key-value pairs
|
||||
inputs = {"text": "my prompt", "seed": 123}
|
||||
|
||||
New format: inputs is a list of connection objects
|
||||
inputs = [
|
||||
{"name": "text", "type": "STRING", "link": null,
|
||||
"widget": {"name": "text"}},
|
||||
{"name": "clip", "type": "CLIP", "link": 11}
|
||||
]
|
||||
"""
|
||||
if not isinstance(node, dict):
|
||||
return {}
|
||||
|
||||
inputs = node.get("inputs", {})
|
||||
|
||||
# If inputs is already a dict, return it
|
||||
if isinstance(inputs, dict):
|
||||
return inputs
|
||||
|
||||
# If inputs is a list, convert to dict format
|
||||
if isinstance(inputs, list):
|
||||
inputs_dict = {}
|
||||
for input_item in inputs:
|
||||
if isinstance(input_item, dict) and "name" in input_item:
|
||||
name = input_item["name"]
|
||||
inputs_dict[name] = input_item
|
||||
return inputs_dict
|
||||
|
||||
return {}
|
||||
|
||||
def _find_text_in_node(self, node):
|
||||
"""Try to find text content in a node using various strategies.
|
||||
|
||||
Handles both old and new workflow formats.
|
||||
"""
|
||||
if not isinstance(node, dict):
|
||||
return None
|
||||
|
||||
# Strategy 1: Check normalized inputs for 'text' field
|
||||
inputs = self._get_node_inputs(node)
|
||||
if "text" in inputs and isinstance(inputs["text"], str):
|
||||
return inputs["text"]
|
||||
|
||||
# Strategy 2: For text encoder nodes, check widgets_values
|
||||
class_type = node.get("class_type", node.get("type", ""))
|
||||
text_encoder_types = [
|
||||
"CLIPTextEncode",
|
||||
"CLIPTextEncodeSDXL",
|
||||
"CLIPTextEncodeSDXLRefiner",
|
||||
"CLIPTextEncodeFlux",
|
||||
"PromptManager",
|
||||
"PromptManagerText",
|
||||
"BNK_CLIPTextEncoder",
|
||||
"Text Encoder",
|
||||
"CLIP Text Encode",
|
||||
]
|
||||
|
||||
if any(
|
||||
encoder_type.lower() in class_type.lower()
|
||||
for encoder_type in text_encoder_types
|
||||
):
|
||||
widgets_values = node.get("widgets_values", [])
|
||||
if widgets_values and len(widgets_values) > 0:
|
||||
if isinstance(widgets_values[0], str) and widgets_values[0].strip():
|
||||
return widgets_values[0]
|
||||
|
||||
return None
|
||||
|
||||
def _extract_positive_prompt_from_comfyui_data(self, data):
|
||||
"""Extract positive prompt from ComfyUI data.
|
||||
|
||||
Handles both old and new formats.
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
|
||||
# Build nodes dictionary
|
||||
nodes_by_id = {}
|
||||
if "nodes" in data:
|
||||
# Handle nodes array format
|
||||
for node in data["nodes"]:
|
||||
if isinstance(node, dict):
|
||||
nid = node.get("id")
|
||||
if nid is not None:
|
||||
nodes_by_id[nid] = node
|
||||
else:
|
||||
# Handle flat dictionary format (node_id -> node_data)
|
||||
for nid_str, node in data.items():
|
||||
try:
|
||||
nid = int(nid_str)
|
||||
except (ValueError, TypeError):
|
||||
nid = nid_str
|
||||
if isinstance(node, dict):
|
||||
if "id" in node:
|
||||
nid = node["id"]
|
||||
nodes_by_id[nid] = node
|
||||
|
||||
if not nodes_by_id:
|
||||
return None
|
||||
|
||||
# First, try to find positive/negative connection pattern
|
||||
pos_id = None
|
||||
for node in nodes_by_id.values():
|
||||
if isinstance(node, dict):
|
||||
inputs = self._get_node_inputs(node)
|
||||
if "positive" in inputs and "negative" in inputs:
|
||||
try:
|
||||
pos_input = inputs["positive"]
|
||||
if isinstance(pos_input, list) and len(pos_input) > 0:
|
||||
pos_id = int(pos_input[0])
|
||||
break
|
||||
except (ValueError, TypeError, IndexError):
|
||||
continue
|
||||
|
||||
# Get text from the positive node
|
||||
if pos_id is not None and pos_id in nodes_by_id:
|
||||
text_val = self._find_text_in_node(nodes_by_id[pos_id])
|
||||
if text_val:
|
||||
return text_val
|
||||
|
||||
# Fallback: find any text encoder node with text content
|
||||
text_nodes = []
|
||||
for node in nodes_by_id.values():
|
||||
if isinstance(node, dict):
|
||||
text_val = self._find_text_in_node(node)
|
||||
if text_val:
|
||||
# Try to determine if this is positive or negative
|
||||
node_title = node.get("title", "").lower()
|
||||
if "neg" not in node_title and "negative" not in node_title:
|
||||
# Prioritize non-negative prompts
|
||||
text_nodes.insert(0, text_val)
|
||||
else:
|
||||
text_nodes.append(text_val)
|
||||
|
||||
# Return the first positive-looking prompt
|
||||
if text_nodes:
|
||||
return text_nodes[0]
|
||||
|
||||
return None
|
||||
|
||||
+8
-19
@@ -47,7 +47,7 @@ try:
|
||||
extract_generation_params,
|
||||
parse_page_json,
|
||||
)
|
||||
from ...utils.prompt_pair_resolver import resolve_prompt_pair, sampler_less_prompt
|
||||
from ...utils.prompt_pair_resolver import resolve_prompt_pair
|
||||
from ...utils.review_capture import reviewable
|
||||
except ImportError:
|
||||
from database.pair_ingestion import blank_positive, ingest_resolved_pair
|
||||
@@ -62,7 +62,7 @@ except ImportError:
|
||||
extract_generation_params,
|
||||
parse_page_json,
|
||||
)
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair, sampler_less_prompt
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair
|
||||
from utils.review_capture import reviewable
|
||||
|
||||
|
||||
@@ -1754,24 +1754,13 @@ class AdminRoutesMixin:
|
||||
canonical.get("parameters"), str
|
||||
):
|
||||
return resolution["state"]
|
||||
elif canonical.get("prompt"):
|
||||
# No node takes a positive or negative input (a guider or wrapper
|
||||
# graph): the one wired text, never a guess (#252).
|
||||
found = sampler_less_prompt(canonical["prompt"])
|
||||
prompt_text = found["text"]
|
||||
if prompt_text is None and isinstance(canonical.get("parameters"), str):
|
||||
# A metadata saver also wrote A1111 text: its positive prompt.
|
||||
parsed = self._parse_comfyui_prompt(metadata)["positive_prompt"]
|
||||
prompt_text = parsed.strip() if parsed and parsed.strip() else None
|
||||
if prompt_text is None:
|
||||
linked = self._link_recorded_run(
|
||||
media_file, canonical, detailed, history
|
||||
)
|
||||
if linked is not None:
|
||||
return linked
|
||||
return "unresolved" if found["state"] == "ambiguous" else "found"
|
||||
else:
|
||||
prompt_text = self._parse_comfyui_prompt(metadata)["positive_prompt"]
|
||||
parsed = self._parse_comfyui_prompt(metadata)
|
||||
if not (parsed.get("prompt") or parsed.get("parameters")):
|
||||
return None
|
||||
prompt_text = self._extract_readable_prompt(parsed)
|
||||
if prompt_text is not None and not isinstance(prompt_text, str):
|
||||
prompt_text = str(prompt_text)
|
||||
if not (prompt_text and prompt_text.strip()):
|
||||
return "found"
|
||||
prompt_text = prompt_text.strip()
|
||||
|
||||
@@ -296,7 +296,6 @@ class AnnotationRoutesMixin:
|
||||
self._captures_closed = True
|
||||
unsubscribe, self._captures_unsubscribe = self._captures_unsubscribe, None
|
||||
store, self._captures = self._captures, None
|
||||
self._suggested_tokens = set()
|
||||
if unsubscribe is not None:
|
||||
unsubscribe()
|
||||
if store is not None:
|
||||
@@ -516,7 +515,6 @@ class AnnotationRoutesMixin:
|
||||
store = self._captures
|
||||
if store is not None and _TOKEN.fullmatch(token) is not None:
|
||||
store.release(token)
|
||||
self._suggested_tokens.discard(token)
|
||||
return web.json_response(
|
||||
{"success": True}, headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
@@ -566,9 +564,6 @@ class AnnotationRoutesMixin:
|
||||
return _coded("capture-expired", _OTHER_FILE)
|
||||
if facts.sha256 != capture.sha256:
|
||||
return _coded("file-changed")
|
||||
# A save after Generate suggestions ran on this capture marks the
|
||||
# image auto-tagged (#379), so a Skip batch leaves the review alone.
|
||||
suggested = token in self._suggested_tokens
|
||||
try:
|
||||
receipt = await self._run_in_executor(
|
||||
self.db.replace_image_tags,
|
||||
@@ -579,7 +574,6 @@ class AnnotationRoutesMixin:
|
||||
expected_epoch=expected_epoch,
|
||||
expected_tag_revision=revision,
|
||||
tags=tags,
|
||||
mark_autotagged=suggested,
|
||||
)
|
||||
except AnnotationCommandError as exc:
|
||||
current = exc.current or {}
|
||||
@@ -596,23 +590,10 @@ class AnnotationRoutesMixin:
|
||||
except Exception:
|
||||
self.logger.exception("Image tag save failed")
|
||||
return _error(500, "save-failed", "The tags could not be saved.")
|
||||
# Dropped only after a saved or replayed receipt: a retry after a
|
||||
# tag-conflict still marks.
|
||||
self._suggested_tokens.discard(token)
|
||||
return web.json_response(
|
||||
{"success": True, **receipt}, headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
|
||||
def _remember_suggested(self, store, token: str) -> None:
|
||||
"""Note that Generate ran on *token*; forget tokens whose capture is
|
||||
gone. ``expires_in`` does not reset a capture's idle clock."""
|
||||
self._suggested_tokens = {
|
||||
kept
|
||||
for kept in self._suggested_tokens
|
||||
if store.expires_in(kept) is not None
|
||||
}
|
||||
self._suggested_tokens.add(token)
|
||||
|
||||
# -- suggest ---------------------------------------------------------------
|
||||
|
||||
async def suggest_image_tags(self, request: web.Request) -> web.Response:
|
||||
@@ -694,7 +675,6 @@ class AnnotationRoutesMixin:
|
||||
finally:
|
||||
if not started:
|
||||
store.release_use(token)
|
||||
self._remember_suggested(store, token)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
|
||||
+560
-572
File diff suppressed because it is too large
Load Diff
+33
-464
@@ -2,9 +2,7 @@
|
||||
|
||||
``POST /prompt_manager/jobs/image-tags`` starts one ``image-tags`` job over
|
||||
1..500 listed images, named by the listing identity ``{root_index,
|
||||
root_scope, path}``, or over every image a listing ``filter`` admits (at most
|
||||
10,000, resolved at start in the listing's order; "Select all N matching",
|
||||
#392). The job is record-changing (it writes image tag sets and
|
||||
root_scope, path}``. The job is record-changing (it writes image tag sets and
|
||||
new vocabulary names), so it never runs beside the metadata scan.
|
||||
|
||||
- **Add tags** (``mode: "add"``) hashes each current file and unions a fixed
|
||||
@@ -20,21 +18,13 @@ the suggestion worker and the storage call ``add_image_tags``. Files are
|
||||
handled one at a time and the job stops at a file boundary when cancelled,
|
||||
keeping what it already wrote. Rows and log lines carry root-relative paths
|
||||
and public root labels only.
|
||||
|
||||
Auto-tag marks each image it ran the model on as auto-tagged (#379), with the
|
||||
union of its tags. The ``autotag`` job ("Tag output images", started by
|
||||
``POST /jobs/autotag``) runs the same per-file loop over every still image the
|
||||
output folders list (:meth:`ImageTagJobsMixin._run_autotag_batch`): image
|
||||
tags only, never a prompt tag or prompt text.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import functools
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import NamedTuple
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -55,24 +45,10 @@ from .annotations import (
|
||||
_submit,
|
||||
_target,
|
||||
)
|
||||
from .autotag_routes import BATCH_CANCELLED, MODEL_BUSY_ERROR, model_busy_response
|
||||
from .images import (
|
||||
MAX_LISTING_SELECTION,
|
||||
ListingSourceChanged,
|
||||
_current_fingerprint,
|
||||
_listing_posix,
|
||||
_public_path,
|
||||
parse_listing_filter,
|
||||
)
|
||||
from .autotag_routes import model_busy_response
|
||||
from .images import _public_path
|
||||
from .job_routes import busy_response
|
||||
from .jobs import (
|
||||
AUTOTAG_BATCH_JOB_KIND,
|
||||
AUTOTAG_GROUP_NAMES,
|
||||
IMAGE_TAG_GROUP_NAMES,
|
||||
IMAGE_TAG_JOB_KIND,
|
||||
JobBusy,
|
||||
)
|
||||
from .maintenance import disk_plan, short_space_message
|
||||
from .jobs import IMAGE_TAG_GROUP_NAMES, IMAGE_TAG_JOB_KIND, JobBusy
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
@@ -99,11 +75,6 @@ except ImportError:
|
||||
from utils.video_metadata import is_video_path
|
||||
|
||||
MAX_JOB_TARGETS = 500
|
||||
# A filter start resolves on the server, so the request body does not bound it.
|
||||
MAX_FILTER_TARGETS = MAX_LISTING_SELECTION
|
||||
NO_IMAGES = "No images match this view."
|
||||
FILTER_LISTING_FAILED = "The matching images could not be listed. Try again."
|
||||
TOO_MANY_IMAGES = "Narrow the view to at most {cap:,} images."
|
||||
# 500 maximal targets plus the options: a path of MAX_TARGET_PATH code points
|
||||
# takes at most 6 bytes each on the wire (a \uXXXX escape; raw UTF-8 is at most
|
||||
# 4), and the root index and scope fit in 256. The annotation routes' 64 KiB
|
||||
@@ -117,10 +88,9 @@ ADD_TITLE = "Add tags to images"
|
||||
AUTO_TITLE = "Auto-tag images"
|
||||
GATE_OWNER_KIND = "image-tag-job"
|
||||
|
||||
# Each mode's fields besides the images, named by ``targets`` or ``filter``.
|
||||
_FIELDS = {
|
||||
"add": frozenset({"mode", "tags"}),
|
||||
"auto": frozenset({"mode", "model_type", "options"}),
|
||||
"add": frozenset({"mode", "targets", "tags"}),
|
||||
"auto": frozenset({"mode", "targets", "model_type", "options"}),
|
||||
}
|
||||
_VERBS = {"add": "Add tags", "auto": "Auto-tag"}
|
||||
_LOG_PREFIXES = {"tagged": "+", "unchanged": "=", "skipped": "-", "errors": "!"}
|
||||
@@ -137,14 +107,6 @@ READ_FAILED = "The file could not be read."
|
||||
SOURCE_CHANGED = "The source changed. Refresh the image collection."
|
||||
STORAGE_RESTORED = "Prompt storage was restored; start the job again."
|
||||
BUSY_RETRY = "The file could not be read now. Run the job again for it."
|
||||
# Tag output images (#379): the per-image batch.
|
||||
ALREADY_AUTOTAGGED = "Already auto-tagged"
|
||||
LISTING_FAILED = "The output folders could not be listed. Try again."
|
||||
LISTING_STATUS = "Listing output images…"
|
||||
_INTERNAL_ERROR = "An internal error occurred. Check server logs for details."
|
||||
# The pre-pass stats and looks up this many targets at a time, checking for a
|
||||
# cancel between chunks.
|
||||
PREPASS_CHUNK = 500
|
||||
|
||||
_monotonic = time.monotonic # the job's clock (a test can fake it)
|
||||
|
||||
@@ -179,8 +141,6 @@ class _Spec:
|
||||
tags: tuple = ()
|
||||
model_type: str | None = None
|
||||
options: dict | None = None
|
||||
# A listing filter the start resolves into ``targets`` (#392).
|
||||
filter: dict | None = None
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
@@ -239,35 +199,21 @@ def parse_image_tag_request(data: dict):
|
||||
"""A :class:`_Spec` for a start body, or ``(None, error response)``.
|
||||
|
||||
``{mode:"add", targets, tags}`` or ``{mode:"auto", targets, model_type,
|
||||
options}`` exactly; options follow the Tag dialog's Generate rules. A
|
||||
``filter`` (a listing query object, parse_listing_filter) may stand in for
|
||||
``targets``; the start resolves it.
|
||||
options}`` exactly; options follow the Tag dialog's Generate rules.
|
||||
"""
|
||||
fields = _FIELDS.get(data.get("mode"))
|
||||
if fields is None:
|
||||
return None, bad_request('mode must be "add" or "auto"')
|
||||
fields = fields | {"filter" if "filter" in data else "targets"}
|
||||
if set(data) != fields:
|
||||
return None, bad_request(
|
||||
"Request must contain exactly " + ", ".join(sorted(fields))
|
||||
)
|
||||
listing_filter, targets = None, ()
|
||||
if "filter" in data:
|
||||
try:
|
||||
listing_filter = parse_listing_filter(data["filter"])
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
else:
|
||||
targets, error = _parse_targets(data["targets"])
|
||||
if error is not None:
|
||||
return None, error
|
||||
targets, error = _parse_targets(data["targets"])
|
||||
if error is not None:
|
||||
return None, error
|
||||
if data["mode"] == "add":
|
||||
spec, error = _parse_add(targets, data)
|
||||
else:
|
||||
spec, error = _parse_auto(targets, data)
|
||||
if spec is not None and listing_filter is not None:
|
||||
spec = dataclasses.replace(spec, filter=listing_filter)
|
||||
return spec, error
|
||||
return _parse_add(targets, data)
|
||||
return _parse_auto(targets, data)
|
||||
|
||||
|
||||
def _capture_outcome(exc: CaptureError, root, relative: str):
|
||||
@@ -300,92 +246,6 @@ async def _capacity_retried(job, attempt):
|
||||
return await attempt()
|
||||
|
||||
|
||||
class Refusal(NamedTuple):
|
||||
"""Why a Tag output images start was refused.
|
||||
|
||||
``response`` is the native route's JSON answer; ``message`` the sentence
|
||||
a legacy stream frame carries.
|
||||
"""
|
||||
|
||||
status: int
|
||||
code: str
|
||||
message: str
|
||||
response: web.Response
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AutotagRun:
|
||||
"""What one Tag output images job runs with (validated at its start)."""
|
||||
|
||||
model_type: str
|
||||
existing: str
|
||||
options: dict
|
||||
keep_in_memory: bool
|
||||
backup_on: bool
|
||||
owner: object
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _BatchTarget:
|
||||
"""One still image of the batch: the job target and its private keys."""
|
||||
|
||||
target: tuple # (root_index, root_scope, relative): the image-tags target
|
||||
path: str # the listed file, for the pre-pass stat (never sent out)
|
||||
locator: str # the image tag key: normcase(join(realpath(root), relative))
|
||||
label: str # the root's public label
|
||||
|
||||
|
||||
def _still_targets(api, index, root, files, seen: set) -> list:
|
||||
"""Worker thread: one root's listed still images as _BatchTargets.
|
||||
|
||||
Newest first (the listing's order). A locator an earlier root already
|
||||
produced (overlapping roots) is dropped; ``seen`` collects them.
|
||||
"""
|
||||
scope = api._gallery_root_scope(root, index)
|
||||
label = _public_path(root)
|
||||
real = os.path.realpath(root)
|
||||
prefix = os.path.join(str(root), "")
|
||||
found = []
|
||||
for path, _mtime in files:
|
||||
relative = _listing_posix(root, prefix, path)
|
||||
if is_video_path(relative):
|
||||
continue
|
||||
locator = os.path.normcase(os.path.join(real, *relative.split("/")))
|
||||
if locator in seen:
|
||||
continue
|
||||
seen.add(locator)
|
||||
found.append(_BatchTarget((index, scope, relative), str(path), locator, label))
|
||||
return found
|
||||
|
||||
|
||||
def _prepass_chunk(db, chunk, skip_marked: bool, max_file: int):
|
||||
"""Worker thread: ``(remaining, skips)`` for a chunk of _BatchTargets.
|
||||
|
||||
Stats each file (no open, no hash): gone is "File not found", over the
|
||||
capture limit "Too large to verify"; under "Skip already tagged" a file
|
||||
whose current bytes are marked is "Already auto-tagged". ``skips`` are
|
||||
``(target, detail)`` pairs in target order.
|
||||
"""
|
||||
stats, skips, alive = {}, [], []
|
||||
for item in chunk:
|
||||
fingerprint = _current_fingerprint(item.path)
|
||||
if fingerprint is None:
|
||||
skips.append((item, FILE_NOT_FOUND))
|
||||
elif fingerprint[0] > max_file:
|
||||
skips.append((item, TOO_LARGE))
|
||||
else:
|
||||
stats[item.locator] = fingerprint
|
||||
alive.append(item)
|
||||
marked = db.autotag_state(stats) if skip_marked and stats else {}
|
||||
remaining = []
|
||||
for item in alive:
|
||||
if marked.get(item.locator):
|
||||
skips.append((item, ALREADY_AUTOTAGGED))
|
||||
else:
|
||||
remaining.append(item)
|
||||
return remaining, skips
|
||||
|
||||
|
||||
def _load_model(service, model_type: str, use_gpu: bool) -> None:
|
||||
"""Worker thread: load (or swap to) *model_type*; never downloads."""
|
||||
if service.is_model_loaded() and service.get_loaded_model_type() == model_type:
|
||||
@@ -394,25 +254,6 @@ def _load_model(service, model_type: str, use_gpu: bool) -> None:
|
||||
raise RuntimeError(f"{model_type} could not be loaded")
|
||||
|
||||
|
||||
def _refusal(status: int, code: str, message: str) -> Refusal:
|
||||
response = web.json_response(
|
||||
{"success": False, "code": code, "error": message}, status=status
|
||||
)
|
||||
return Refusal(status, code, message, response)
|
||||
|
||||
|
||||
def _busy_refusal(busy: JobBusy) -> Refusal:
|
||||
title = getattr(busy.running, "title", "") or "Another job"
|
||||
return Refusal(409, "busy", f"{title} already running", busy_response(busy))
|
||||
|
||||
|
||||
def _batch_cancelled(job) -> str:
|
||||
"""Say the batch was cancelled; its end state (the runner ends the job)."""
|
||||
job.update(status_line=BATCH_CANCELLED)
|
||||
job.log_line(BATCH_CANCELLED)
|
||||
return "cancelled"
|
||||
|
||||
|
||||
class ImageTagJobsMixin:
|
||||
"""The ``image-tags`` job kind: start route and runner."""
|
||||
|
||||
@@ -429,10 +270,6 @@ class ImageTagJobsMixin:
|
||||
spec, error = parse_image_tag_request(data)
|
||||
if error is not None:
|
||||
return error
|
||||
if spec.filter is not None:
|
||||
spec, error = await self._resolve_job_filter(spec)
|
||||
if error is not None:
|
||||
return error
|
||||
roots = await self._run_in_executor(self._job_roots, spec.targets)
|
||||
if any(root is None for root, _label in roots.values()):
|
||||
return _coded("source-changed")
|
||||
@@ -464,246 +301,6 @@ class ImageTagJobsMixin:
|
||||
job.task.add_done_callback(lambda _task: model_gate.release(owner))
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
# -- Tag output images (#379) ---------------------------------------------
|
||||
|
||||
async def _autotag_targets(self) -> list:
|
||||
"""Every still image the output folders list, as _BatchTargets.
|
||||
|
||||
Root by root in configured order, each newest first (the shared
|
||||
listing, ``_get_gallery_files``, used as it is even when stale). Videos
|
||||
are not targets; a file under two overlapping roots is one target.
|
||||
"""
|
||||
seen: set = set()
|
||||
found = []
|
||||
for index, root in enumerate(list(self._get_all_output_dirs())):
|
||||
files = await self._get_gallery_files(root)
|
||||
found.extend(
|
||||
await self._run_in_executor(
|
||||
_still_targets, self, index, root, files, seen
|
||||
)
|
||||
)
|
||||
return found
|
||||
|
||||
async def _start_autotag(
|
||||
self, *, model_type, existing, options, keep_in_memory, title, owner_kind
|
||||
):
|
||||
"""Register a Tag output images job: ``(job, None)`` or ``(None, Refusal)``.
|
||||
|
||||
``options`` are already validated (``validate_options``). The checks
|
||||
run in this order: 409 ``model-unavailable``; 507
|
||||
``insufficient-space`` (only when the safety backup is on); 409
|
||||
``busy`` (another records job); 409 ``model-busy``. No await follows
|
||||
the busy check. The gate is taken before the job registers and
|
||||
released when the task ends, also before its runner ever starts.
|
||||
"""
|
||||
from ..autotag import MODELS, get_autotag_service
|
||||
|
||||
backup_on = self._safety_backup_policy() == "on"
|
||||
status = await self._run_in_executor(get_autotag_service().get_models_status)
|
||||
_disk, short, _size = await self._run_in_executor(
|
||||
functools.partial(
|
||||
disk_plan, self.db.model.db_path, backup=backup_on, vacuum=False
|
||||
)
|
||||
)
|
||||
label = MODELS[model_type]["label"]
|
||||
if not (status.get(model_type) or {}).get("downloaded"):
|
||||
message = (
|
||||
f"{label} is not downloaded. Download it in System › Auto-tagging."
|
||||
)
|
||||
return None, _refusal(409, "model-unavailable", message)
|
||||
if backup_on and short is not None:
|
||||
return None, _refusal(507, "insufficient-space", short_space_message(short))
|
||||
busy = self._jobs.blocker(AUTOTAG_BATCH_JOB_KIND, admission="records")
|
||||
if busy is not None:
|
||||
return None, _busy_refusal(JobBusy(busy))
|
||||
owner = model_gate.new_owner(owner_kind)
|
||||
if not model_gate.try_acquire(owner):
|
||||
response = model_busy_response()
|
||||
return None, Refusal(409, "model-busy", MODEL_BUSY_ERROR, response)
|
||||
run = AutotagRun(
|
||||
model_type, existing, options, keep_in_memory, backup_on, owner
|
||||
)
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
AUTOTAG_BATCH_JOB_KIND,
|
||||
title,
|
||||
admission="records",
|
||||
cancellable=True,
|
||||
scope={"model_type": model_type, "existing": existing, "label": label},
|
||||
groups=AUTOTAG_GROUP_NAMES,
|
||||
runner=functools.partial(self._run_autotag_batch, run),
|
||||
)
|
||||
except BaseException as refused:
|
||||
model_gate.release(owner)
|
||||
if isinstance(refused, JobBusy):
|
||||
return None, _busy_refusal(refused)
|
||||
raise
|
||||
job.task.add_done_callback(lambda _task: model_gate.release(owner))
|
||||
return job, None
|
||||
|
||||
async def _run_autotag_batch(self, run: AutotagRun, job) -> None:
|
||||
"""The ``autotag`` job: backup, model, listing, pre-pass, per-image loop.
|
||||
|
||||
``job.cancelled`` is checked between phases (and per pre-pass chunk
|
||||
and per image). Each phase publishes ``{"type": "phase", "phase"}``.
|
||||
With ``keep_in_memory`` false the model is unloaded under the gate.
|
||||
The job ends only after that cleanup and the gate release, so a job
|
||||
that reads as ended never still holds the gate.
|
||||
"""
|
||||
state = "failed"
|
||||
try:
|
||||
state = await self._autotag_batch_phases(run, job)
|
||||
except Exception:
|
||||
self.logger.exception("Tag output images failed")
|
||||
job.update(status_line=_INTERNAL_ERROR)
|
||||
job.log_line(f"Tagging failed · {_INTERNAL_ERROR}")
|
||||
finally:
|
||||
try:
|
||||
if not run.keep_in_memory:
|
||||
await self._unload_batch_model(run.owner)
|
||||
if run.backup_on:
|
||||
await self._prune_backup_pool(job, self.db)
|
||||
finally:
|
||||
model_gate.release(run.owner)
|
||||
job.publish({"type": state, "progress": job.progress})
|
||||
job.finish(state)
|
||||
|
||||
async def _unload_batch_model(self, owner) -> None:
|
||||
from ..autotag import get_autotag_service
|
||||
|
||||
service = get_autotag_service()
|
||||
try:
|
||||
if service.is_model_loaded():
|
||||
await model_gate.run(owner, service.unload_model)
|
||||
except Exception:
|
||||
self.logger.exception("The tagging model could not be unloaded")
|
||||
|
||||
async def _autotag_batch_phases(self, run: AutotagRun, job) -> str:
|
||||
"""Run the phases; the job's end state (the caller ends the job)."""
|
||||
job.publish({"type": "phase", "phase": "backup"})
|
||||
ok, _backup = await self._safety_backup_step(job, self.db, run.backup_on)
|
||||
if not ok:
|
||||
return "failed"
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
job.publish({"type": "phase", "phase": "model"})
|
||||
job.update(status_line="Loading the tagging model")
|
||||
spec = _Spec("auto", (), model_type=run.model_type, options=run.options)
|
||||
service = await self._prepare_tagging_model(job, spec, run.owner)
|
||||
if service is None:
|
||||
return "failed"
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
job.publish({"type": "phase", "phase": "listing"})
|
||||
job.update(status_line=LISTING_STATUS)
|
||||
epoch = await self._run_in_executor(self.db.get_execution_epoch)
|
||||
try:
|
||||
targets = await self._autotag_targets()
|
||||
except Exception:
|
||||
self.logger.exception("Tag output images could not list the outputs")
|
||||
job.update(status_line=LISTING_FAILED)
|
||||
job.log_line(f"! {LISTING_FAILED}")
|
||||
return "failed"
|
||||
total = len(targets)
|
||||
job.scope["images"] = total
|
||||
job.update(total=total)
|
||||
policy = (
|
||||
"skip images already auto-tagged" if run.existing == "skip" else "merge"
|
||||
)
|
||||
job.log_line(
|
||||
f"Tagging started · {job.scope.get('label') or run.model_type} · "
|
||||
f"{_plural(total, 'image')} · {policy}"
|
||||
)
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
remaining = await self._autotag_prepass(job, targets, run.existing == "skip")
|
||||
if remaining is None:
|
||||
return _batch_cancelled(job)
|
||||
done = total - len(remaining)
|
||||
spec = dataclasses.replace(
|
||||
spec, targets=tuple(item.target for item in remaining)
|
||||
)
|
||||
loop_run = _Run(job, spec, epoch, run.owner, service)
|
||||
if not await self._tag_each_file(loop_run, done_before=done):
|
||||
job.update(status_line=STORAGE_RESTORED)
|
||||
job.log_line(f"Tagging stopped · {STORAGE_RESTORED}")
|
||||
return "failed"
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
finished = f"Tagged {_plural(job.counters.get('tagged', 0), 'image')}"
|
||||
job.update(status_line=finished, progress=100)
|
||||
job.log_line(finished)
|
||||
return "finished"
|
||||
|
||||
async def _autotag_prepass(self, job, targets, skip_marked: bool):
|
||||
"""The targets left for the model, or None when cancelled meanwhile.
|
||||
|
||||
Opens and hashes nothing: one stat per file and one indexed read per
|
||||
chunk. Skips are counted with ``record_counted`` (exact counters, rows
|
||||
up to the cap, no log line each).
|
||||
"""
|
||||
max_file = self._capture_store().max_file
|
||||
remaining, skipped = [], 0
|
||||
total = len(targets)
|
||||
for start in range(0, total, PREPASS_CHUNK):
|
||||
if job.cancelled:
|
||||
return None
|
||||
chunk = targets[start : start + PREPASS_CHUNK]
|
||||
left, skips = await self._run_in_executor(
|
||||
_prepass_chunk, self.db, chunk, skip_marked, max_file
|
||||
)
|
||||
remaining.extend(left)
|
||||
if skips:
|
||||
job.record_counted(
|
||||
"skipped",
|
||||
len(skips),
|
||||
[
|
||||
{
|
||||
"path": item.target[2],
|
||||
"root": item.label,
|
||||
"root_index": item.target[0],
|
||||
"detail": detail,
|
||||
}
|
||||
for item, detail in skips
|
||||
],
|
||||
)
|
||||
skipped += len(skips)
|
||||
job.update(processed=skipped, progress=skipped * 100 // total)
|
||||
job.publish({"type": "progress", "progress": job.progress})
|
||||
if skipped:
|
||||
job.log_line(f"Skipped {_plural(skipped, 'image')} before tagging")
|
||||
return remaining
|
||||
|
||||
async def _resolve_job_filter(self, spec: _Spec):
|
||||
"""``spec`` with its filter resolved into targets, or ``(None, error)``.
|
||||
|
||||
The listing's own identities, in its order, kept only where a target
|
||||
could have named them (a media path the job can tag).
|
||||
"""
|
||||
cap = MAX_FILTER_TARGETS
|
||||
try:
|
||||
selection = await self.resolve_listing_filter(spec.filter, cap)
|
||||
except ListingSourceChanged:
|
||||
return None, _coded("source-changed")
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
except Exception:
|
||||
# Never echo it: it can carry SQL or database text.
|
||||
self.logger.error("A tagging filter could not be resolved", exc_info=True)
|
||||
return None, _error(500, "listing-failed", FILTER_LISTING_FAILED)
|
||||
if selection.truncated:
|
||||
return None, _error(400, "too-many-images", TOO_MANY_IMAGES.format(cap=cap))
|
||||
targets = []
|
||||
for index, scope, relative in selection.targets:
|
||||
target, error = _target(
|
||||
{"root_index": index, "root_scope": scope, "path": relative}
|
||||
)
|
||||
if error is None:
|
||||
targets.append(target)
|
||||
if not targets:
|
||||
return None, _error(400, "no-images", NO_IMAGES)
|
||||
return dataclasses.replace(spec, targets=tuple(targets)), None
|
||||
|
||||
# -- runner ----------------------------------------------------------------
|
||||
|
||||
def _job_roots(self, targets) -> dict:
|
||||
@@ -734,39 +331,28 @@ class ImageTagJobsMixin:
|
||||
job.finish("failed")
|
||||
return
|
||||
if not await self._tag_each_file(run):
|
||||
job.update(status_line=STORAGE_RESTORED)
|
||||
job.log_line(f"{verb} stopped · {STORAGE_RESTORED}")
|
||||
job.finish("failed")
|
||||
return
|
||||
state = "cancelled" if job.cancelled else "finished"
|
||||
# An ended job never keeps the loop's "about X min left" (#379).
|
||||
ended = f"{verb} {state} · {_plural(job.processed, 'file')} processed"
|
||||
job.update(status_line=ended)
|
||||
job.log_line(ended)
|
||||
job.log_line(f"{verb} {state} · {_plural(job.processed, 'file')} processed")
|
||||
job.publish({"type": state, "progress": job.progress})
|
||||
job.finish(state)
|
||||
finally:
|
||||
if owner is not None:
|
||||
model_gate.release(owner)
|
||||
|
||||
async def _tag_each_file(self, run: _Run, *, done_before: int = 0) -> bool:
|
||||
"""Tag the targets in order; False when the job had to stop.
|
||||
|
||||
``done_before`` files were already counted (the batch's pre-pass): file
|
||||
n of this loop is "file done_before + n" of the job, and the time left
|
||||
uses this loop's rate only.
|
||||
"""
|
||||
job, count = run.job, len(run.spec.targets)
|
||||
total = done_before + count
|
||||
async def _tag_each_file(self, run: _Run) -> bool:
|
||||
"""Tag the targets in order; False when the job had to stop."""
|
||||
job, total = run.job, len(run.spec.targets)
|
||||
run.roots = await self._run_in_executor(self._job_roots, run.spec.targets)
|
||||
started = _monotonic()
|
||||
for number, target in enumerate(run.spec.targets, 1):
|
||||
if job.cancelled:
|
||||
break
|
||||
# The scan's estimate (D56), with each finished file as one batch.
|
||||
left = _time_left(number - 1, count, number - 1, _monotonic() - started)
|
||||
shown = done_before + number
|
||||
job.update(status_line=f"Processing file {shown:,} of {total:,}{left}")
|
||||
left = _time_left(number - 1, total, number - 1, _monotonic() - started)
|
||||
job.update(status_line=f"Processing file {number:,} of {total:,}{left}")
|
||||
job.publish({"type": "progress", "progress": job.progress})
|
||||
stopped = False
|
||||
try:
|
||||
@@ -777,7 +363,7 @@ class ImageTagJobsMixin:
|
||||
self.logger.exception("Image tagging failed for one file")
|
||||
group, detail = "errors", SAVE_FAILED
|
||||
self._record_tag_outcome(run, target, group, detail)
|
||||
job.update(processed=shown, progress=shown * 100 // total)
|
||||
job.update(processed=number, progress=number * 100 // total)
|
||||
if stopped:
|
||||
return False
|
||||
return True
|
||||
@@ -853,17 +439,9 @@ class ImageTagJobsMixin:
|
||||
store.release(capture.token)
|
||||
|
||||
async def _auto_tag_capture(self, run: _Run, store, capture, root, relative):
|
||||
"""Suggest on the capture, then union only onto the same current bytes.
|
||||
|
||||
The bytes are marked auto-tagged with the union, also when the model
|
||||
returned no names; captured bytes no model can read are marked too
|
||||
(and stay skipped), so a "Skip already tagged" run passes them by.
|
||||
"""
|
||||
"""Suggest on the capture, then union only onto the same current bytes."""
|
||||
if not capture.inferable:
|
||||
group, detail = await self._union_image_tags(
|
||||
capture, [], run.epoch, mark_autotagged=True
|
||||
)
|
||||
return (group, detail) if group == "errors" else ("skipped", NOT_A_STILL)
|
||||
return "skipped", NOT_A_STILL
|
||||
if store.acquire(capture.token) is None:
|
||||
raise _JobStopped # every capture was invalidated (restore)
|
||||
spec = run.spec
|
||||
@@ -889,30 +467,21 @@ class ImageTagJobsMixin:
|
||||
if facts.locator != capture.locator or facts.sha256 != capture.sha256:
|
||||
return "skipped", FILE_CHANGED
|
||||
names = [str(name) for name, _score in pairs]
|
||||
return await self._union_image_tags(
|
||||
facts, names, run.epoch, mark_autotagged=True
|
||||
)
|
||||
if not names:
|
||||
return "unchanged", NO_NEW_TAGS
|
||||
return await self._union_image_tags(facts, names, run.epoch)
|
||||
|
||||
async def _union_image_tags(
|
||||
self, facts, tags: list, epoch: str, *, mark_autotagged: bool = False
|
||||
):
|
||||
"""``add_image_tags`` on the verified bytes, as a (group, detail).
|
||||
|
||||
``mark_autotagged`` (a model's result) marks the bytes auto-tagged in
|
||||
the same write; then ``tags`` may be empty.
|
||||
"""
|
||||
async def _union_image_tags(self, facts, tags: list, epoch: str):
|
||||
"""``add_image_tags`` on the verified bytes, as a (group, detail)."""
|
||||
try:
|
||||
receipt = await self._run_in_executor(
|
||||
functools.partial(
|
||||
self.db.add_image_tags,
|
||||
locator=facts.locator,
|
||||
sha256=facts.sha256,
|
||||
size=facts.size,
|
||||
mtime_ns=facts.mtime_ns,
|
||||
expected_epoch=epoch,
|
||||
tags=tags,
|
||||
mark_autotagged=mark_autotagged,
|
||||
)
|
||||
self.db.add_image_tags,
|
||||
locator=facts.locator,
|
||||
sha256=facts.sha256,
|
||||
size=facts.size,
|
||||
mtime_ns=facts.mtime_ns,
|
||||
expected_epoch=epoch,
|
||||
tags=tags,
|
||||
)
|
||||
except LibraryCommandError as exc:
|
||||
if exc.code == "storage-changed":
|
||||
|
||||
+61
-204
@@ -575,6 +575,12 @@ def _current_fingerprint(path: Path) -> tuple[int, int] | None:
|
||||
return observed.st_size, observed.st_mtime_ns
|
||||
|
||||
|
||||
def _image_tags_relevant(names: list, wanted: set, query: str) -> bool:
|
||||
if wanted and not wanted.isdisjoint(names):
|
||||
return True
|
||||
return bool(query) and any(query in name.casefold() for name in names)
|
||||
|
||||
|
||||
def _workspace_key(entry: tuple, order: str) -> tuple:
|
||||
"""The keyset sort key of *entry* (a _Listed or a 4-tuple entry)."""
|
||||
if not isinstance(entry, _Listed):
|
||||
@@ -1099,94 +1105,37 @@ def _workspace_filters(query: Mapping) -> dict:
|
||||
}
|
||||
|
||||
|
||||
# The listing parameters a bulk request may name as its filter (#392): the
|
||||
# workspace query without mode, limit, cursor and the Activity-only ``path``,
|
||||
# plus the chosen root's ``root_scope`` (a renumbered source is refused).
|
||||
LISTING_FILTER_KEYS = frozenset(
|
||||
{
|
||||
"q",
|
||||
"subfolder",
|
||||
"category",
|
||||
"tags",
|
||||
"root",
|
||||
"root_scope",
|
||||
"media",
|
||||
"sort",
|
||||
"favorites",
|
||||
}
|
||||
)
|
||||
MAX_LISTING_FILTER_VALUE = 2048
|
||||
# The most images "Select all N matching" acts on in one request (#392).
|
||||
MAX_LISTING_SELECTION = 10_000
|
||||
|
||||
|
||||
def parse_listing_filter(raw: object) -> dict:
|
||||
"""A bulk request's listing filter (#392): the listing's own query as an object.
|
||||
|
||||
Keys are a subset of LISTING_FILTER_KEYS, each value a string of at most
|
||||
MAX_LISTING_FILTER_VALUE characters, validated as the listing validates its
|
||||
query. The root's bounds need the current roots, so resolve_listing_filter
|
||||
checks those. Raises ValueError.
|
||||
"""
|
||||
if not isinstance(raw, dict) or not set(raw) <= LISTING_FILTER_KEYS:
|
||||
raise ValueError("filter must be an object of listing parameters")
|
||||
if any(
|
||||
not isinstance(value, str) or len(value) > MAX_LISTING_FILTER_VALUE
|
||||
for value in raw.values()
|
||||
):
|
||||
raise ValueError("filter values must be short strings")
|
||||
if "root_scope" in raw and "root" not in raw:
|
||||
raise ValueError("root_scope needs root")
|
||||
_workspace_filters(raw)
|
||||
return dict(raw)
|
||||
|
||||
|
||||
class ListingSourceChanged(Exception):
|
||||
"""A listing filter's ``root_scope`` no longer names its root (#392)."""
|
||||
|
||||
|
||||
class ListingSelection(NamedTuple):
|
||||
"""Every image a listing filter admits, in the listing's order (#392).
|
||||
|
||||
``targets`` are listing identities ``(root_index, root_scope, path)``;
|
||||
``positive_ids`` the distinct linked positives (first seen), ``unlinked``
|
||||
the admitted images without one, and ``truncated`` that more than the cap
|
||||
matched (the lists then stop at the cap).
|
||||
"""
|
||||
|
||||
targets: list
|
||||
positive_ids: list
|
||||
unlinked: int
|
||||
truncated: bool
|
||||
|
||||
|
||||
# A plain listing's cursor page leaves positive_count out (see
|
||||
# _select_workspace_page); None instead means the count could not be read.
|
||||
_COUNT_OMITTED = object()
|
||||
|
||||
|
||||
def _admitted_entries(
|
||||
def _filtered_page(
|
||||
entries: list,
|
||||
start: int,
|
||||
size: int,
|
||||
filters: dict,
|
||||
links: dict | None,
|
||||
favorites: frozenset | None = None,
|
||||
image_tags: dict | None = None,
|
||||
):
|
||||
"""Yield ``(position, entry, link match)`` for each entry the filters admit.
|
||||
) -> tuple[int, list, int | None]:
|
||||
"""Count the *entries* the filters admit and take *size* from *start* on.
|
||||
|
||||
*links* maps normalized paths to (hit, rating, positive id) for every
|
||||
unambiguously linked image (see PromptDatabase.get_image_link_matches); a
|
||||
category or tag filter admits only those, and ``q`` admits a filename match
|
||||
or a linked-positive hit. *favorites*, when given, admits only those listed
|
||||
paths (``favorites=1``). *image_tags* maps normalized paths to their
|
||||
_ImageTagMatch: a tag filter also admits a file whose current image tags
|
||||
(with its linked positive's tags) carry every tag, and ``q`` also admits an
|
||||
image tag name hit.
|
||||
or a linked-positive hit. Returns the total, the page and the number of
|
||||
distinct linked positives among all admitted entries (None without links).
|
||||
*favorites*, when given, admits only those listed paths (``favorites=1``).
|
||||
*image_tags* maps normalized paths to their _ImageTagMatch: a tag filter
|
||||
also admits a file whose current image tags (with its linked positive's
|
||||
tags) carry every tag, and ``q`` also admits an image tag name hit.
|
||||
"""
|
||||
query, media, folder = filters["query"], filters["media"], filters["folder"]
|
||||
members_only = filters["category"] is not None or bool(filters["tags"])
|
||||
native_folder = folder.replace("/", os.sep)
|
||||
nested = native_folder + os.sep
|
||||
total, page, positives = 0, [], set()
|
||||
exact = filters.get("path", "")
|
||||
for position, entry in enumerate(entries):
|
||||
if exact and entry.folded != exact:
|
||||
@@ -1210,28 +1159,6 @@ def _admitted_entries(
|
||||
and not (image and image.query)
|
||||
):
|
||||
continue
|
||||
yield position, entry, match
|
||||
|
||||
|
||||
def _filtered_page(
|
||||
entries: list,
|
||||
start: int,
|
||||
size: int,
|
||||
filters: dict,
|
||||
links: dict | None,
|
||||
favorites: frozenset | None = None,
|
||||
image_tags: dict | None = None,
|
||||
) -> tuple[int, list, int | None]:
|
||||
"""Count the *entries* the filters admit and take *size* from *start* on.
|
||||
|
||||
Admission is _admitted_entries'. Returns the total, the page and the number
|
||||
of distinct linked positives among all admitted entries (None without
|
||||
links).
|
||||
"""
|
||||
total, page, positives = 0, [], set()
|
||||
for position, entry, match in _admitted_entries(
|
||||
entries, filters, links, favorites, image_tags
|
||||
):
|
||||
total += 1
|
||||
if match is not None:
|
||||
positives.add(match[2])
|
||||
@@ -2164,9 +2091,8 @@ class ImageRoutesMixin:
|
||||
def _image_tag_matches(self, listings: list[tuple], filters: dict) -> dict:
|
||||
"""Normalized path -> _ImageTagMatch for files whose image tags count.
|
||||
|
||||
Looks the relevant tag names up (exact under Tags, a case-folded
|
||||
substring for ``q``) and their files through the tag index, keeping
|
||||
only listed locators; only those candidates are
|
||||
Streams the tagged byte records once and keeps only listed locators
|
||||
carrying a filter tag or a ``q`` hit; only those candidates are
|
||||
stat'ed (never every listed file), and a candidate's tags count only
|
||||
while a byte record is verified at its present ``(size, mtime_ns)``
|
||||
(spec amendment S1). Under a Tags filter the candidates' linked
|
||||
@@ -2175,15 +2101,16 @@ class ImageRoutesMixin:
|
||||
tags, query, category = filters["tags"], filters["query"], filters["category"]
|
||||
wanted = set(tags)
|
||||
located = self._workspace_locators(listings)
|
||||
# The tag index answers which files carry a relevant name (#379); no
|
||||
# read streams every image tag row.
|
||||
tag_ids = self.db.image_tag_ids(wanted, query)
|
||||
candidates = {}
|
||||
if tag_ids:
|
||||
for locator in self.db.image_tag_locators(tag_ids):
|
||||
records = self.db.iter_image_tag_records()
|
||||
try:
|
||||
for locator, _size, _mtime_ns, names in records:
|
||||
entry = located.get(locator)
|
||||
if entry is not None:
|
||||
candidates[locator] = entry
|
||||
if entry is not None and locator not in candidates:
|
||||
if _image_tags_relevant(names, wanted, query):
|
||||
candidates[locator] = entry
|
||||
finally:
|
||||
records.close()
|
||||
stats = {}
|
||||
for locator, entry in candidates.items():
|
||||
fingerprint = _current_fingerprint(entry.path)
|
||||
@@ -2210,23 +2137,35 @@ class ImageRoutesMixin:
|
||||
)
|
||||
return matches
|
||||
|
||||
def _workspace_admission(
|
||||
self, listings: list[tuple], filters: dict, *, read_links: bool, strict=False
|
||||
def _select_workspace_page(
|
||||
self,
|
||||
listings: list[tuple],
|
||||
filters: dict,
|
||||
after: tuple | None,
|
||||
limit: int,
|
||||
scope: str,
|
||||
) -> tuple:
|
||||
"""The ordered entries and what admits them (blocking; run in executor).
|
||||
"""One keyset page of the filtered listing (blocking; run in executor).
|
||||
|
||||
Returns ``(entries, key, links, image_tags, favorites)``: the listing in
|
||||
the filters' order, its keyset key, and the link, image-tag and favorite
|
||||
maps _admitted_entries reads. One SELECT (get_image_link_matches) reads
|
||||
every linked positive when *read_links*. A failed read raises when the
|
||||
filters need it (category, tags, the rating sort) or *strict*; else the
|
||||
listing stays usable without links (a search then matches filenames
|
||||
only). A failed image-tag read raises under a Tags filter; a search then
|
||||
matches without image tags.
|
||||
One SELECT (get_image_link_matches) reads every linked positive of the
|
||||
listing: the filters, the rating sort and the positive count share it.
|
||||
A plain listing (no ``q``, category, tags or rating sort) reads it only
|
||||
for its first page; its cursor pages read no links and the returned
|
||||
count is _COUNT_OMITTED, so the response leaves ``positive_count`` out
|
||||
and the frontend keeps the first page's value. On that plain first
|
||||
page a failed read leaves the listing usable and the count unknown
|
||||
(None); so does a failed read under ``q`` alone, whose search then
|
||||
matches filenames only (as before the linked search existed).
|
||||
"""
|
||||
order, category, tags = filters["order"], filters["category"], filters["tags"]
|
||||
needed = (
|
||||
bool(filters["query"])
|
||||
or category is not None
|
||||
or bool(tags)
|
||||
or order == "rating"
|
||||
)
|
||||
links = None
|
||||
if read_links:
|
||||
if needed or after is None:
|
||||
try:
|
||||
links = self.db.get_image_link_matches(
|
||||
filters["text"],
|
||||
@@ -2236,7 +2175,10 @@ class ImageRoutesMixin:
|
||||
wanted=self._workspace_wanted(listings),
|
||||
)
|
||||
except Exception:
|
||||
if strict or category is not None or tags or order == "rating":
|
||||
# Category, tags and the rating sort cannot be served without
|
||||
# the links. A plain listing and a filename search can: the
|
||||
# search then matches filenames only, and the count is None.
|
||||
if category is not None or tags or order == "rating":
|
||||
raise
|
||||
self.logger.warning(
|
||||
"Workspace prompt links could not be read; %s",
|
||||
@@ -2252,6 +2194,8 @@ class ImageRoutesMixin:
|
||||
try:
|
||||
image_tags = self._image_tag_matches(listings, filters)
|
||||
except Exception:
|
||||
# A Tags filter cannot be served without them (as without the
|
||||
# links); a search then matches without image tags.
|
||||
if tags:
|
||||
raise
|
||||
self.logger.warning(
|
||||
@@ -2290,41 +2234,6 @@ class ImageRoutesMixin:
|
||||
def key(entry):
|
||||
return _workspace_key(entry, order)
|
||||
|
||||
favorites = (
|
||||
self._workspace_favorite_paths(listings) if filters["favorites"] else None
|
||||
)
|
||||
return entries, key, links, image_tags, favorites
|
||||
|
||||
def _select_workspace_page(
|
||||
self,
|
||||
listings: list[tuple],
|
||||
filters: dict,
|
||||
after: tuple | None,
|
||||
limit: int,
|
||||
scope: str,
|
||||
) -> tuple:
|
||||
"""One keyset page of the filtered listing (blocking; run in executor).
|
||||
|
||||
The filters, the rating sort and the positive count share one link
|
||||
read (_workspace_admission). A plain listing (no ``q``, category, tags
|
||||
or rating sort) reads it only for its first page; its cursor pages read
|
||||
no links and the returned count is _COUNT_OMITTED, so the response
|
||||
leaves ``positive_count`` out and the frontend keeps the first page's
|
||||
value. On that plain first page a failed read leaves the listing usable
|
||||
and the count unknown (None); so does a failed read under ``q`` alone,
|
||||
whose search then matches filenames only (as before the linked search
|
||||
existed).
|
||||
"""
|
||||
order = filters["order"]
|
||||
needed = (
|
||||
bool(filters["query"])
|
||||
or filters["category"] is not None
|
||||
or bool(filters["tags"])
|
||||
or order == "rating"
|
||||
)
|
||||
entries, key, links, image_tags, favorites = self._workspace_admission(
|
||||
listings, filters, read_links=needed or after is None
|
||||
)
|
||||
start = 0
|
||||
if after is not None:
|
||||
if order == "name-desc":
|
||||
@@ -2332,6 +2241,9 @@ class ImageRoutesMixin:
|
||||
start = len(entries) - bisect_left(ascending, after, key=key)
|
||||
else:
|
||||
start = bisect_right(entries, after, key=key)
|
||||
favorites = (
|
||||
self._workspace_favorite_paths(listings) if filters["favorites"] else None
|
||||
)
|
||||
total, page, positive_count = _filtered_page(
|
||||
entries, start, limit + 1, filters, links, favorites, image_tags
|
||||
)
|
||||
@@ -2348,61 +2260,6 @@ class ImageRoutesMixin:
|
||||
positive_count,
|
||||
)
|
||||
|
||||
def _select_workspace_matches(
|
||||
self, listings: list[tuple], filters: dict, cap: int
|
||||
) -> ListingSelection:
|
||||
"""Every admitted entry, up to *cap*, as a ListingSelection (blocking).
|
||||
|
||||
The links are read strictly: the positives must be the listing's own.
|
||||
"""
|
||||
entries, _key, links, image_tags, favorites = self._workspace_admission(
|
||||
listings, filters, read_links=True, strict=True
|
||||
)
|
||||
targets, positives, unlinked, scopes = [], {}, 0, {}
|
||||
for _position, entry, match in _admitted_entries(
|
||||
entries, filters, links, favorites, image_tags
|
||||
):
|
||||
if len(targets) == cap:
|
||||
return ListingSelection(targets, list(positives), unlinked, True)
|
||||
scope = scopes.get(entry.index)
|
||||
if scope is None:
|
||||
scope = scopes[entry.index] = self._gallery_root_scope(
|
||||
entry.root, entry.index
|
||||
)
|
||||
relative = entry.path.relative_to(entry.root).as_posix()
|
||||
targets.append((entry.index, scope, relative))
|
||||
if match is None or match[2] is None:
|
||||
unlinked += 1
|
||||
else:
|
||||
positives.setdefault(match[2])
|
||||
return ListingSelection(targets, list(positives), unlinked, False)
|
||||
|
||||
async def resolve_listing_filter(
|
||||
self, params: Mapping, cap: int = MAX_LISTING_SELECTION
|
||||
) -> ListingSelection:
|
||||
"""The images a parse_listing_filter result admits right now (#392).
|
||||
|
||||
Raises ValueError for a root outside the current roots, and
|
||||
ListingSourceChanged when the filter's ``root_scope`` no longer names
|
||||
its root (the sources were renumbered since the listing).
|
||||
"""
|
||||
output_dirs = list(self._get_all_output_dirs())
|
||||
root_index = _workspace_root(params, output_dirs)
|
||||
error = self._gallery_scope_error(params, output_dirs)
|
||||
if error is not None:
|
||||
if error.status == 409:
|
||||
raise ListingSourceChanged()
|
||||
raise ValueError("Invalid gallery root scope")
|
||||
filters = _workspace_filters(params)
|
||||
listings = [
|
||||
(index, root, await self._get_gallery_files(root))
|
||||
for index, root in enumerate(output_dirs)
|
||||
if root_index is None or index == root_index
|
||||
]
|
||||
return await self._run_in_executor(
|
||||
self._select_workspace_matches, listings, filters, cap
|
||||
)
|
||||
|
||||
async def serve_image(self, request):
|
||||
"""Serve the actual image file using streamed FileResponse."""
|
||||
try:
|
||||
|
||||
@@ -1178,8 +1178,6 @@ class PromptRoutesMixin:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Expected a JSON object"}, status=400
|
||||
)
|
||||
if "images_filter" in data:
|
||||
return await self._bulk_category_by_filter(data)
|
||||
if not data.get("prompt_ids"):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "No prompt IDs provided"}, status=400
|
||||
@@ -1216,66 +1214,6 @@ class PromptRoutesMixin:
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def _bulk_category_by_filter(self, data: dict) -> web.Response:
|
||||
"""Set the category on every linked positive a listing filter admits.
|
||||
|
||||
#392 "Select all N matching": ``{images_filter, category}``, the filter
|
||||
being the Images listing's own query (parse_listing_filter), resolved
|
||||
now. The answer adds ``prompt_count`` (the distinct linked positives)
|
||||
and ``skipped`` (matching images without one); none linked changes
|
||||
nothing.
|
||||
"""
|
||||
from .images import (
|
||||
MAX_LISTING_SELECTION,
|
||||
ListingSourceChanged,
|
||||
parse_listing_filter,
|
||||
)
|
||||
|
||||
def refused(message, **extra):
|
||||
return web.json_response(
|
||||
{"success": False, "error": message, **extra}, status=400
|
||||
)
|
||||
|
||||
if set(data) != {"images_filter", "category"}:
|
||||
return refused("Request must contain exactly category, images_filter")
|
||||
category = data["category"]
|
||||
try:
|
||||
params = parse_listing_filter(data["images_filter"])
|
||||
validate_category(category)
|
||||
selection = await self.resolve_listing_filter(params)
|
||||
except ListingSourceChanged:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "source-changed",
|
||||
"error": "The source changed. Refresh the image collection.",
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return refused(str(exc))
|
||||
if selection.truncated:
|
||||
return refused(
|
||||
f"Narrow the view to at most {MAX_LISTING_SELECTION:,} images.",
|
||||
code="too-many-images",
|
||||
)
|
||||
category = (category or "").strip()
|
||||
ids = selection.positive_ids
|
||||
updated = (
|
||||
await self._run_in_executor(self.db.bulk_set_category, ids, category)
|
||||
if ids
|
||||
else 0
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"message": f"Set category for {updated} prompts",
|
||||
"updated_count": updated,
|
||||
"prompt_count": len(ids),
|
||||
"skipped": selection.unlinked,
|
||||
}
|
||||
)
|
||||
|
||||
def _all_prompts_for_export(self):
|
||||
"""Every prompt, fetched in EXPORT_PAGE_SIZE pages (blocking)."""
|
||||
prompts, offset = [], 0
|
||||
|
||||
@@ -756,24 +756,6 @@ class AutoTagService:
|
||||
|
||||
self.logger.info("Model unloaded, memory freed")
|
||||
|
||||
def wd14_device(self) -> str:
|
||||
"""``"gpu"`` when WD14 runs (or would run) on CUDA, else ``"cpu"``.
|
||||
|
||||
A loaded WD14 session answers from its own first provider; otherwise
|
||||
ONNX Runtime's available providers say whether CUDA would be offered
|
||||
(a session can still fall back to the processor). Loads no model.
|
||||
"""
|
||||
tagger = self._tagger
|
||||
if tagger is not None and tagger[0] == "wd14":
|
||||
providers = tagger[1][0].get_providers()
|
||||
return "gpu" if providers[:1] == ["CUDAExecutionProvider"] else "cpu"
|
||||
try:
|
||||
import onnxruntime as ort
|
||||
except ImportError:
|
||||
return "cpu"
|
||||
available = ort.get_available_providers()
|
||||
return "gpu" if "CUDAExecutionProvider" in available else "cpu"
|
||||
|
||||
def is_model_loaded(self) -> bool:
|
||||
"""Check if a model is currently loaded."""
|
||||
return self._tagger is not None
|
||||
|
||||
+1
-17
@@ -13,23 +13,7 @@ LLM_MODEL_TYPES = frozenset({"gguf", "hf"})
|
||||
MODEL_TYPES = WD14_MODEL_TYPES | LLM_MODEL_TYPES
|
||||
MAX_INSTRUCTION_CHARS = 4000
|
||||
MAX_INSTRUCTION_BYTES = 16 * 1024
|
||||
# Tag output images' review estimate (#379): seconds per image for the WD14
|
||||
# engines, by where ONNX Runtime runs them. The graphics card figure was
|
||||
# measured in PR 1's live check (RTX 4090, CUDA: Auto-tag selected on 100
|
||||
# images in 16.7 s with the model load, 0.167 s/image). The processor figure
|
||||
# is an upper bound from the maintainer's cancelled CPU run (19 min / 697
|
||||
# images, backup, model load and skips included), kept as the fallback. The
|
||||
# language-model engines have no figure.
|
||||
WD14_SECONDS_PER_IMAGE = 1.6
|
||||
WD14_GPU_SECONDS_PER_IMAGE = 0.17
|
||||
|
||||
|
||||
def wd14_seconds_per_image(device: str) -> float:
|
||||
"""The estimate's rate for ``"gpu"`` or ``"cpu"`` (anything else)."""
|
||||
return WD14_GPU_SECONDS_PER_IMAGE if device == "gpu" else WD14_SECONDS_PER_IMAGE
|
||||
|
||||
|
||||
# What a batch does with an image that already has tags (#328, #379).
|
||||
# What a batch does with a prompt that already has tags (#328).
|
||||
EXISTING_POLICIES = ("skip", "merge", "replace_after_review")
|
||||
|
||||
# The language-model tag template every page starts from until one is saved.
|
||||
|
||||
@@ -676,73 +676,3 @@ export function measureInspector(selector) {
|
||||
primary:node.classList.contains('ca-primary'),minHeight:getComputedStyle(node).minHeight})),
|
||||
};
|
||||
}
|
||||
|
||||
// #249 System S7 Task 20, shared since S12 (#329): the maintenance plan dialog's measurement and its D100 mapping,
|
||||
// moved unchanged from system-maintenance-concept.spec.mjs so the rotated-log review uses the same ones.
|
||||
/** Runs in the page: the open review dialog (plan or duplicates): its title, body and the concept's tokens. */
|
||||
export function measureReview() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const body = dialog.querySelector('#modal-body');
|
||||
const grid = body.querySelector(':scope > dl.meta-grid');
|
||||
const compare = body.querySelector('.compare-grid');
|
||||
const item = compare?.querySelector('.compare-item');
|
||||
const notes = [...body.querySelectorAll(':scope > p.small')];
|
||||
const note = notes.at(-1);
|
||||
const actions = body.querySelector(':scope > .dialog-actions');
|
||||
return {
|
||||
title: text(dialog.querySelector('#modal-title')),
|
||||
tree: [...body.children].map(name),
|
||||
heading: body.querySelector(':scope > h3') && { text: text(body.querySelector(':scope > h3')),
|
||||
...css(body.querySelector(':scope > h3'), 'fontSize', 'fontWeight', 'color', 'marginBottom') },
|
||||
grid: grid && { rows: [...grid.children].map(row => [text(row.querySelector('dt')), text(row.querySelector('dd'))]),
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
dt: css(grid.querySelector('dt'), 'fontSize', 'color', 'textTransform'), dd: css(grid.querySelector('dd'), 'fontSize', 'color') },
|
||||
details: body.querySelector(':scope > details') && { summary: text(body.querySelector(':scope > details > summary')),
|
||||
tree: [...body.querySelector(':scope > details').children].map(name) },
|
||||
backup: body.querySelector(':scope > .notice') && { tree: [...body.querySelector(':scope > .notice').children].map(name),
|
||||
text: [...body.querySelectorAll(':scope > .notice > div:first-child > *')].map(node => [node.localName, text(node)]),
|
||||
buttons: [...body.querySelectorAll(':scope > .notice button')].map(text) },
|
||||
compare: compare && { tree: [...compare.children].map(name), ...css(compare, 'display', 'gridTemplateColumns', 'columnGap'),
|
||||
items: [...compare.children].map(entry => ({ tree: [...entry.children].map(name), label: text(entry.querySelector('label')),
|
||||
checked: entry.querySelector('input[type=radio]').checked, image: entry.querySelector('img')?.className ?? null })),
|
||||
item: css(item, 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius'),
|
||||
image: css(item.querySelector('img'), 'width', 'objectFit'),
|
||||
label: css(item.querySelector('label'), 'fontSize', 'color'),
|
||||
radio: { name: item.querySelector('input[type=radio]').name.startsWith('keep-') ? 'keep-*' : 'other',
|
||||
type: item.querySelector('input').type } },
|
||||
note: note && { text: text(note), ...css(note, 'fontSize', 'color', 'marginTop') },
|
||||
actions: { tree: [...actions.children].map(name), ...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(item => ({ text: text(item), className: item.className,
|
||||
...css(item, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor') })) },
|
||||
};
|
||||
}
|
||||
|
||||
export const BACKUP_BOX = Object.freeze({ tree: ['div', 'div.row-actions'], text: [['h3', 'Back up the database first (recommended)'],
|
||||
['p', 'Downloads a copy of the prompts database before this operation changes any records.']], buttons: ['Back up now'] });
|
||||
|
||||
/**
|
||||
* D100: the server's plan in the concept's dialog: the operation's row title as its `h3` (the concept prints its
|
||||
* `data-plan` name), the four rows with the plan's values, the listed records (`details`), the hold line in the
|
||||
* concept's `p.small`, the D55 backup box before the actions, and "Apply reviewed operation". No sample note.
|
||||
*/
|
||||
export function mapPlanConcept(reference, title, plan) {
|
||||
const backup = plan.backup.policy === 'off' ? 'No safety backup (the policy is off)'
|
||||
: `Create and verify before changes (${plan.backup.destination_label})`;
|
||||
const tree = ['h3', 'dl.meta-grid', ...(plan.details.length ? ['details'] : []), 'p.small', 'div.notice', 'div.dialog-actions'];
|
||||
const [cancel, apply] = reference.actions.buttons;
|
||||
return {
|
||||
...reference,
|
||||
tree,
|
||||
heading: { ...reference.heading, text: title },
|
||||
grid: { ...reference.grid, rows: [['Scope', plan.scope.label],
|
||||
['Protected', plan.protected.reasons.length ? plan.protected.reasons.join(' · ') : 'Nothing is protected'],
|
||||
['Backup', backup], ['Media on disk', plan.media.label]] },
|
||||
details: plan.details.length ? { summary: `Listed records (${plan.details.length})`, tree: ['summary', 'pre'] } : null,
|
||||
backup: BACKUP_BOX,
|
||||
note: { ...reference.note, text: 'Nothing changes until you apply. This review holds for 15 minutes.' },
|
||||
actions: { ...reference.actions, buttons: [cancel, { ...apply, text: 'Apply reviewed operation' }] },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,283 +0,0 @@
|
||||
"""Run a browser-test fixture seeder off the live database, then publish once (#384).
|
||||
|
||||
The seeders make hundreds of small commits. On a slow CI disk the live
|
||||
database's commits queue behind WAL checkpoint syncs and a reset took over
|
||||
30 s. ``ScratchSeeder.run`` instead:
|
||||
|
||||
1. copies the live database into a scratch database (SQLite backup API) whose
|
||||
connections never sync (a crash loses only the copy);
|
||||
2. runs the unchanged seeder against an API proxy whose ``db`` is the scratch
|
||||
database; everything else (files, thumbnails, caches, command state) is
|
||||
the live API's, written exactly as before;
|
||||
3. replaces the rows of every table the seed changed in the live database, in
|
||||
one transaction on the live connection. Tables the seed left alone are not
|
||||
touched, so state another fixture owns stays as it was.
|
||||
|
||||
A commit by any other live connection while the seed ran fails the reset
|
||||
instead of being overwritten (``PRAGMA data_version``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import weakref
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, TypeVar
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
# Set to 1 to run seeders on the live database as before #384 (the equivalence
|
||||
# proof, tests/browser/reset_equivalence.py, compares both ways).
|
||||
SEED_ON_LIVE_ENV = "PM_TEST_SEED_ON_LIVE"
|
||||
SCRATCH_SCHEMA = "reset_seed"
|
||||
# API calls a seed makes that must see the published rows (see _SeedApi).
|
||||
DEFERRED_CALLS = frozenset({"invalidate_gallery_cache"})
|
||||
# Linux's shared-memory tmpfs (present in the Playwright image and on runners).
|
||||
MEMORY_ROOT = Path("/dev/shm")
|
||||
# The largest fixtures' scratch copy is about 8 MB (database plus a WAL that
|
||||
# autocheckpoints near 4 MB). Docker's default /dev/shm is 64 MB, shared with
|
||||
# the browsers and the other workers; with less free than this, the scratch
|
||||
# copy goes on disk instead.
|
||||
MEMORY_FREE_BYTES = 32 * 1024 * 1024
|
||||
# Scratch folders are named <prefix><owner pid>-<random>.
|
||||
SCRATCH_PREFIX = "pm-reset-"
|
||||
|
||||
|
||||
def _quote(name: str) -> str:
|
||||
return '"' + name.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def _tables(conn: sqlite3.Connection, schema: str = "main") -> list[str]:
|
||||
rows = conn.execute(
|
||||
f"SELECT name, sql FROM {schema}.sqlite_master WHERE type = 'table' "
|
||||
"ORDER BY name"
|
||||
).fetchall()
|
||||
for name, sql in rows:
|
||||
if sql and sql.lstrip().upper().startswith("CREATE VIRTUAL"):
|
||||
raise NotImplementedError(f"virtual table {name} cannot be copied")
|
||||
return [name for name, _sql in rows]
|
||||
|
||||
|
||||
def _columns(conn: sqlite3.Connection, table: str, schema: str = "main") -> list[str]:
|
||||
"""Writable columns (hidden and generated ones excluded), plus ``rowid``
|
||||
when the table has a rowid that no INTEGER PRIMARY KEY column carries."""
|
||||
info = conn.execute(f"PRAGMA {schema}.table_xinfo({_quote(table)})").fetchall()
|
||||
columns = [row[1] for row in info if row[6] == 0]
|
||||
keys = [row for row in info if row[5]]
|
||||
sql = conn.execute(
|
||||
f"SELECT sql FROM {schema}.sqlite_master WHERE type = 'table' AND name = ?",
|
||||
(table,),
|
||||
).fetchone()[0]
|
||||
without_rowid = sql is not None and "WITHOUT ROWID" in sql.upper()
|
||||
integer_key = len(keys) == 1 and str(keys[0][2]).upper() == "INTEGER"
|
||||
if not without_rowid and not integer_key and table != "sqlite_sequence":
|
||||
columns.insert(0, "rowid")
|
||||
return columns
|
||||
|
||||
|
||||
def table_digests(conn: sqlite3.Connection, schema: str = "main") -> dict[str, str]:
|
||||
"""A digest of every table's rows (rowids included), by table name."""
|
||||
result = {}
|
||||
for table in _tables(conn, schema):
|
||||
selected = ", ".join(
|
||||
_quote(c) if c != "rowid" else "rowid"
|
||||
for c in _columns(conn, table, schema)
|
||||
)
|
||||
digest = hashlib.sha256()
|
||||
rows = conn.execute(f"SELECT {selected} FROM {schema}.{_quote(table)}")
|
||||
for row in sorted(repr(tuple(row)) for row in rows):
|
||||
digest.update(row.encode("utf-8", "surrogatepass"))
|
||||
digest.update(b"\n")
|
||||
result[table] = digest.hexdigest()
|
||||
return result
|
||||
|
||||
|
||||
def _process_alive(pid: int) -> bool:
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except OSError:
|
||||
return True # it exists but belongs to someone else
|
||||
return True
|
||||
|
||||
|
||||
def remove_orphaned_scratch(root: Path) -> None:
|
||||
"""Remove scratch folders of servers that died without cleaning up.
|
||||
|
||||
A server killed outright (SIGKILL, a crash) never runs ``close``, and
|
||||
tmpfs keeps its folder in memory until reboot. The folder name carries
|
||||
the owner's process id; a folder whose process is gone is removed.
|
||||
"""
|
||||
if os.name != "posix":
|
||||
return # os.kill(pid, 0) would terminate the process on Windows
|
||||
for folder in root.glob(f"{SCRATCH_PREFIX}*-*"):
|
||||
pid = folder.name[len(SCRATCH_PREFIX) :].split("-", 1)[0]
|
||||
if pid.isdigit() and not _process_alive(int(pid)):
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
|
||||
|
||||
class _SeedApi:
|
||||
"""The live API, except that ``db`` is the scratch database.
|
||||
|
||||
Cache drops the seed asks for (``DEFERRED_CALLS``) are recorded and made
|
||||
after the publish: dropped earlier, a request in between would cache the
|
||||
live database's old rows again.
|
||||
"""
|
||||
|
||||
def __init__(self, api: Any, db: Any) -> None:
|
||||
object.__setattr__(self, "_api", api)
|
||||
object.__setattr__(self, "db", db)
|
||||
object.__setattr__(self, "deferred", [])
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
if name in DEFERRED_CALLS:
|
||||
|
||||
def record(*args: Any, **kwargs: Any) -> None:
|
||||
self.deferred.append((name, args, kwargs))
|
||||
|
||||
return record
|
||||
return getattr(self._api, name)
|
||||
|
||||
def __setattr__(self, name: str, value: Any) -> None:
|
||||
setattr(self._api, name, value)
|
||||
|
||||
|
||||
class ScratchSeeder:
|
||||
"""Seed into a scratch copy and publish the changed tables in one commit."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api: Any,
|
||||
directory: Path,
|
||||
*,
|
||||
on_live: bool = False,
|
||||
memory_root: Path | None = MEMORY_ROOT,
|
||||
) -> None:
|
||||
self._api = api
|
||||
self._fallback = Path(directory)
|
||||
self._memory_root = memory_root
|
||||
self._directory: Path | None = None
|
||||
self._on_live = on_live
|
||||
self._scratch = None
|
||||
self._unsynced: "weakref.WeakSet[sqlite3.Connection]" = weakref.WeakSet()
|
||||
|
||||
def run(self, seed: Callable[[Any], T]) -> T:
|
||||
"""Run ``seed(api)`` (blocking; call it in an executor thread)."""
|
||||
if self._on_live:
|
||||
return seed(self._api)
|
||||
live = self._api.db.model.get_connection()
|
||||
if live.in_transaction:
|
||||
raise RuntimeError("the live connection has an open transaction")
|
||||
version = live.execute("PRAGMA data_version").fetchone()[0]
|
||||
scratch = self._scratch_database()
|
||||
target = self._scratch_connection(scratch)
|
||||
with target.using():
|
||||
live.backup(target)
|
||||
before = table_digests(target)
|
||||
target_api = _SeedApi(self._api, scratch)
|
||||
result = seed(target_api)
|
||||
target = self._scratch_connection(scratch)
|
||||
if target.in_transaction:
|
||||
raise RuntimeError("the seed left a scratch transaction open")
|
||||
after = table_digests(target)
|
||||
changed = [table for table in after if after[table] != before.get(table)]
|
||||
if set(before) != set(after):
|
||||
raise RuntimeError("the seed changed the database schema")
|
||||
if live.execute("PRAGMA data_version").fetchone()[0] != version:
|
||||
raise RuntimeError("the live database changed while the seed ran")
|
||||
self._publish(live, scratch.model.db_path, changed, version)
|
||||
for name, args, kwargs in target_api.deferred:
|
||||
getattr(self._api, name)(*args, **kwargs)
|
||||
return result
|
||||
|
||||
def _scratch_database(self):
|
||||
if self._scratch is None:
|
||||
from database.operations import PromptDatabase
|
||||
|
||||
self._directory = self._scratch_directory()
|
||||
self._scratch = PromptDatabase(str(self._directory / "seed.db"))
|
||||
return self._scratch
|
||||
|
||||
def _scratch_connection(self, scratch) -> sqlite3.Connection:
|
||||
"""The calling thread's scratch connection, set never to sync."""
|
||||
conn = scratch.model.get_connection()
|
||||
if conn not in self._unsynced:
|
||||
conn.execute("PRAGMA synchronous = OFF")
|
||||
self._unsynced.add(conn)
|
||||
return conn
|
||||
|
||||
@staticmethod
|
||||
def _publish(
|
||||
live: sqlite3.Connection, path: str, tables: list[str], version: int
|
||||
) -> None:
|
||||
"""Replace ``tables`` with the scratch rows in one live transaction.
|
||||
|
||||
``version`` is the live ``data_version`` read before the seed. It is
|
||||
checked again under the write lock: a writer whose transaction was
|
||||
open during the seed commits before ``BEGIN IMMEDIATE`` returns, and
|
||||
its rows must fail the reset instead of being overwritten.
|
||||
"""
|
||||
if not tables:
|
||||
return
|
||||
foreign_keys = live.execute("PRAGMA foreign_keys").fetchone()[0]
|
||||
# Rows are copied as they are; no cascade may reach a table left alone.
|
||||
live.execute("PRAGMA foreign_keys = OFF")
|
||||
try:
|
||||
live.execute(f"ATTACH DATABASE ? AS {SCRATCH_SCHEMA}", (path,))
|
||||
try:
|
||||
live.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
if live.execute("PRAGMA data_version").fetchone()[0] != version:
|
||||
raise RuntimeError(
|
||||
"the live database changed while the seed ran"
|
||||
)
|
||||
for table in tables:
|
||||
columns = ", ".join(
|
||||
_quote(c) if c != "rowid" else "rowid"
|
||||
for c in _columns(live, table)
|
||||
)
|
||||
name = _quote(table)
|
||||
live.execute(f"DELETE FROM main.{name}")
|
||||
live.execute(
|
||||
f"INSERT INTO main.{name} ({columns}) "
|
||||
f"SELECT {columns} FROM {SCRATCH_SCHEMA}.{name}"
|
||||
)
|
||||
live.commit()
|
||||
except BaseException:
|
||||
live.rollback()
|
||||
raise
|
||||
finally:
|
||||
live.execute(f"DETACH DATABASE {SCRATCH_SCHEMA}")
|
||||
finally:
|
||||
live.execute(f"PRAGMA foreign_keys = {int(foreign_keys)}")
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close every scratch connection, then remove the scratch files."""
|
||||
if self._on_live:
|
||||
return
|
||||
if self._scratch is not None:
|
||||
self._scratch.close_all()
|
||||
self._scratch = None
|
||||
if self._directory is not None:
|
||||
shutil.rmtree(self._directory, ignore_errors=True)
|
||||
self._directory = None
|
||||
|
||||
def _scratch_directory(self) -> Path:
|
||||
"""A folder in memory (tmpfs) where there is one, else ``directory``.
|
||||
|
||||
Even unsynced, scratch writes on the runner's disk wait whenever the
|
||||
kernel is writing back the pages they rewrite.
|
||||
"""
|
||||
root = self._memory_root
|
||||
if root is not None and root.is_dir() and os.access(root, os.W_OK):
|
||||
remove_orphaned_scratch(root)
|
||||
if shutil.disk_usage(root).free >= MEMORY_FREE_BYTES:
|
||||
prefix = f"{SCRATCH_PREFIX}{os.getpid()}-"
|
||||
return Path(tempfile.mkdtemp(prefix=prefix, dir=root))
|
||||
self._fallback.mkdir(parents=True, exist_ok=True)
|
||||
return self._fallback
|
||||
@@ -1,78 +0,0 @@
|
||||
"""Empty a browser-test media folder without waiting for the disk (#384).
|
||||
|
||||
A reset used to ``shutil.rmtree`` the output and input folders the last
|
||||
fixture wrote. Unlinking a file whose pages are still being written back
|
||||
waits for that write, so on a slow CI disk a reset stalled in ``rmtree``.
|
||||
``replace_folder`` renames the old folder into ``<parent>/reset-trash`` (one
|
||||
metadata change) and makes a new empty one; a background thread deletes the
|
||||
trash, and ``close_trash`` stops that thread and removes what is left before
|
||||
the server's temp root goes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import shutil
|
||||
import threading
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
TRASH_NAME = "reset-trash"
|
||||
|
||||
|
||||
class _Trash:
|
||||
def __init__(self, directory: Path) -> None:
|
||||
self.directory = directory
|
||||
self._queue: "queue.SimpleQueue[Path | None]" = queue.SimpleQueue()
|
||||
self._thread = threading.Thread(
|
||||
target=self._work, name="pm-reset-trash", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def _work(self) -> None:
|
||||
while (folder := self._queue.get()) is not None:
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
|
||||
def discard(self, folder: Path) -> None:
|
||||
self.directory.mkdir(exist_ok=True)
|
||||
moved = self.directory / f"{folder.name}-{uuid.uuid4().hex}"
|
||||
folder.rename(moved)
|
||||
self._queue.put(moved)
|
||||
|
||||
def close(self) -> None:
|
||||
self._queue.put(None)
|
||||
self._thread.join()
|
||||
shutil.rmtree(self.directory, ignore_errors=True)
|
||||
|
||||
|
||||
_trashes: dict[Path, _Trash] = {}
|
||||
# Trash folders whose server has cleaned up: a reset still running then
|
||||
# deletes in place instead of starting a thread nobody would stop.
|
||||
_closed: set[Path] = set()
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def replace_folder(folder: Path) -> None:
|
||||
"""Make ``folder`` an empty folder; its old content is deleted later."""
|
||||
folder = Path(folder)
|
||||
if folder.exists():
|
||||
directory = folder.parent / TRASH_NAME
|
||||
with _lock:
|
||||
trash = _trashes.get(directory)
|
||||
if trash is None and directory not in _closed:
|
||||
trash = _trashes[directory] = _Trash(directory)
|
||||
if trash is not None:
|
||||
trash.discard(folder)
|
||||
if folder.exists():
|
||||
shutil.rmtree(folder)
|
||||
folder.mkdir()
|
||||
|
||||
|
||||
def close_trash(parent: Path) -> None:
|
||||
"""Stop deleting under ``parent``'s trash and remove it (blocking)."""
|
||||
directory = Path(parent) / TRASH_NAME
|
||||
with _lock:
|
||||
_closed.add(directory)
|
||||
trash = _trashes.pop(directory, None)
|
||||
if trash is not None:
|
||||
trash.close()
|
||||
@@ -95,14 +95,8 @@ async function watchProgress(page) {
|
||||
await page.evaluate(() => {
|
||||
window.progressSeen = [];
|
||||
const node = document.getElementById('autoTagProgressPercent');
|
||||
// Each change from its own record: frames that land in one task share one
|
||||
// callback, and the node's text by then is only the last of them.
|
||||
new MutationObserver(records => {
|
||||
for (const record of records) {
|
||||
if (record.type === 'characterData') window.progressSeen.push(record.target.data);
|
||||
for (const added of record.addedNodes) window.progressSeen.push(added.textContent);
|
||||
}
|
||||
}).observe(node, { childList: true, characterData: true, subtree: true });
|
||||
new MutationObserver(() => window.progressSeen.push(node.textContent))
|
||||
.observe(node, { childList: true, characterData: true, subtree: true });
|
||||
});
|
||||
}
|
||||
|
||||
@@ -114,58 +108,25 @@ async function runGalleryBatch(page) {
|
||||
await page.locator('#startAutoTagBtn').click();
|
||||
await done;
|
||||
await expect(page.locator('#autoTagProgressModal')).toBeHidden();
|
||||
// #379: the batch counts images, never prompts.
|
||||
await expect(page.getByText(/^Auto tagging complete! Tagged \d+ images?\.$/)).toBeVisible();
|
||||
}
|
||||
|
||||
const CLOSE_NOTE = 'Closing this window stops tagging. Tags already added are kept. Prompt Manager\'s Activity page shows the '
|
||||
+ 'time left.';
|
||||
|
||||
async function serverJobs(request) {
|
||||
const response = await request.get('/prompt_manager/jobs');
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).jobs;
|
||||
}
|
||||
|
||||
async function runNativeBatch(request) {
|
||||
const started = await request.post('/prompt_manager/jobs/autotag', { data: { model_type: 'wd14-swinv2', existing: 'skip',
|
||||
options: { general_threshold: 0.35, character_threshold: 0.85 } } });
|
||||
expect(started.status()).toBe(202);
|
||||
const { job } = await started.json();
|
||||
await expect.poll(async () => (await serverJobs(request)).find(item => item.id === job.id)?.state).toBe('finished');
|
||||
await expect(page.getByText(/^Auto tagging complete! Applied tags to \d+ prompts\.$/)).toBeVisible();
|
||||
}
|
||||
|
||||
test('gallery batch modal counts progress and closes on complete', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await runGalleryBatch(page);
|
||||
const seen = await page.evaluate(() => window.progressSeen);
|
||||
// The phase frames (model 5%, listing 8%) then the per-image frames (from 10%) move the bar.
|
||||
expect(seen).toContain('8%');
|
||||
expect(seen.some(value => parseInt(value, 10) >= 10)).toBe(true);
|
||||
await expect(page.locator('#autoTagCloseNote')).toHaveText(CLOSE_NOTE);
|
||||
// The batch ran as the autotag job of the legacy page.
|
||||
expect((await serverJobs(request)).filter(job => job.kind === 'autotag').map(job => [job.title, job.state]))
|
||||
.toEqual([['Tag output images (legacy page)', 'finished']]);
|
||||
// The fixture's prompts are already tagged, so the batch skips them; its progress frames still move the bar.
|
||||
expect(seen).toContain('95%');
|
||||
expect((await downloads(request)).length).toBe(0);
|
||||
});
|
||||
|
||||
test('admin batch copy names images, not prompts (#379)', async ({ page }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await expect(page.locator('input[name="autoTagMode"][value="skip"] + span')).toHaveText('Skip images already auto-tagged');
|
||||
await expect(page.getByText('"auto-scanned" is not counted as a real tag')).toHaveCount(0);
|
||||
await expect(page.locator('#autoTagCloseNote')).toHaveText(CLOSE_NOTE);
|
||||
await expect(page.locator('#retagConfirmModal h3')).toHaveText('Image Already Auto-tagged');
|
||||
await expect(page.locator('#retagConfirmModal')).toContainText('A tagging model already tagged this image. Generate new tags?');
|
||||
});
|
||||
|
||||
test('admin review opens the first image', async ({ page }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check();
|
||||
await page.locator('#startReviewBtn').click();
|
||||
// #379: no image is auto-tagged yet (prompt tags no longer count), so the review opens on the first image.
|
||||
await expect(page.locator('#retagConfirmModal')).toBeHidden();
|
||||
// The first image's prompt is already tagged: confirm a new pass, then the review opens on that image.
|
||||
await page.locator('#retagConfirmBtn').click();
|
||||
await expect(page.locator('#autoTagReviewModal')).toBeVisible();
|
||||
await expect(page.locator('#reviewImageName')).not.toHaveText(/^(image_name\.png|Unknown|)$/);
|
||||
await expect(page.getByText('Failed to start review mode')).toHaveCount(0);
|
||||
@@ -189,70 +150,3 @@ test('gallery unload works and is refused while the gate is held', async ({ page
|
||||
await unload.click();
|
||||
await expect(page.getByText('WD14-SWINV2 model unloaded successfully')).toBeVisible();
|
||||
});
|
||||
|
||||
/** The admin review lists every row, videos too: Skip any image that is not one until the confirmation shows. */
|
||||
async function untilAutoTagged(page, confirm) {
|
||||
// Each image's skip check answers before its confirmation or its error shows.
|
||||
const failed = page.locator('#reviewTagsVisible .text-pm-error');
|
||||
for (let tries = 0; tries < 5; tries++) {
|
||||
await expect.poll(async () => (await confirm.isVisible()) || (await failed.count()) > 0).toBe(true);
|
||||
if (await confirm.isVisible()) return;
|
||||
const answered = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/single'));
|
||||
await page.locator('#skipReviewBtn').click();
|
||||
await answered;
|
||||
}
|
||||
await expect(confirm).toBeVisible();
|
||||
}
|
||||
|
||||
// #379: the admin review asks before re-tagging an image a model already tagged, and Apply & Next writes that image's
|
||||
// tags (its bytes, never its prompt).
|
||||
test('admin review: an auto-tagged image asks first; Re-tag then Apply & Next tags the image', async ({ page, request }) => {
|
||||
await runNativeBatch(request);
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check();
|
||||
await page.locator('#startReviewBtn').click();
|
||||
const confirm = page.locator('#retagConfirmModal');
|
||||
await untilAutoTagged(page, confirm);
|
||||
await expect(confirm.locator('#retagExistingTags span').first()).toHaveText('sandstone');
|
||||
await page.locator('#retagConfirmBtn').click();
|
||||
await expect(page.locator('#autoTagReviewModal')).toBeVisible();
|
||||
await expect(page.locator('#reviewTagsVisible .tag-chip').first()).toContainText('sandstone');
|
||||
const applied = page.waitForRequest(item => item.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
const answer = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
await page.locator('#applyReviewBtn').click();
|
||||
const body = (await applied).postDataJSON();
|
||||
expect(body.prompt_id).toBeUndefined();
|
||||
expect(body.sha256).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect((await answer).status()).toBe(200);
|
||||
});
|
||||
|
||||
test('admin review: Skip All Tagged passes every auto-tagged image', async ({ page, request }) => {
|
||||
await runNativeBatch(request);
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check();
|
||||
await page.locator('#startReviewBtn').click();
|
||||
const confirm = page.locator('#retagConfirmModal');
|
||||
await untilAutoTagged(page, confirm);
|
||||
await page.locator('#retagSkipAllBtn').click();
|
||||
await expect(page.locator('#retagConfirmModal')).toBeHidden();
|
||||
await expect(page.getByText('Review complete!')).toBeVisible();
|
||||
});
|
||||
|
||||
test('gallery review: Apply & Next tags the image by its path and bytes', async ({ page }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check();
|
||||
const suggested = page.waitForRequest(item => item.url().endsWith('/prompt_manager/autotag/single'));
|
||||
await page.locator('#startReviewBtn').click();
|
||||
expect((await suggested).postDataJSON().existing).toBe('merge');
|
||||
await expect(page.locator('#reviewTagsContainer .tag-chip').first()).toContainText('sandstone');
|
||||
const applied = page.waitForRequest(item => item.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
const answer = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
await page.locator('#applyReviewBtn').click();
|
||||
const body = (await applied).postDataJSON();
|
||||
expect(Object.keys(body).sort()).toEqual(['path', 'root_index', 'sha256', 'tags'].filter(key => key in body).sort());
|
||||
expect(body.prompt_id).toBeUndefined();
|
||||
expect((await answer).status()).toBe(200);
|
||||
});
|
||||
|
||||
@@ -1,163 +0,0 @@
|
||||
// System S13 (#330): the legacy admin and gallery pages open native System for their System tools. Each replaced
|
||||
// control is a plain link (`workspace#/system/<id>`, a new tab), and the legacy flows S13 keeps still work.
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
const PAGES = Object.freeze({ admin: '/prompt_manager/admin', gallery: '/prompt_manager/gallery' });
|
||||
const LABELS = Object.freeze({
|
||||
overview: 'Overview',
|
||||
appearance: 'Appearance',
|
||||
sources: 'Sources & scans',
|
||||
integrations: 'Integrations',
|
||||
tagging: 'Auto-tagging',
|
||||
maintenance: 'Maintenance',
|
||||
backups: 'Backup & export',
|
||||
diagnostics: 'Diagnostics & logs',
|
||||
});
|
||||
|
||||
// One row per System link of the design's §2 map: the page, the control's label, how to find it, the section it opens
|
||||
// and what a test must arrange before the control shows.
|
||||
const LINKS = Object.freeze([
|
||||
{ page: 'admin', label: 'Diagnostics', locate: page => page.locator('#diagnosticsBtn'), section: 'diagnostics' },
|
||||
{ page: 'admin', label: 'View Logs', locate: page => page.locator('#logsBtn'), section: 'diagnostics' },
|
||||
]);
|
||||
|
||||
async function post(request, url, data) {
|
||||
await expect(await request.post(url, data === undefined ? {} : { data })).toBeOK();
|
||||
}
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await post(request, '/__test__/reset');
|
||||
await post(request, '/__test__/system/reset');
|
||||
await post(request, '/__test__/tagging/reset');
|
||||
// The 3.2.0 update notice would cover the admin page's toolbar.
|
||||
await page.addInitScript(() => { localStorage.setItem('pm_update_notice_dismissed', '3.2.0'); });
|
||||
page.legacyErrors = [];
|
||||
page.on('pageerror', error => page.legacyErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await post(request, '/__test__/tagging/reset');
|
||||
await post(request, '/__test__/system/reset');
|
||||
expect(page.legacyErrors).toEqual([]);
|
||||
});
|
||||
|
||||
/** Opens the legacy page with the link showing; returns the link. */
|
||||
async function openLink(page, request, link) {
|
||||
if (link.arrange) await link.arrange(page, request);
|
||||
await page.goto(PAGES[link.page]);
|
||||
if (link.reveal) await link.reveal(page);
|
||||
const control = link.locate(page);
|
||||
await expect(control).toBeVisible();
|
||||
return control;
|
||||
}
|
||||
|
||||
for (const link of LINKS) {
|
||||
test(`legacy ${link.page} ${link.label} opens System ${link.section}`, async ({ page, context, request }) => {
|
||||
const control = await openLink(page, request, link);
|
||||
const legacyUrl = page.url();
|
||||
await expect(control).toHaveAttribute('href', `workspace#/system/${link.section}`);
|
||||
await expect(control).toHaveAttribute('target', '_blank');
|
||||
await expect(control).toHaveAttribute('rel', 'noopener');
|
||||
const opened = context.waitForEvent('page');
|
||||
await control.click();
|
||||
const system = await opened;
|
||||
await expect(system).toHaveURL(new RegExp(`/prompt_manager/workspace#/system/${link.section}$`));
|
||||
// System's page heading is "System"; the open section is the pressed button of its section switcher.
|
||||
const main = system.locator('#workspace-main');
|
||||
await expect(main.getByRole('heading', { level: 1, name: 'System', exact: true })).toBeVisible();
|
||||
await expect(main.getByRole('group', { name: 'System sections', exact: true })
|
||||
.getByRole('button', { name: LABELS[link.section], exact: true })).toHaveAttribute('aria-pressed', 'true');
|
||||
expect(page.url()).toBe(legacyUrl);
|
||||
await system.close();
|
||||
});
|
||||
}
|
||||
|
||||
test('a System link sends no request from the legacy page', async ({ page, context, request }) => {
|
||||
for (const link of LINKS) {
|
||||
const control = await openLink(page, request, link);
|
||||
// The page's own start-up reads (list, film strips, settings) finish first; then the click must add none.
|
||||
await page.waitForLoadState('networkidle');
|
||||
const sent = [];
|
||||
const record = item => sent.push(`${item.method()} ${item.url()}`);
|
||||
page.on('request', record);
|
||||
const opened = context.waitForEvent('page');
|
||||
await control.click();
|
||||
const system = await opened;
|
||||
await system.waitForLoadState('domcontentloaded');
|
||||
page.off('request', record);
|
||||
expect(sent, `${link.page} ${link.label}`).toEqual([]);
|
||||
await system.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('kept admin flows still work', async ({ page, context }) => {
|
||||
await page.goto(PAGES.admin);
|
||||
const texts = page.locator('#resultsList .prompt-text');
|
||||
await expect(texts.first()).toBeVisible();
|
||||
|
||||
// K1: sorting reorders the list.
|
||||
for (const [value, order] of [['text_asc', 1], ['text_desc', -1]]) {
|
||||
const listed = page.waitForResponse(item => /\/prompt_manager\/(recent|search)\?/.test(item.url()) && item.url().includes(`sort=${value}`));
|
||||
await page.locator('#sortBy').selectOption(value);
|
||||
await listed;
|
||||
await expect.poll(async () => {
|
||||
const shown = await texts.allTextContents();
|
||||
// The server sorts case-insensitively (SQLite NOCASE), not by locale.
|
||||
const key = text => text.toLowerCase();
|
||||
const sorted = [...shown].sort((a, b) => order * (key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0));
|
||||
return shown.length > 1 && shown.join('\n') === sorted.join('\n');
|
||||
}).toBe(true);
|
||||
}
|
||||
// K3: Add Prompt saves a prompt the list then shows.
|
||||
await page.locator('#addPromptBtn').click();
|
||||
await page.locator('#addPromptText').fill('Kept flow lighthouse');
|
||||
const saved = page.waitForResponse(item => item.request().method() === 'POST' && item.url().endsWith('/prompt_manager/save'));
|
||||
await page.locator('#saveNewPromptBtn').click();
|
||||
expect((await saved).ok()).toBe(true);
|
||||
await expect(page.locator('#addPromptModal')).toBeHidden();
|
||||
await expect(texts.filter({ hasText: 'Kept flow lighthouse' })).toBeVisible();
|
||||
|
||||
// K1: a search narrows the list, and a selection enables the bulk buttons.
|
||||
const all = await texts.allTextContents();
|
||||
const word = all[0].trim().split(/\s+/)[0];
|
||||
const searched = page.waitForResponse(item => item.url().includes('/prompt_manager/search?') && item.url().includes('text='));
|
||||
await page.locator('#searchText').fill(word);
|
||||
await page.locator('#searchBtn').click();
|
||||
await searched;
|
||||
await expect.poll(async () => {
|
||||
const shown = await texts.allTextContents();
|
||||
return shown.length > 0 && shown.length < all.length && shown.every(text => text.toLowerCase().includes(word.toLowerCase()));
|
||||
}).toBe(true);
|
||||
await expect(page.locator('#bulkDeleteBtn')).toBeDisabled();
|
||||
await page.locator('#resultsList .prompt-checkbox').first().check();
|
||||
for (const id of ['#bulkDeleteBtn', '#bulkTagBtn', '#bulkCategoryBtn']) await expect(page.locator(id)).toBeEnabled();
|
||||
|
||||
// K4: Stats reads the statistics again; Export downloads the prompts.
|
||||
const stats = page.waitForResponse(item => item.url().endsWith('/prompt_manager/stats') && item.ok());
|
||||
await page.locator('#statsBtn').click();
|
||||
await stats;
|
||||
await expect(page.getByText('Stats refreshed successfully!')).toBeVisible();
|
||||
const download = page.waitForEvent('download');
|
||||
await page.locator('#exportBtn').click();
|
||||
expect((await download).suggestedFilename()).toMatch(/^prompts-\d{4}-\d{2}-\d{2}\.json$/);
|
||||
|
||||
// K5: Scan Images opens its dialog (with the quick backup) without starting a scan.
|
||||
await page.locator('#scanBtn').click();
|
||||
await expect(page.locator('#scanModal')).toBeVisible();
|
||||
await expect(page.locator('#quickBackupBtn')).toBeVisible();
|
||||
await page.locator('#cancelScan').click();
|
||||
await expect(page.locator('#scanModal')).toBeHidden();
|
||||
|
||||
// K7: Metadata and Gallery open their pages in a new tab.
|
||||
for (const [id, path] of [['#metadataBtn', '/prompt_manager/gallery.html'], ['#galleryBtn', '/prompt_manager/gallery']]) {
|
||||
const opened = context.waitForEvent('page');
|
||||
await page.locator(id).click();
|
||||
const tab = await opened;
|
||||
await tab.waitForLoadState('domcontentloaded');
|
||||
expect(new URL(tab.url()).pathname).toBe(path);
|
||||
await tab.close();
|
||||
}
|
||||
});
|
||||
@@ -1,279 +0,0 @@
|
||||
"""Prove the scratch-seeded resets leave the state the old resets left (#384).
|
||||
|
||||
Run by hand on Linux (the browser server's platform):
|
||||
|
||||
python -m tests.browser.reset_equivalence
|
||||
|
||||
It starts the browser server three times: twice seeding on the live database
|
||||
as before #384 (``PM_TEST_SEED_ON_LIVE=1``) and once with the scratch seeder.
|
||||
Each server gets the same sequence: every reset kind of ``/__test__/reset``
|
||||
and ``/__test__/concept/reset``, with other fixtures' resets and API writes
|
||||
between them. After every reset it reads every table's rows (by rowid), the
|
||||
output and input file trees (size, SHA-256, mtime) and the response body.
|
||||
|
||||
Values that differ between the two old runs are wall-clock values (created_at,
|
||||
an unpinned concept "now", file mtimes); they are masked. Everything else must
|
||||
be equal between the old runs and the new one. Exit status 0 means equal.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
READY_TIMEOUT_S = 60
|
||||
CLOCK = "<clock>"
|
||||
|
||||
RESETS = [
|
||||
("/__test__/reset", {}),
|
||||
("/__test__/reset", {"extra_images": 2}),
|
||||
("/__test__/reset", {"extra_images": 108}),
|
||||
("/__test__/reset", {"include_video": True}),
|
||||
("/__test__/reset", {"prompt_inspector": True}),
|
||||
("/__test__/reset", {"native_library": True}),
|
||||
("/__test__/reset", {"native_library": True, "extra_prompts": 125}),
|
||||
("/__test__/reset", {"native_library": True, "extra_variants": 41}),
|
||||
("/__test__/reset", {"workspace_visual": True}),
|
||||
("/__test__/reset", {"full_inspector": True}),
|
||||
(
|
||||
"/__test__/reset",
|
||||
{"full_inspector": True, "include_video": True, "extra_images": 2},
|
||||
),
|
||||
("/__test__/reset", {"chronological_history": True}),
|
||||
("/__test__/reset", {"chronological_history": True, "history_runs": 85}),
|
||||
("/__test__/concept/reset", {}),
|
||||
("/__test__/concept/reset", {"extra_images": 100}),
|
||||
("/__test__/concept/reset", {"seedless_images": 2, "short_seedless": True}),
|
||||
("/__test__/concept/reset", {"now": 1_780_000_000}),
|
||||
]
|
||||
# Writes by other fixtures and the API between resets; the resets must keep or
|
||||
# replace them exactly as before.
|
||||
DIRTY = [
|
||||
("/__test__/tagging/reset", {}),
|
||||
("/__test__/system/reset", None),
|
||||
(
|
||||
"/__test__/saved-views/raw",
|
||||
{"page": "images", "name": "dirty", "state": "{}"},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class Server:
|
||||
def __init__(self, on_live: bool) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory(prefix="pm-equivalence-")
|
||||
env = {**os.environ, "TMPDIR": self.tmp.name}
|
||||
env.update(TEMP=self.tmp.name, TMP=self.tmp.name)
|
||||
env.pop("PM_TEST_SEED_ON_LIVE", None)
|
||||
if on_live:
|
||||
env["PM_TEST_SEED_ON_LIVE"] = "1"
|
||||
self.process = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"tests.browser.server",
|
||||
"--port",
|
||||
"0",
|
||||
"--exit-on-stdin-eof",
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
line = self.process.stdout.readline().decode()
|
||||
if not line.startswith("PM_TEST_SERVER_PORT="):
|
||||
raise RuntimeError(f"no port line: {line!r}")
|
||||
self.base = f"http://127.0.0.1:{int(line.split('=', 1)[1])}"
|
||||
deadline = time.monotonic() + READY_TIMEOUT_S
|
||||
while True:
|
||||
try:
|
||||
self.post_json("GET", "/prompt_manager/test", None)
|
||||
break
|
||||
except OSError:
|
||||
if time.monotonic() > deadline or self.process.poll() is not None:
|
||||
raise
|
||||
time.sleep(0.1)
|
||||
(self.root,) = Path(self.tmp.name).glob("pm-browser-*")
|
||||
|
||||
def post_json(self, method: str, path: str, data) -> dict:
|
||||
body = None if data is None else json.dumps(data).encode()
|
||||
request = urllib.request.Request(
|
||||
self.base + path,
|
||||
data=body if method == "POST" else None,
|
||||
method=method,
|
||||
headers={"Content-Type": "application/json"} if body else {},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=120) as response: # nosec B310
|
||||
text = response.read().decode()
|
||||
try:
|
||||
return json.loads(text)
|
||||
except ValueError:
|
||||
return {"text": text[:200]}
|
||||
|
||||
def stop(self) -> None:
|
||||
self.process.stdin.close()
|
||||
self.process.wait(timeout=READY_TIMEOUT_S)
|
||||
self.process.stdout.close()
|
||||
self.tmp.cleanup()
|
||||
|
||||
|
||||
def read_tables(path: Path) -> dict:
|
||||
conn = sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)
|
||||
try:
|
||||
tables = {}
|
||||
names = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
|
||||
)
|
||||
]
|
||||
for name in names:
|
||||
quoted = '"' + name.replace('"', '""') + '"'
|
||||
columns = [row[1] for row in conn.execute(f"PRAGMA table_info({quoted})")]
|
||||
try:
|
||||
cursor = conn.execute(f"SELECT rowid, * FROM {quoted} ORDER BY rowid")
|
||||
columns = ["rowid", *columns]
|
||||
except sqlite3.OperationalError:
|
||||
cursor = conn.execute(f"SELECT * FROM {quoted} ORDER BY 1")
|
||||
rows = [
|
||||
[value.hex() if isinstance(value, bytes) else value for value in row]
|
||||
for row in cursor
|
||||
]
|
||||
tables[name] = {"columns": columns, "rows": rows}
|
||||
return tables
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def read_files(root: Path) -> dict:
|
||||
files = {}
|
||||
for top in ("output", "input"):
|
||||
for path in sorted((root / top).rglob("*")):
|
||||
if path.is_file():
|
||||
stat = path.stat()
|
||||
files[path.relative_to(root).as_posix()] = [
|
||||
stat.st_size,
|
||||
hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
stat.st_mtime_ns,
|
||||
]
|
||||
return files
|
||||
|
||||
|
||||
def flatten(value, prefix="") -> dict:
|
||||
if isinstance(value, dict):
|
||||
out = {}
|
||||
for key, item in value.items():
|
||||
out.update(flatten(item, f"{prefix}/{key}"))
|
||||
return out
|
||||
if isinstance(value, list):
|
||||
out = {f"{prefix}#len": len(value)}
|
||||
for index, item in enumerate(value):
|
||||
out.update(flatten(item, f"{prefix}[{index}]"))
|
||||
return out
|
||||
return {prefix: value}
|
||||
|
||||
|
||||
def snapshot(server: Server, response: dict) -> dict:
|
||||
flat = {}
|
||||
for table, data in read_tables(server.root / "browser.db").items():
|
||||
flat[f"table {table}#rows"] = len(data["rows"])
|
||||
for index, row in enumerate(data["rows"]):
|
||||
for column, value in zip(data["columns"], row):
|
||||
flat[f"table {table}[{index}].{column}"] = value
|
||||
for name, (size, digest, mtime) in read_files(server.root).items():
|
||||
flat[f"file {name}"] = [size, digest]
|
||||
flat[f"file {name}#mtime"] = mtime
|
||||
flat["files#count"] = sum(
|
||||
1 for key in flat if key.startswith("file ") and "#" not in key
|
||||
)
|
||||
for key, value in flatten(response).items():
|
||||
flat[f"response{key}"] = value
|
||||
# Each server has its own temp root and a random storage epoch; name them
|
||||
# instead of masking every column that carries them.
|
||||
epoch = flat.get("table execution_history_metadata[0].storage_epoch")
|
||||
names = [(str(server.root), "<root>")]
|
||||
if isinstance(epoch, str) and epoch:
|
||||
names.append((epoch, "<epoch>"))
|
||||
for key, value in flat.items():
|
||||
if isinstance(value, str):
|
||||
for text, name in names:
|
||||
value = value.replace(text, name)
|
||||
flat[key] = value
|
||||
return flat
|
||||
|
||||
|
||||
def run(on_live: bool) -> list[dict]:
|
||||
server = Server(on_live)
|
||||
try:
|
||||
states = []
|
||||
for index, (path, data) in enumerate(RESETS * 2):
|
||||
dirty_path, dirty_data = DIRTY[index % len(DIRTY)]
|
||||
server.post_json("POST", dirty_path, dirty_data)
|
||||
started = time.monotonic()
|
||||
response = server.post_json("POST", path, data)
|
||||
elapsed = time.monotonic() - started
|
||||
print(f" {'old' if on_live else 'new'} {path} {data}: {elapsed:.2f}s")
|
||||
states.append(snapshot(server, response))
|
||||
return states
|
||||
finally:
|
||||
server.stop()
|
||||
|
||||
|
||||
def column_key(key: str) -> str:
|
||||
"""``table t[3].created_at`` -> ``table t.created_at`` (mask whole columns)."""
|
||||
if key.startswith("table ") and "[" in key:
|
||||
head, _, tail = key.partition("[")
|
||||
return head + tail.partition("]")[2]
|
||||
return key
|
||||
|
||||
|
||||
def main() -> int:
|
||||
old_a, old_b, new = run(True), run(True), run(False)
|
||||
failures = 0
|
||||
masked_all = set()
|
||||
for step, (path, data) in enumerate(RESETS * 2):
|
||||
a, b, n = old_a[step], old_b[step], new[step]
|
||||
masked = {
|
||||
column_key(key)
|
||||
for key in set(a) | set(b)
|
||||
if a.get(key, CLOCK) != b.get(key, CLOCK)
|
||||
}
|
||||
masked_all |= masked
|
||||
keys = set(a) | set(n)
|
||||
diffs = [
|
||||
key
|
||||
for key in sorted(keys)
|
||||
if column_key(key) not in masked
|
||||
and a.get(key, "<missing>") != n.get(key, "<missing>")
|
||||
]
|
||||
tables = len(
|
||||
{k.split("[")[0].split("#")[0] for k in a if k.startswith("table ")}
|
||||
)
|
||||
status = "EQUAL" if not diffs else f"DIFFERENT ({len(diffs)} values)"
|
||||
print(
|
||||
f"{status}: {path} {json.dumps(data)}: {tables} tables, "
|
||||
f"{a['files#count']} files, {len(keys)} values, "
|
||||
f"{len(masked)} masked columns"
|
||||
)
|
||||
for key in diffs[:20]:
|
||||
old, now = a.get(key, "<missing>"), n.get(key, "<missing>")
|
||||
print(f" {key}: old {old!r} new {now!r}")
|
||||
failures += bool(diffs)
|
||||
print("masked (differ between the two old runs):")
|
||||
for key in sorted(masked_all):
|
||||
print(f" {key}")
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+38
-69
@@ -19,10 +19,8 @@ from aiohttp.web_runner import GracefulExit
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
from tests.browser.folder_trash import close_trash, replace_folder
|
||||
from tests.browser.request_hold import register_request_holds
|
||||
from tests.browser import slow_reset_watch
|
||||
from tests.browser.fast_reset import SEED_ON_LIVE_ENV, ScratchSeeder
|
||||
from tests.browser.workspace_activity_fixture import register_activity_commands
|
||||
from tests.browser.workspace_system_fixture import register_system_commands
|
||||
from tests.browser.workspace_integrations_fixture import (
|
||||
@@ -47,10 +45,6 @@ from tests.browser.workspace_library_editor_fixture import (
|
||||
from tests.browser.workspace_vocabulary_fixture import register_vocabulary_commands
|
||||
from tests.browser.workspace_tagging_fixture import register_tagging_commands
|
||||
from tests.browser.workspace_review_fixture import register_review_commands
|
||||
from tests.browser.workspace_diagnostics_fixture import (
|
||||
register_diagnostics_commands,
|
||||
repoint_logs,
|
||||
)
|
||||
|
||||
CANONICAL_ROOT = Path(__file__).resolve().parents[2]
|
||||
CANONICAL_ASSETS = {
|
||||
@@ -205,8 +199,6 @@ def create_app(root: Path) -> web.Application:
|
||||
directory.mkdir()
|
||||
setattr(folder_paths, f"get_{name}_directory", lambda path=directory: str(path))
|
||||
sys.modules["folder_paths"] = folder_paths
|
||||
# S12: the logger writes into this server's root before any record is written.
|
||||
repoint_logs(root)
|
||||
|
||||
from py.config import GalleryConfig, PromptManagerConfig
|
||||
|
||||
@@ -231,13 +223,8 @@ def create_app(root: Path) -> web.Application:
|
||||
register_activity_commands(app, routes, api)
|
||||
register_system_commands(app, routes, api, root)
|
||||
register_integrations_commands(app, routes, api, root)
|
||||
reset_logs = register_diagnostics_commands(app, routes, api, root)
|
||||
request_holds = register_request_holds(app, routes)
|
||||
slow_reset_watch.install(app, api)
|
||||
# Resets seed a scratch copy and publish it in one commit (#384).
|
||||
seeder = ScratchSeeder(
|
||||
api, root / "reset-scratch", on_live=os.environ.get(SEED_ON_LIVE_ENV) == "1"
|
||||
)
|
||||
|
||||
async def canonical_asset(request: web.Request) -> web.Response:
|
||||
filename, content_type = CANONICAL_ASSETS[request.path]
|
||||
@@ -251,7 +238,6 @@ def create_app(root: Path) -> web.Application:
|
||||
routes.get(path)(canonical_asset)
|
||||
|
||||
def seed(
|
||||
target,
|
||||
extra_images: int = 0,
|
||||
include_video: bool = False,
|
||||
prompt_inspector: bool = False,
|
||||
@@ -259,7 +245,7 @@ def create_app(root: Path) -> web.Application:
|
||||
extra_prompts: int = 0,
|
||||
extra_variants: int = 0,
|
||||
) -> None:
|
||||
with target.db.model.get_connection() as conn:
|
||||
with api.db.model.get_connection() as conn:
|
||||
for table in (
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
@@ -278,7 +264,8 @@ def create_app(root: Path) -> web.Application:
|
||||
):
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for name in ("output", "input"):
|
||||
replace_folder(root / name)
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
for index, (directory, text, tag) in enumerate(
|
||||
[
|
||||
("output", "Cat portrait", "cats"),
|
||||
@@ -321,30 +308,28 @@ def create_app(root: Path) -> web.Application:
|
||||
image.save(image_path, pnginfo=info)
|
||||
stamp = time.time() - index * 10
|
||||
os.utime(image_path, (stamp, stamp))
|
||||
pid = target.db.save_prompt(
|
||||
pid = api.db.save_prompt(
|
||||
text, tags=[tag], prompt_hash=generate_prompt_hash(text)
|
||||
)
|
||||
target.db.link_image_to_prompt(pid, str(image_path))
|
||||
api.db.link_image_to_prompt(pid, str(image_path))
|
||||
if index == 0:
|
||||
video_prompt_id = pid
|
||||
for number in range(extra_images):
|
||||
extra = root / directory / "bulk" / f"{number:04}.png"
|
||||
extra.parent.mkdir(exist_ok=True)
|
||||
shutil.copyfile(image_path, extra)
|
||||
target.db.link_image_to_prompt(pid, str(extra))
|
||||
api.db.link_image_to_prompt(pid, str(extra))
|
||||
if native_library:
|
||||
positive = target.db.get_prompt_by_hash(
|
||||
generate_prompt_hash("Cat portrait")
|
||||
)
|
||||
positive = api.db.get_prompt_by_hash(generate_prompt_hash("Cat portrait"))
|
||||
positive_id = positive["id"]
|
||||
target.db.update_prompt_metadata(
|
||||
api.db.update_prompt_metadata(
|
||||
positive_id, category="Animals", tags=["cats", "studio"]
|
||||
)
|
||||
variants = []
|
||||
for index, negative in enumerate(
|
||||
["blurry, watermark", "low quality", "low contrast", ""]
|
||||
):
|
||||
pair_id = target.db.save_prompt_pair(
|
||||
pair_id = api.db.save_prompt_pair(
|
||||
"Cat portrait",
|
||||
negative,
|
||||
title=f"Portrait variant {index + 1}",
|
||||
@@ -352,30 +337,28 @@ def create_app(root: Path) -> web.Application:
|
||||
tags=["studio"] if index == 0 else ["outdoors"],
|
||||
)
|
||||
variants.append(pair_id)
|
||||
with target.db.model.get_connection() as conn:
|
||||
with api.db.model.get_connection() as conn:
|
||||
images = conn.execute(
|
||||
"SELECT id FROM generated_images WHERE prompt_id=? ORDER BY id",
|
||||
(positive_id,),
|
||||
).fetchall()
|
||||
for index, image in enumerate(images[:2]):
|
||||
target.db.link_image_to_pair(
|
||||
api.db.link_image_to_pair(
|
||||
variants[index], image["id"], {"source": "fixture", "anchor": index}
|
||||
)
|
||||
target.db.link_image_to_pair(
|
||||
api.db.link_image_to_pair(
|
||||
variants[0], images[0]["id"], {"source": "fixture second evidence"}
|
||||
)
|
||||
target.db.record_prompt_pair_use(variants[0], times=3)
|
||||
api.db.record_prompt_pair_use(variants[0], times=3)
|
||||
for number in range(extra_variants):
|
||||
target.db.save_prompt_pair(
|
||||
"Cat portrait", f"Negative extra {number:04}"
|
||||
)
|
||||
api.db.save_prompt_pair("Cat portrait", f"Negative extra {number:04}")
|
||||
legacy = root / "output" / "legacy-unassigned.png"
|
||||
shutil.copyfile(root / "output" / "shared" / "same.png", legacy)
|
||||
target.db.link_image_to_prompt(positive_id, str(legacy))
|
||||
target.db.save_prompt("Dog landscape", category="Animals", tags=["dogs"])
|
||||
target.db.save_prompt_pair("Dual role portrait", "Cat portrait")
|
||||
api.db.link_image_to_prompt(positive_id, str(legacy))
|
||||
api.db.save_prompt("Dog landscape", category="Animals", tags=["dogs"])
|
||||
api.db.save_prompt_pair("Dual role portrait", "Cat portrait")
|
||||
for number in range(extra_prompts):
|
||||
target.db.save_prompt(
|
||||
api.db.save_prompt(
|
||||
f"Library extra {number:04}", category="Bulk", tags=["bulk"]
|
||||
)
|
||||
if prompt_inspector:
|
||||
@@ -396,8 +379,8 @@ def create_app(root: Path) -> web.Application:
|
||||
from py.api.thumbnail_settings import ThumbnailSettings
|
||||
|
||||
# 150 px previews in the default format (#317 thumbnail settings).
|
||||
target._generate_gallery_thumbnails_sync(
|
||||
target._get_all_output_dirs(), ThumbnailSettings("webp", 80, 150)
|
||||
api._generate_gallery_thumbnails_sync(
|
||||
api._get_all_output_dirs(), ThumbnailSettings("webp", 80, 150)
|
||||
)
|
||||
if include_video:
|
||||
# Seed after thumbnail generation to exercise a real unthumbnailed video.
|
||||
@@ -405,8 +388,8 @@ def create_app(root: Path) -> web.Application:
|
||||
shutil.copyfile(
|
||||
Path(__file__).resolve().parents[1] / "fixtures" / "tiny.webm", video
|
||||
)
|
||||
target.db.link_image_to_prompt(video_prompt_id, str(video))
|
||||
target.invalidate_gallery_cache()
|
||||
api.db.link_image_to_prompt(video_prompt_id, str(video))
|
||||
api.invalidate_gallery_cache()
|
||||
|
||||
history_fixture = None
|
||||
|
||||
@@ -432,7 +415,6 @@ def create_app(root: Path) -> web.Application:
|
||||
# No preview queued by the last test may land in the new fixture.
|
||||
await api._run_in_executor(api.close_auto_thumbnails)
|
||||
library_editor_commands.reset()
|
||||
await api._run_in_executor(reset_logs)
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
if not isinstance(data, dict):
|
||||
raise web.HTTPBadRequest()
|
||||
@@ -452,10 +434,7 @@ def create_app(root: Path) -> web.Application:
|
||||
raise web.HTTPBadRequest()
|
||||
if chronological_history:
|
||||
manifest = await api._run_in_executor(
|
||||
seeder.run,
|
||||
lambda target: seed_workspace_history(
|
||||
target, root, history_runs=history_runs
|
||||
),
|
||||
lambda: seed_workspace_history(api, root, history_runs=history_runs)
|
||||
)
|
||||
history_fixture = manifest
|
||||
return web.json_response({"success": True, **manifest})
|
||||
@@ -513,31 +492,25 @@ def create_app(root: Path) -> web.Application:
|
||||
raise web.HTTPBadRequest()
|
||||
if workspace_visual:
|
||||
history_fixture = None
|
||||
await api._run_in_executor(
|
||||
seeder.run, lambda target: seed_workspace_visual(target, root)
|
||||
)
|
||||
await api._run_in_executor(seed_workspace_visual, api, root)
|
||||
return web.json_response({"success": True})
|
||||
if full_inspector:
|
||||
history_fixture = None
|
||||
await api._run_in_executor(
|
||||
seeder.run,
|
||||
lambda target: seed_workspace_inspector(
|
||||
target, root, extra_images=extra, include_video=include_video
|
||||
),
|
||||
lambda: seed_workspace_inspector(
|
||||
api, root, extra_images=extra, include_video=include_video
|
||||
)
|
||||
)
|
||||
return web.json_response({"success": True})
|
||||
history_fixture = None
|
||||
await api._run_in_executor(
|
||||
seeder.run,
|
||||
lambda target: seed(
|
||||
target,
|
||||
extra,
|
||||
include_video,
|
||||
prompt_inspector,
|
||||
native_library,
|
||||
extra_prompts,
|
||||
extra_variants,
|
||||
),
|
||||
seed,
|
||||
extra,
|
||||
include_video,
|
||||
prompt_inspector,
|
||||
native_library,
|
||||
extra_prompts,
|
||||
extra_variants,
|
||||
)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@@ -577,15 +550,14 @@ def create_app(root: Path) -> web.Application:
|
||||
reveal_stub.reset()
|
||||
history_fixture = None
|
||||
manifest = await api._run_in_executor(
|
||||
seeder.run,
|
||||
lambda target: seed_workspace_concept(
|
||||
target,
|
||||
lambda: seed_workspace_concept(
|
||||
api,
|
||||
root,
|
||||
extra_images=data.get("extra_images", 0),
|
||||
seedless_images=data.get("seedless_images", 0),
|
||||
short_seedless=data.get("short_seedless", False),
|
||||
now=data.get("now"),
|
||||
),
|
||||
)
|
||||
)
|
||||
return web.json_response({"success": True, **manifest})
|
||||
|
||||
@@ -653,12 +625,9 @@ def create_app(root: Path) -> web.Application:
|
||||
async def cleanup(application: web.Application) -> None:
|
||||
# A scan of an ambiguous fixture file captures it for Needs review.
|
||||
api.close_annotation_captures()
|
||||
seeder.close()
|
||||
api.db.close_all()
|
||||
# Old media folders still waiting for deletion go before the temp root.
|
||||
close_trash(root)
|
||||
|
||||
seed(api)
|
||||
seed()
|
||||
# Open Needs review change streams end first, as on the real host.
|
||||
app.on_shutdown.append(api.close_review_events)
|
||||
app.on_cleanup.append(cleanup)
|
||||
|
||||
@@ -1,396 +0,0 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate, measureReview, mapPlanConcept }
|
||||
from './concept-inventory.mjs';
|
||||
|
||||
// #249 System S12 Task 7 (#329): System › Diagnostics & logs (app.js renderSystem 694–695) and its log cleanup review
|
||||
// (the concept's `maintenancePlan("rotated logs")`, 953–958) against the immutable concept, on the concept-shaped
|
||||
// fixture with the logger writing into this server's own root and seeded with the concept's sample (four lines, a
|
||||
// 1.4 MB active log, three rotated copies, console logging off). Each concept measurement is mapped only by the recorded
|
||||
// deviations (D109, D110, and D100 for the review), each commented where it applies; after that the difference must be
|
||||
// empty, and a removed term, a renamed button, a reordered option, a primary Refresh, a squeezed select or a swapped
|
||||
// token must be noticed.
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(await request.post('/__test__/logs/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/logs/seed', { data: { mode: 'concept' } })).toBeOK();
|
||||
page.conceptErrors = [];
|
||||
page.on('pageerror', error => page.conceptErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/logs/reset');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.conceptErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const REPORT_URL = '/prompt_manager/system/diagnostics';
|
||||
const PREVIEW_URL = '/prompt_manager/maintenance/preview';
|
||||
const LINE_FORMAT = /^\d\d:\d\d:\d\d [A-Z]+ +\S/;
|
||||
|
||||
/** Runs in the page: Diagnostics & logs under `mainSelector`: every surface's text, controls, geometry and tokens. */
|
||||
function measureDiagnostics([mainSelector, lineFormat]) {
|
||||
const format = new RegExp(lineFormat);
|
||||
const layout = document.querySelector(`${mainSelector} .settings-layout`);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const tokens = (node, ...names) => Object.fromEntries(names.map(key => [key, getComputedStyle(node).getPropertyValue(key).trim()]));
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const button = node => ({ text: text(node), className: node.className });
|
||||
const control = row => {
|
||||
const select = row.querySelector(':scope > select');
|
||||
if (select) return { kind: 'select', id: select.id, options: [...select.options].map(option => option.textContent),
|
||||
selected: select.selectedOptions[0]?.textContent ?? null, disabled: select.disabled };
|
||||
const toggle = row.querySelector(':scope > * input[type="checkbox"]');
|
||||
return toggle ? { kind: 'switch', id: toggle.id, wrapper: name(toggle.parentElement), checked: toggle.checked,
|
||||
disabled: toggle.disabled } : null;
|
||||
};
|
||||
const operations = parent => [...parent.querySelectorAll(':scope > .operation')].map(row => ({ tree: [...row.children].map(name),
|
||||
title: text(row.querySelector('h3')), description: text(row.querySelector('p')),
|
||||
actions: [...row.querySelectorAll('.row-actions > button')].map(button) }));
|
||||
const toolbarItem = node => ({ name: name(node), text: node.localName === 'select' || node.localName === 'input' ? null : text(node),
|
||||
for: node.localName === 'label' ? node.htmlFor : null, id: node.id || null,
|
||||
options: node.localName === 'select' ? [...node.options].map(option => option.textContent) : null,
|
||||
// The chosen option, so a wrong default level is noticed.
|
||||
selected: node.localName === 'select' ? node.selectedOptions[0]?.textContent ?? null : null,
|
||||
input: node.localName === 'input' ? [node.type, node.min, node.max, node.value] : null,
|
||||
className: node.localName === 'button' ? node.className : null });
|
||||
const surfaces = [...layout.querySelectorAll(':scope > section.surface')];
|
||||
const pre = layout.querySelector('pre');
|
||||
const lines = (pre?.textContent ?? '').split('\n').filter(Boolean);
|
||||
const details = layout.querySelector('details');
|
||||
const firstRow = layout.querySelector('.setting-row');
|
||||
const toolbar = layout.querySelector('.toolbar');
|
||||
return {
|
||||
layout: [...layout.children].map(name),
|
||||
surfaces: surfaces.map(surface => {
|
||||
const heading = surface.querySelector(':scope > .panel-heading');
|
||||
const grid = surface.querySelector(':scope > .meta-grid');
|
||||
return {
|
||||
tree: [...surface.children].map(name),
|
||||
heading: text(surface.querySelector(':scope > h2, :scope > .panel-heading > h2')),
|
||||
panelHeading: heading && [...heading.children].map(node => [name(node), text(node)]),
|
||||
grid: grid && [...grid.children].map(entry => [name(entry), text(entry.querySelector('dt')), text(entry.querySelector('dd'))]),
|
||||
buttons: [...surface.querySelectorAll(':scope > button')].map(button),
|
||||
toolbar: surface.querySelector(':scope > .toolbar') && [...surface.querySelector(':scope > .toolbar').children].map(toolbarItem),
|
||||
pre: surface.querySelector(':scope > pre') && { tabIndex: pre.tabIndex, tree: [...pre.children].map(name), lines,
|
||||
format: lines.every(line => format.test(line)) },
|
||||
notes: [...surface.querySelectorAll(':scope > p')].map(node => [name(node), text(node), node.getAttribute('role'),
|
||||
node.getAttribute('aria-label')]),
|
||||
rows: [...surface.querySelectorAll(':scope > .setting-row')].map(row => ({ tree: [...row.children].map(name),
|
||||
text: [...row.firstElementChild.children].map(node => [name(node), text(node), node.localName === 'label' ? node.htmlFor : null]),
|
||||
control: control(row) })),
|
||||
details: surface.querySelector(':scope > details') && {
|
||||
summary: text(details.querySelector(':scope > summary')), tree: [...details.children].map(name),
|
||||
notes: [...details.querySelectorAll(':scope > p')].map(node => [name(node), text(node)]),
|
||||
buttons: [...details.querySelectorAll(':scope > button')].map(button),
|
||||
files: [...details.querySelectorAll(':scope > .operation-list > .operation')].map(row => ({
|
||||
title: text(row.querySelector('h3')), detail: text(row.querySelector('p')), actions: [...row.querySelectorAll('.row-actions > button')].map(item =>
|
||||
[text(item), item.getAttribute('aria-label')]) })),
|
||||
operations: operations(details) },
|
||||
operations: operations(surface),
|
||||
};
|
||||
}),
|
||||
tokens: tokens(surfaces[0], '--line', '--panel', '--accent'),
|
||||
surface: css(surfaces[1], 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius',
|
||||
'marginTop'),
|
||||
grid: css(layout.querySelector('.meta-grid'), 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
toolbar: css(toolbar, 'display', 'flexWrap', 'alignItems', 'columnGap', 'rowGap', 'marginBottom'),
|
||||
pre: css(pre, 'fontFamily', 'fontSize', 'backgroundColor', 'borderTopColor', 'borderTopLeftRadius', 'paddingTop', 'overflowX'),
|
||||
row: css(firstRow, 'display', 'alignItems', 'justifyContent', 'columnGap', 'paddingTop', 'paddingBottom', 'borderTopWidth',
|
||||
'borderTopColor'),
|
||||
label: css(firstRow.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
description: css(firstRow.querySelector('p'), 'fontSize', 'color', 'marginTop'),
|
||||
select: css(firstRow.querySelector('select'), 'fontSize', 'minWidth', 'maxWidth', 'backgroundColor', 'color', 'borderTopColor'),
|
||||
summary: css(details.querySelector('summary'), 'fontSize', 'fontWeight', 'color', 'cursor'),
|
||||
small: css(details.querySelector(':scope > p.small'), 'fontSize', 'color'),
|
||||
// Whether each select is as wide as the browser lays out a copy holding only its chosen option, i.e. it shows
|
||||
// that option whole (the browser's own select sizing, arrow included).
|
||||
selectsFit: [...layout.querySelectorAll('select')].map(select => {
|
||||
const copy = select.cloneNode(false);
|
||||
copy.removeAttribute('id');
|
||||
copy.append(new Option(select.selectedOptions[0]?.textContent ?? ''));
|
||||
Object.assign(copy.style, { position: 'absolute', visibility: 'hidden', width: 'max-content', minWidth: '0',
|
||||
maxWidth: 'none', flex: 'none' });
|
||||
select.parentElement.append(copy);
|
||||
const needed = copy.getBoundingClientRect().width;
|
||||
copy.remove();
|
||||
return select.getBoundingClientRect().width + 0.5 >= needed;
|
||||
}),
|
||||
pageScroll: document.documentElement.scrollWidth > document.documentElement.clientWidth,
|
||||
};
|
||||
}
|
||||
|
||||
const plural = (count, one, many = `${one}s`) => `${count.toLocaleString('en-US')} ${count === 1 ? one : many}`;
|
||||
/** The server's size label (1 KB = 1024 bytes), as the section prints it. */
|
||||
function sizeText(bytes) {
|
||||
if (bytes < 1024) return plural(bytes, 'byte');
|
||||
if (bytes < 2 ** 20) return `${(bytes / 2 ** 10).toFixed(1)} KB`;
|
||||
if (bytes < 2 ** 30) return `${(bytes / 2 ** 20).toFixed(1)} MB`;
|
||||
return `${(bytes / 2 ** 30).toFixed(1)} GB`;
|
||||
}
|
||||
|
||||
const KINDS = Object.freeze({ active: 'Active log', rotated: 'Rotated copy', other: 'Other file' });
|
||||
const RENDERER_NONE = 'No ComfyUI window is open in this browser, so its renderer is not known.';
|
||||
const COPY = Object.freeze({
|
||||
testLink: 'Write a placeholder image record to check that linking works.',
|
||||
runtimeLevel: 'How much the server writes to the log. Applied until the server restarts.',
|
||||
console: 'Print Prompt Manager’s warnings and errors in the ComfyUI console (everything at DEBUG). Applied until the server restarts.',
|
||||
clear: 'Empty the active log and delete its rotated copies.',
|
||||
retention: 'Rotates at 10 MB · keeps 5 rotated copies · file logging on · memory buffer 1,000 entries',
|
||||
});
|
||||
|
||||
/**
|
||||
* The concept's Diagnostics & logs with the recorded deviations applied, for the server's `report` and the seeded
|
||||
* `files` (`/logs/files`). The concept's own sample values the fixture reproduces stay as they are: the four log lines,
|
||||
* the Viewer.js line, "prompt_manager.log · 1.4 MB · rotated copies: 3", INFO and console logging off.
|
||||
*/
|
||||
function mapDiagnosticsConcept(reference, { report, files, stats }) {
|
||||
const [diagnostics, logs] = reference.surfaces;
|
||||
const [levelLabel, level, refresh, download] = logs.toolbar;
|
||||
const [runtime, consoleRow] = logs.rows;
|
||||
const database = report.checks.database;
|
||||
const { configured, watching, unavailable } = report.sources;
|
||||
const sources = configured === 0 ? `ComfyUI output folder · ${watching ? 'watching' : 'manual scan'}`
|
||||
: unavailable > 0 ? `${plural(configured, 'source')} · ${unavailable} not available`
|
||||
: `${plural(configured, 'source')} · ${watching} watching`;
|
||||
const errors = (stats.level_counts.ERROR ?? 0) + (stats.level_counts.CRITICAL ?? 0);
|
||||
return {
|
||||
...reference,
|
||||
surfaces: [
|
||||
{ ...diagnostics,
|
||||
// D110: the Test image link operation follows the concept's Image-link diagnostics.
|
||||
tree: [...diagnostics.tree, 'div.operation'],
|
||||
// D110: live values. Database and Sources from `GET /system/diagnostics`, Host renderers is D106's
|
||||
// renderer line (no ComfyUI window in this browser); the pinned Viewer.js line is the concept's own.
|
||||
grid: diagnostics.grid.map(([entry, term, value]) => [entry, term, {
|
||||
Database: `Readable · ${sizeText(report.database_info.database_size_bytes)} · ${plural(database.image_count, 'image')}`,
|
||||
Sources: sources,
|
||||
'Host renderers': RENDERER_NONE }[term] ?? value]),
|
||||
// D110: the buttons drop the concept's "sample".
|
||||
buttons: diagnostics.buttons.map(item => ({ ...item, text: item.text.replace('Download sample diagnostic report',
|
||||
'Download diagnostic report') })),
|
||||
operations: [
|
||||
// D110: "Inspect" opens the read-only Inspect a file page.
|
||||
...diagnostics.operations.map(row => ({ ...row, actions: row.actions.map(item => ({ ...item, text: 'Inspect' })) })),
|
||||
// D110: the legacy synthetic link (S12-11), enabled: the fixture has a most recent prompt.
|
||||
{ ...diagnostics.operations[0], title: 'Test image link', description: COPY.testLink,
|
||||
actions: [{ ...diagnostics.operations[0].actions[0], text: 'Create test link…' }] },
|
||||
] },
|
||||
{ ...logs,
|
||||
// D109: the statistics line follows the viewer.
|
||||
tree: [logs.tree[0], logs.tree[1], logs.tree[2], 'p.small', ...logs.tree.slice(3)],
|
||||
toolbar: [
|
||||
levelLabel,
|
||||
// D109: all six minimum levels (the concept has three).
|
||||
{ ...level, options: ['All levels', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] },
|
||||
// D109: the added Lines control, 1–500, default 100.
|
||||
{ name: 'label.small', text: 'Lines', for: 'log-limit', id: null, options: null, selected: null, input: null,
|
||||
className: null },
|
||||
{ name: 'input', text: null, for: null, id: 'log-limit', options: null, selected: null,
|
||||
input: ['number', '1', '500', '100'], className: null },
|
||||
refresh,
|
||||
// D109: the active log, not a sample.
|
||||
{ ...download, text: 'Download log' },
|
||||
],
|
||||
// D109: "Log statistics" (errors are ERROR + CRITICAL).
|
||||
notes: [['p.small', `${plural(stats.buffer_count, 'entry', 'entries')} in memory · ${plural(errors, 'error')} · `
|
||||
+ `${plural(stats.level_counts.WARNING ?? 0, 'warning')} · ${plural(stats.log_files_count, 'file')} · `
|
||||
+ `${sizeText(stats.total_log_size)} on disk`, 'status', 'Log statistics']],
|
||||
rows: [
|
||||
// D109: CRITICAL after the concept's levels; the value lasts until the server restarts.
|
||||
{ ...runtime, text: [runtime.text[0], ['p', COPY.runtimeLevel, null]],
|
||||
control: { ...runtime.control, options: [...runtime.control.options, 'CRITICAL'] } },
|
||||
// D109: what the console setting does and that it lasts until the server restarts (off, as the
|
||||
// concept's sample, by the fixture's seed).
|
||||
{ ...consoleRow, text: [consoleRow.text[0], ['p', COPY.console, null]] },
|
||||
],
|
||||
details: { ...logs.details,
|
||||
// D109: the retention line, the files and Clear all logs follow the concept's line and review.
|
||||
tree: [...logs.details.tree, 'p.small', 'div.operation-list', 'div.operation'],
|
||||
notes: [...logs.details.notes, ['p.small', COPY.retention]],
|
||||
files: files.map(row => ({ title: row.filename, detail: `${sizeText(row.size)} · ${row.modified.slice(0, 16)
|
||||
.replace('T', ' ')} · ${KINDS[row.kind]}`, actions: [['Download', `Download ${row.filename}`]] })),
|
||||
operations: [{ tree: ['div', 'div.row-actions'], title: 'Clear all logs', description: COPY.clear,
|
||||
actions: [{ text: 'Clear all logs…', className: 'danger' }] }] } },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptDiagnostics(page, theme) {
|
||||
await openConcept(page, 'system', theme);
|
||||
await page.locator('#main .section-nav').getByRole('button', { name: 'Diagnostics & logs', exact: true }).click();
|
||||
await expect(page.locator('#main .settings-layout > section.surface')).toHaveCount(2);
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureDiagnostics, ['#main', LINE_FORMAT.source]);
|
||||
}
|
||||
|
||||
/** Production's section once every read answered, with the server's report, files and statistics it shows. */
|
||||
async function productionDiagnostics(page, request, theme) {
|
||||
await openProduction(page, 'system/diagnostics', theme);
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(main.locator('.settings-layout > section.surface')).toHaveCount(2);
|
||||
await expect(main.locator('.meta-grid dd')).not.toContainText(['Loading…']);
|
||||
await expect(main.locator('.meta-grid dd').last()).toHaveText(RENDERER_NONE);
|
||||
await expect(main.locator('#log-lines')).not.toHaveText('Loading…');
|
||||
await expect(main.locator('#log-stats')).not.toHaveText('Loading…');
|
||||
await expect(main.locator('details > p.small')).not.toContainText(['Loading…']);
|
||||
await expect(main.locator('#runtime-level')).toBeEnabled();
|
||||
await expect(main.locator('.operation').filter({ hasText: 'Create test link…' }).locator('button')).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await page.mouse.move(0, 0);
|
||||
const production = await page.evaluate(measureDiagnostics, ['#workspace-main', LINE_FORMAT.source]);
|
||||
const read = async url => (await request.get(url)).json();
|
||||
const report = await read(REPORT_URL);
|
||||
const { files } = await read('/prompt_manager/logs/files');
|
||||
const { stats } = await read('/prompt_manager/logs/stats');
|
||||
return { production, report, files, stats };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`diagnostics and logs match the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const concept = await conceptDiagnostics(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-diagnostics-${theme}-${width}.png`), fullPage: true });
|
||||
const { production, ...live } = await productionDiagnostics(page, request, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-diagnostics-${theme}-${width}.png`), fullPage: true });
|
||||
expect(diffInventory(mapDiagnosticsConcept(concept, live), production)).toEqual([]);
|
||||
expect(production.surfaces[1].pre.format).toBe(true);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
}
|
||||
|
||||
// Below the concept's smallest size: on a 320px phone the Level and Runtime log level selects still show their whole
|
||||
// option, without page scroll.
|
||||
for (const theme of THEMES) test(`diagnostics selects fit at 320 wide ${theme}`, async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 320, height: 700 });
|
||||
const { production } = await productionDiagnostics(page, request, theme);
|
||||
expect(production.selectsFit).toEqual([true, true]);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
|
||||
/**
|
||||
* D100 with D109's rule for this plan: the server's `rotated_logs` plan in the concept's dialog, "Rotated log copies"
|
||||
* as its `h3`, the plan's grid values (its Backup row says none is needed), "Listed files (N)", the hold line, no D55
|
||||
* backup box (nothing in the library changes) and "Apply reviewed operation".
|
||||
*/
|
||||
function mapCleanupConcept(reference, plan) {
|
||||
const mapped = mapPlanConcept(reference, 'Rotated log copies', plan);
|
||||
return {
|
||||
...mapped,
|
||||
// D109: no backup box for a plan that changes no library record.
|
||||
tree: mapped.tree.filter(item => item !== 'div.notice'),
|
||||
grid: { ...mapped.grid, rows: mapped.grid.rows.map(([term, value]) =>
|
||||
[term, term === 'Backup' ? 'Not needed: no library record changes' : value]) },
|
||||
backup: null,
|
||||
// D109: the plan lists files, not records.
|
||||
details: { ...mapped.details, summary: `Listed files (${plan.details.length})` },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptCleanup(page, theme) {
|
||||
await conceptDiagnostics(page, theme);
|
||||
await page.locator('#main details > summary').click();
|
||||
await page.locator('#main [data-plan="rotated logs"]').click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review maintenance plan');
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureReview);
|
||||
}
|
||||
|
||||
async function productionCleanup(page, request, theme) {
|
||||
await productionDiagnostics(page, request, theme);
|
||||
await page.locator('#workspace-main details > summary').click();
|
||||
const answer = page.waitForResponse(response => response.url().endsWith(PREVIEW_URL));
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Review log cleanup', exact: true }).click();
|
||||
const plan = await (await answer).json();
|
||||
await expect(page.locator('dialog#modal[open] #modal-body > h3')).toHaveText('Rotated log copies');
|
||||
await page.mouse.move(0, 0);
|
||||
return { plan, production: await page.evaluate(measureReview) };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`log cleanup review matches the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const concept = await conceptCleanup(page, theme);
|
||||
expect(concept.heading.text).toBe('rotated logs');
|
||||
expect(concept.tree).toEqual(['p.demo-note', 'h3', 'dl.meta-grid', 'p.small', 'div.dialog-actions']);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-log-cleanup-${theme}-${width}.png`) });
|
||||
const { plan, production } = await productionCleanup(page, request, theme);
|
||||
expect(plan.operation).toBe('rotated_logs');
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-log-cleanup-${theme}-${width}.png`) });
|
||||
expect(diffInventory(mapCleanupConcept(concept, plan), production)).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
// Every path where production differs from the unmapped concept: D110's live values (Viewer.js is the concept's own),
|
||||
// renamed buttons and added operation, and D109's toolbar, statistics line, runtime rows and log files.
|
||||
const DEVIATED_PATHS = [
|
||||
'surfaces.0.buttons.0', 'surfaces.0.grid.0', 'surfaces.0.grid.1', 'surfaces.0.grid.3', 'surfaces.0.operations.0',
|
||||
'surfaces.0.operations.1', 'surfaces.0.operations.length', 'surfaces.0.tree.4', 'surfaces.0.tree.length',
|
||||
'surfaces.1.details.files', 'surfaces.1.details.notes', 'surfaces.1.details.operations', 'surfaces.1.details.tree',
|
||||
'surfaces.1.notes.0', 'surfaces.1.notes.length', 'surfaces.1.rows.0', 'surfaces.1.rows.1', 'surfaces.1.toolbar.1',
|
||||
'surfaces.1.toolbar.2', 'surfaces.1.toolbar.3', 'surfaces.1.toolbar.4', 'surfaces.1.toolbar.5', 'surfaces.1.toolbar.length',
|
||||
'surfaces.1.tree.3', 'surfaces.1.tree.5', 'surfaces.1.tree.6', 'surfaces.1.tree.length'].sort();
|
||||
|
||||
test('diagnostics comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const concept = await conceptDiagnostics(page, 'dark');
|
||||
const { production, ...live } = await productionDiagnostics(page, request, 'dark');
|
||||
const expected = mapDiagnosticsConcept(concept, live);
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Unmapped, the section differs only where D109 and D110 are recorded.
|
||||
expect(diffInventory(concept, production).map(item => item.path.split('.').slice(0, 4).join('.'))
|
||||
.filter((path, index, all) => all.indexOf(path) === index).sort()).toEqual(DEVIATED_PATHS);
|
||||
const measure = () => page.evaluate(measureDiagnostics, ['#workspace-main', LINE_FORMAT.source]);
|
||||
const paths = async () => diffInventory(expected, await measure()).map(item => item.path);
|
||||
// A removed term.
|
||||
const undoTerm = await mutate(page, { remove: ['#workspace-main .meta-grid > div:nth-child(2)'] });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.0.grid.length']));
|
||||
await undoTerm();
|
||||
// A renamed button.
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main section.surface > button').textContent = 'Download report'; });
|
||||
expect(await paths()).toEqual(['surfaces.0.buttons.0.text']);
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main section.surface > button').textContent = 'Download diagnostic report'; });
|
||||
// A reordered level option.
|
||||
await page.evaluate(() => { const select = document.querySelector('#workspace-main #log-level'); select.append(select.options[1]); });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.1.toolbar.1.options.1']));
|
||||
await page.evaluate(() => { const select = document.querySelector('#workspace-main #log-level'); select.insertBefore(select.options[5], select.options[1]); });
|
||||
// A wrong default level.
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main #log-level').value = 'INFO'; });
|
||||
expect(await paths()).toEqual(['surfaces.1.toolbar.1.selected']);
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main #log-level').value = ''; });
|
||||
// A primary Refresh.
|
||||
await page.evaluate(() => { document.querySelectorAll('#workspace-main .toolbar > button')[0].classList.add('ca-primary'); });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.1.toolbar.4.className']));
|
||||
await page.evaluate(() => { document.querySelectorAll('#workspace-main .toolbar > button')[0].classList.remove('ca-primary'); });
|
||||
// A select squeezed until its chosen option is cut.
|
||||
const undoSqueeze = await mutate(page, { css: '#workspace-main #runtime-level { max-width: 30px !important; min-width: 0 !important; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['selectsFit.1']));
|
||||
await undoSqueeze();
|
||||
// A swapped token.
|
||||
const undoToken = await mutate(page, { css: '#workspace-main .settings-layout { --line: rgb(255, 0, 0); }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['tokens.--line', 'row.borderTopColor']));
|
||||
await undoToken();
|
||||
expect(diffInventory(expected, await measure())).toEqual([]);
|
||||
|
||||
// The log cleanup review: a changed grid term, a backup box that came back and a grid token are noticed.
|
||||
const conceptReview = await conceptCleanup(page, 'dark');
|
||||
const { plan } = await productionCleanup(page, request, 'dark');
|
||||
const expectedReview = mapCleanupConcept(conceptReview, plan);
|
||||
const reviewPaths = async () => diffInventory(expectedReview, await page.evaluate(measureReview)).map(item => item.path);
|
||||
expect(await reviewPaths()).toEqual([]);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[2].textContent = 'Safety'; });
|
||||
expect(await reviewPaths()).toEqual(['grid.rows.2.0']);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[2].textContent = 'Backup'; });
|
||||
const undoGrid = await mutate(page, { css: 'dialog#modal dl.meta-grid { column-gap: 3px; }' });
|
||||
expect(await reviewPaths()).toEqual(expect.arrayContaining(['grid.columnGap']));
|
||||
await undoGrid();
|
||||
expect(await reviewPaths()).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate, measureReview, mapPlanConcept }
|
||||
from './concept-inventory.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
|
||||
// #249 System S7 Task 20: System › Maintenance (app.js renderSystem 688–690), its plan dialog (`maintenancePlan`,
|
||||
// 953–958) for each operation and its duplicates dialog (`duplicates`, 1457–1461) against the immutable concept, on
|
||||
@@ -104,6 +103,47 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
});
|
||||
}
|
||||
|
||||
/** Runs in the page: the open review dialog (plan or duplicates): its title, body and the concept's tokens. */
|
||||
function measureReview() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const body = dialog.querySelector('#modal-body');
|
||||
const grid = body.querySelector(':scope > dl.meta-grid');
|
||||
const compare = body.querySelector('.compare-grid');
|
||||
const item = compare?.querySelector('.compare-item');
|
||||
const notes = [...body.querySelectorAll(':scope > p.small')];
|
||||
const note = notes.at(-1);
|
||||
const actions = body.querySelector(':scope > .dialog-actions');
|
||||
return {
|
||||
title: text(dialog.querySelector('#modal-title')),
|
||||
tree: [...body.children].map(name),
|
||||
heading: body.querySelector(':scope > h3') && { text: text(body.querySelector(':scope > h3')),
|
||||
...css(body.querySelector(':scope > h3'), 'fontSize', 'fontWeight', 'color', 'marginBottom') },
|
||||
grid: grid && { rows: [...grid.children].map(row => [text(row.querySelector('dt')), text(row.querySelector('dd'))]),
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
dt: css(grid.querySelector('dt'), 'fontSize', 'color', 'textTransform'), dd: css(grid.querySelector('dd'), 'fontSize', 'color') },
|
||||
details: body.querySelector(':scope > details') && { summary: text(body.querySelector(':scope > details > summary')),
|
||||
tree: [...body.querySelector(':scope > details').children].map(name) },
|
||||
backup: body.querySelector(':scope > .notice') && { tree: [...body.querySelector(':scope > .notice').children].map(name),
|
||||
text: [...body.querySelectorAll(':scope > .notice > div:first-child > *')].map(node => [node.localName, text(node)]),
|
||||
buttons: [...body.querySelectorAll(':scope > .notice button')].map(text) },
|
||||
compare: compare && { tree: [...compare.children].map(name), ...css(compare, 'display', 'gridTemplateColumns', 'columnGap'),
|
||||
items: [...compare.children].map(entry => ({ tree: [...entry.children].map(name), label: text(entry.querySelector('label')),
|
||||
checked: entry.querySelector('input[type=radio]').checked, image: entry.querySelector('img')?.className ?? null })),
|
||||
item: css(item, 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius'),
|
||||
image: css(item.querySelector('img'), 'width', 'objectFit'),
|
||||
label: css(item.querySelector('label'), 'fontSize', 'color'),
|
||||
radio: { name: item.querySelector('input[type=radio]').name.startsWith('keep-') ? 'keep-*' : 'other',
|
||||
type: item.querySelector('input').type } },
|
||||
note: note && { text: text(note), ...css(note, 'fontSize', 'color', 'marginTop') },
|
||||
actions: { tree: [...actions.children].map(name), ...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(item => ({ text: text(item), className: item.className,
|
||||
...css(item, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor') })) },
|
||||
};
|
||||
}
|
||||
|
||||
// The concept's plan names (`data-plan`) and production's operation, row and button for each.
|
||||
const PLANS = Object.freeze([
|
||||
['duplicate prompts', 'duplicate_prompts', 'Duplicate prompt text'],
|
||||
@@ -111,6 +151,32 @@ const PLANS = Object.freeze([
|
||||
['unused prompt components', 'unused_components', 'Unused prompt components'],
|
||||
['missing-file links', 'missing_links', 'Missing-file links'],
|
||||
]);
|
||||
const BACKUP_BOX = Object.freeze({ tree: ['div', 'div.row-actions'], text: [['h3', 'Back up the database first (recommended)'],
|
||||
['p', 'Downloads a copy of the prompts database before this operation changes any records.']], buttons: ['Back up now'] });
|
||||
|
||||
/**
|
||||
* D100: the server's plan in the concept's dialog: the operation's row title as its `h3` (the concept prints its
|
||||
* `data-plan` name), the four rows with the plan's values, the listed records (`details`), the hold line in the
|
||||
* concept's `p.small`, the D55 backup box before the actions, and "Apply reviewed operation". No sample note.
|
||||
*/
|
||||
function mapPlanConcept(reference, title, plan) {
|
||||
const backup = plan.backup.policy === 'off' ? 'No safety backup (the policy is off)'
|
||||
: `Create and verify before changes (${plan.backup.destination_label})`;
|
||||
const tree = ['h3', 'dl.meta-grid', ...(plan.details.length ? ['details'] : []), 'p.small', 'div.notice', 'div.dialog-actions'];
|
||||
const [cancel, apply] = reference.actions.buttons;
|
||||
return {
|
||||
...reference,
|
||||
tree,
|
||||
heading: { ...reference.heading, text: title },
|
||||
grid: { ...reference.grid, rows: [['Scope', plan.scope.label],
|
||||
['Protected', plan.protected.reasons.length ? plan.protected.reasons.join(' · ') : 'Nothing is protected'],
|
||||
['Backup', backup], ['Media on disk', plan.media.label]] },
|
||||
details: plan.details.length ? { summary: `Listed records (${plan.details.length})`, tree: ['summary', 'pre'] } : null,
|
||||
backup: BACKUP_BOX,
|
||||
note: { ...reference.note, text: 'Nothing changes until you apply. This review holds for 15 minutes.' },
|
||||
actions: { ...reference.actions, buttons: [cancel, { ...apply, text: 'Apply reviewed operation' }] },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptPlan(page, theme, name) {
|
||||
await conceptMaintenance(page, theme);
|
||||
|
||||
@@ -33,7 +33,7 @@ test.afterEach(async ({ page, request, browserName }) => {
|
||||
const PREVIEW_URL = '/prompt_manager/autotag/batch/preview';
|
||||
const MODEL_OPTIONS = ['WD14 SwinV2 · ~400 MB', 'WD14 ViT · ~400 MB', 'GGUF JoyCaption · ~4.5 GB', 'HuggingFace JoyCaption · ~16 GB'];
|
||||
const SUGGESTIONS = 'From 1 to 100: the most suggestions Generate shows for one image.';
|
||||
const BATCH = 'Tag each output image from its own pixels with the model above. Prompts are not changed. Review the scope first.';
|
||||
const BATCH = 'Add tags to each prompt from its linked output images with the model above. Review the scope first.';
|
||||
|
||||
/** Runs in the page: Auto-tagging's surface under `mainSelector`: rows, field, actions, operation, geometry and tokens. */
|
||||
function measureTagging(mainSelector) {
|
||||
@@ -272,19 +272,11 @@ function measureReview() {
|
||||
}
|
||||
|
||||
const number = value => value.toLocaleString('en-US');
|
||||
/** DEC11 (#379): the Time row the preview's estimate reads as (WD14 on the test server's processor). */
|
||||
function timeRow(plan) {
|
||||
const minutes = Math.max(1, Math.ceil(plan.estimate_seconds / 60));
|
||||
const text = minutes < 90 ? `about ${minutes} min`
|
||||
: `about ${Math.floor(minutes / 60)} h${minutes % 60 ? ` ${minutes % 60} min` : ''}`;
|
||||
return `${text} on a processor; a graphics card can be faster.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* D134 (#379): the server's batch preview in the concept's review: "Review tag output images" with "Tag output images"
|
||||
* as its `h3`, the model, policy, image counts, time estimate and settings in the concept's `dl.meta-grid` (Backup in
|
||||
* the concept's wording), the estimate sentence in the concept's `p.small`, the D55 backup box and Cancel / "Start
|
||||
* tagging".
|
||||
* D134: the server's batch preview in the concept's review: "Review tag output images" with "Tag output images" as its
|
||||
* `h3`, the model, policy, counts and settings in the concept's `dl.meta-grid` (Backup in the concept's wording), the
|
||||
* upper-bound sentence in the concept's `p.small`, the D55 backup box and Cancel / "Start tagging".
|
||||
*/
|
||||
function mapBatchConcept(reference, plan) {
|
||||
const [cancel, start] = reference.actions.buttons;
|
||||
@@ -294,14 +286,13 @@ function mapBatchConcept(reference, plan) {
|
||||
tree: ['h3', 'dl.meta-grid', 'p.small', 'div.notice', 'div.dialog-actions'],
|
||||
heading: { ...reference.heading, text: 'Tag output images' },
|
||||
grid: { ...reference.grid, rows: [['Model', 'WD14 SwinV2 · ~400 MB'], ['Existing tags', 'Merge with existing'],
|
||||
['Images', number(plan.images)], ['Already auto-tagged', number(plan.already_tagged)],
|
||||
['Images to tag', `about ${number(plan.to_tag)}`], ['Time', timeRow(plan)],
|
||||
['Settings', `General 0.35 · Character 0.85 · up to ${number(plan.suggestion_limit)} tags per image`],
|
||||
['Images linked', number(plan.images)], ['Prompts', number(plan.prompts)], ['Already tagged', number(plan.already_tagged)],
|
||||
['Prompts to tag', `up to ${number(plan.to_tag)}`], ['Settings', 'General 0.35 · Character 0.85'],
|
||||
['Backup', 'Create and verify before changes']] },
|
||||
backup: { tree: ['div', 'div.row-actions'], text: [['h3', 'Back up the database first (recommended)'],
|
||||
['p', 'Downloads a copy of the prompts database before this operation changes any records.']], buttons: ['Back up now'] },
|
||||
note: { ...reference.note, text: 'Nothing is written until you start. The number to tag is an estimate: images changed '
|
||||
+ 'since they were tagged are tagged again, and missing files are skipped. Activity reports what was tagged.' },
|
||||
note: { ...reference.note, text: 'Nothing is written until you start. Missing files and files outside the output folders '
|
||||
+ 'are counted, so the prompts to tag are an upper bound; Activity reports what was tagged.' },
|
||||
actions: { ...reference.actions, buttons: [cancel, { ...start, text: 'Start tagging' }] },
|
||||
};
|
||||
}
|
||||
@@ -317,8 +308,8 @@ async function conceptReview(page, theme) {
|
||||
|
||||
async function productionReview(page, theme) {
|
||||
await productionTagging(page, theme);
|
||||
// Merge reviews every listed still image (#379): a batch with work, the full review the concept's shell is compared
|
||||
// with.
|
||||
// Every concept-fixture prompt with an image is already tagged, so under the saved Skip there is nothing to start
|
||||
// (the Cancel-only review). Merge reviews a batch with work: the full review the concept's shell is compared with.
|
||||
await page.locator('#workspace-main #tag-existing').selectOption('merge');
|
||||
const answer = page.waitForResponse(response => response.url().endsWith(PREVIEW_URL));
|
||||
await page.locator('#workspace-main .operation').getByRole('button', { name: 'Review', exact: true }).click();
|
||||
@@ -335,9 +326,8 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
expect(concept.tree).toEqual(['p.demo-note', 'h3', 'dl.meta-grid', 'p.small', 'div.dialog-actions']);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-plan-review-${theme}-${width}.png`) });
|
||||
const { plan, production } = await productionReview(page, theme);
|
||||
// Under Merge every listed still image is tagged again: the batch has work to review.
|
||||
// Under Merge every prompt with a linked image is tagged again: the batch has work to review.
|
||||
expect(plan.to_tag).toBeGreaterThan(0);
|
||||
expect(plan.estimate_device).toBe('cpu');
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-tagging-review-${theme}-${width}.png`) });
|
||||
expect(diffInventory(mapBatchConcept(concept, plan), production)).toEqual([]);
|
||||
});
|
||||
@@ -382,17 +372,9 @@ test('tag output images review comparison detects mutations', async ({ page }) =
|
||||
const expected = mapBatchConcept(concept, plan);
|
||||
const paths = async () => diffInventory(expected, await page.evaluate(measureReview)).map(item => item.path);
|
||||
expect(await paths()).toEqual([]);
|
||||
// Every label of the grid is compared (#379's rows included): a changed one is named by its row.
|
||||
for (const [index, [term]] of expected.grid.rows.entries()) {
|
||||
await page.evaluate(([at, label]) => {
|
||||
document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[at].textContent = label;
|
||||
}, [index, `${term}!`]);
|
||||
expect(await paths()).toEqual([`grid.rows.${index}.0`]);
|
||||
await page.evaluate(([at, label]) => {
|
||||
document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[at].textContent = label;
|
||||
}, [index, term]);
|
||||
}
|
||||
expect(await paths()).toEqual([]);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[5].textContent = 'To tag'; });
|
||||
expect(await paths()).toEqual(['grid.rows.5.0']);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[5].textContent = 'Prompts to tag'; });
|
||||
const undoGrid = await mutate(page, { css: 'dialog#modal dl.meta-grid { column-gap: 3px; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['grid.columnGap']));
|
||||
await undoGrid();
|
||||
|
||||
@@ -42,13 +42,8 @@ test('actual gallery form saves a prompt that the admin page can edit and reload
|
||||
await page.locator(`[data-action="edit"][data-prompt-id="${id}"]`).click();
|
||||
const editor = page.locator(`.prompt-text[data-id="${id}"]`);
|
||||
await editor.fill('Browser edited portrait');
|
||||
// The page shows the edit at once and saves it after: reload only once the save answered (a reload that wins the
|
||||
// race abandons the PUT, as seen on WebKit in CI).
|
||||
const savedEdit = page.waitForResponse(item => item.request().method() === 'PUT'
|
||||
&& item.url().endsWith(`/prompt_manager/prompts/${id}`));
|
||||
await editor.press('Enter');
|
||||
await expect(page.locator('.prompt-text').filter({ hasText: 'Browser edited portrait' })).toBeVisible();
|
||||
expect((await savedEdit).ok()).toBe(true);
|
||||
await page.reload();
|
||||
await expect(page.locator('.prompt-text').filter({ hasText: 'Browser edited portrait' })).toBeVisible();
|
||||
const saved = await request.post('/prompt_manager/save', { data: { text: 'Browser saved landscape' } });
|
||||
@@ -132,8 +127,7 @@ test('the gallery cache clear keeps preferences and other tools caches', async (
|
||||
}))).toEqual({ theme: 'light', other: 'keep', prefixed: 'keep', notice: '3.2.0', caches: ['other-tool-gallery-thumbnails'] });
|
||||
});
|
||||
|
||||
// #379: the reviewed tags are that image's own tags; no prompt changes.
|
||||
test('actual gallery review generates and persists tags for the selected root\'s image', async ({ page, request }) => {
|
||||
test('actual gallery review generates and persists tags for the selected root', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2);
|
||||
await page.locator('#autoTagBtn').click();
|
||||
@@ -141,15 +135,13 @@ test('actual gallery review generates and persists tags for the selected root\'s
|
||||
await page.locator('#startReviewBtn').click();
|
||||
const generation = await generated;
|
||||
expect(generation.request().postDataJSON().root_index).toBe(0);
|
||||
await expect(page.locator('#reviewTagsContainer')).toContainText('sandstone');
|
||||
await expect(page.locator('#reviewTagsContainer')).toContainText('generated');
|
||||
const result = page.waitForResponse(response => response.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
await page.locator('#applyReviewBtn').click();
|
||||
expect((await (await result).json()).success).toBe(true);
|
||||
const data = await (await request.get('/prompt_manager/recent?limit=20')).json();
|
||||
expect(data.results.find(prompt => prompt.text === 'Cat portrait').tags).toEqual(['cats']);
|
||||
expect(data.results.find(prompt => prompt.text === 'Dog landscape').tags).toEqual(['dogs']);
|
||||
const tagged = await (await request.get('/prompt_manager/images/output?mode=workspace&tags=sandstone')).json();
|
||||
expect(tagged.images.map(image => `${image.root_index}:${image.path}`)).toEqual(['0:shared/same.png']);
|
||||
expect(data.results.find(prompt => prompt.text === 'Cat portrait').tags).toContain('generated');
|
||||
expect(data.results.find(prompt => prompt.text === 'Dog landscape').tags).not.toContain('generated');
|
||||
});
|
||||
|
||||
test('actual Tags page reloads the selected filter and opens its image viewer', async ({ page }) => {
|
||||
|
||||
@@ -2189,248 +2189,3 @@ test('scan for metadata opens the review dialog from Import', async ({ page, req
|
||||
await expect.poll(async () => (await (await request.get('/prompt_manager/jobs')).json()).jobs
|
||||
.map(job => [job.kind, job.state])).toEqual([['backup', 'finished']]);
|
||||
});
|
||||
|
||||
// #392 (D135): file-manager selection beyond the concept's checkbox.
|
||||
const selectedCount = page => bulkbar(page).locator('.bulkbar > strong');
|
||||
const cardAt = (page, index) => page.locator(`article[data-image-key][data-index="${index}"]`);
|
||||
const boxAt = (page, index) => cardAt(page, index).getByRole('checkbox', { name: /^Select / });
|
||||
const checkedIndexes = page => page.locator('article[data-image-key]').evaluateAll(nodes => nodes
|
||||
.filter(node => node.querySelector('.image-overlays input[type="checkbox"]').checked)
|
||||
.map(node => Number(node.dataset.index)).sort((a, b) => a - b));
|
||||
const range = (from, to) => Array.from({ length: to - from + 1 }, (_, index) => from + index);
|
||||
async function bulkReset(page, request) {
|
||||
await page.addInitScript(() => { window.IntersectionObserver = undefined; });
|
||||
await expect(await request.post('/__test__/reset', { data: { extra_images: 200 } })).toBeOK();
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(40);
|
||||
}
|
||||
/** A touch press on `locator` the way a finger makes one: pointerdown, (held), pointerup. */
|
||||
const touchDown = locator => locator.evaluate(node => {
|
||||
const { left, top } = node.getBoundingClientRect();
|
||||
node.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true, isPrimary: true, pointerType: 'touch', clientX: left + 5, clientY: top + 5 }));
|
||||
});
|
||||
const touchUpAndClick = locator => locator.evaluate(node => {
|
||||
node.dispatchEvent(new PointerEvent('pointerup', { bubbles: true, isPrimary: true, pointerType: 'touch' }));
|
||||
// A tap's click counts one press (`detail` 1); a keyboard click (and `HTMLElement.click()`) counts none.
|
||||
node.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, detail: 1 }));
|
||||
});
|
||||
async function touchTap(locator) {
|
||||
await touchDown(locator);
|
||||
await touchUpAndClick(locator);
|
||||
}
|
||||
|
||||
test('Shift+click selects a range across an appended batch in Grid, List and Masonry', async ({ page, request }) => {
|
||||
await bulkReset(page, request);
|
||||
await page.getByRole('button', { name: 'Load more images', exact: true }).click();
|
||||
await expect(caption(page)).toContainText('80 of 202 images');
|
||||
for (const mode of ['Grid', 'List', 'Masonry']) {
|
||||
await page.getByRole('group', { name: 'Image layout', exact: true }).getByRole('button', { name: mode, exact: true }).click();
|
||||
// The anchor is the last image toggled on its own (List hides the concept's checkbox: the card carries it).
|
||||
await cardAt(page, 3).locator('.media-info h3').click({ modifiers: ['ControlOrMeta'] });
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
// The range runs into the second batch (index 40 on), and backwards from the same anchor.
|
||||
await cardAt(page, 45).locator('.media-info h3').click({ modifiers: ['Shift'] });
|
||||
await expect(selectedCount(page)).toHaveText('43 selected');
|
||||
expect(await checkedIndexes(page)).toEqual(range(3, 45));
|
||||
await cardAt(page, 0).locator('.media-info h3').click({ modifiers: ['Shift'] });
|
||||
await expect(selectedCount(page)).toHaveText('46 selected');
|
||||
expect(await checkedIndexes(page)).toEqual(range(0, 45));
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
if (mode !== 'List') {
|
||||
// A Shift+click on a checkbox gives its range the box's new state: here, unchecked.
|
||||
await boxAt(page, 3).uncheck();
|
||||
await boxAt(page, 10).click({ modifiers: ['Shift'] });
|
||||
await expect(boxAt(page, 10)).not.toBeChecked();
|
||||
await expect(selectedCount(page)).toHaveText('38 selected');
|
||||
expect(await checkedIndexes(page)).toEqual([0, 1, 2, ...range(11, 45)]);
|
||||
}
|
||||
await bulkbar(page).getByRole('button', { name: 'Clear', exact: true }).click();
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
}
|
||||
});
|
||||
|
||||
test('Ctrl/Cmd+click toggles a card without opening it, and a plain click still opens the viewer', async ({ page }) => {
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
const first = cards(page).first();
|
||||
const inspectorBefore = await details(page).innerText();
|
||||
await first.getByRole('button', { name: 'Open same.png', exact: true }).click({ modifiers: ['ControlOrMeta'] });
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
await expect(checkbox(first, 'same.png')).toBeChecked();
|
||||
await cards(page).last().getByRole('button', { name: 'Inspect same.png', exact: true }).click({ modifiers: ['ControlOrMeta'] });
|
||||
await expect(selectedCount(page)).toHaveText('2 selected');
|
||||
await star(first, 'same.png').click({ modifiers: ['ControlOrMeta'] });
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
await expect(star(first, 'same.png')).toHaveAttribute('aria-pressed', 'false');
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
expect(await details(page).innerText()).toBe(inspectorBefore);
|
||||
// A plain click keeps its meaning: the image opens the viewer.
|
||||
await first.getByRole('button', { name: 'Open same.png', exact: true }).click();
|
||||
await expect(page.locator('#viewer-dialog[open]')).toHaveCount(1);
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
});
|
||||
|
||||
test('Ctrl/Cmd+A, Select all N matching and Add tags tag every matching image on the server', async ({ page, request }) => {
|
||||
await bulkReset(page, request);
|
||||
// Ctrl/Cmd+A in the search box stays the field's own.
|
||||
await search(page).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await expect(selectedCount(page)).toHaveText('40 selected');
|
||||
const offer = bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true });
|
||||
await expect(offer).toBeVisible();
|
||||
await expect(bulkbar(page).locator('.bulkbar > *').nth(1)).toHaveText('Select all 202 matching');
|
||||
await offer.click();
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
await expect(offer).toHaveCount(0);
|
||||
await expect(bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true })).toHaveAttribute('aria-disabled', 'true');
|
||||
// A later batch arrives checked.
|
||||
await page.getByRole('button', { name: 'Load more images', exact: true }).click();
|
||||
await expect(caption(page)).toContainText('80 of 202 images');
|
||||
await expect(boxAt(page, 70)).toBeChecked();
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// Unchecking one image leaves "all matching": the other loaded images stay selected.
|
||||
await boxAt(page, 0).uncheck();
|
||||
await expect(selectedCount(page)).toHaveText('79 selected');
|
||||
await expect(bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true })).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await cardAt(page, 1).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }).click();
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// Add tags sends the listing filter, not 202 identities, and the server tags every match.
|
||||
await bulkbar(page).getByRole('button', { name: 'Add tags', exact: true }).click();
|
||||
const dialog = page.getByRole('dialog', { name: 'Tag selected images', exact: true });
|
||||
await dialog.getByRole('textbox', { name: 'Tags, separated by commas', exact: true }).fill('picked392');
|
||||
const started = page.waitForRequest(item => item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/jobs/image-tags');
|
||||
await dialog.getByRole('button', { name: 'Apply', exact: true }).click();
|
||||
expect((await started).postDataJSON()).toEqual({ mode: 'add', filter: { sort: 'newest' }, tags: ['picked392'] });
|
||||
await expect.poll(async () => (await (await request.get('/prompt_manager/images/output?mode=workspace&limit=1&tags=picked392')).json()).total,
|
||||
{ timeout: 20000 }).toBe(202);
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// Escape in the grid clears the selection.
|
||||
await cardAt(page, 2).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
await expect(boxAt(page, 2)).not.toBeChecked();
|
||||
});
|
||||
|
||||
test('Shift+arrow keys extend the selection from the focused card, and Escape in the bar clears it', async ({ page, request }) => {
|
||||
await bulkReset(page, request);
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('Shift+ArrowRight');
|
||||
await page.keyboard.press('Shift+ArrowRight');
|
||||
await expect(cardAt(page, 2).getByRole('button', { name: /^Open / })).toBeFocused();
|
||||
await expect(selectedCount(page)).toHaveText('3 selected');
|
||||
expect(await checkedIndexes(page)).toEqual([0, 1, 2]);
|
||||
// A plain arrow moves without selecting; Shift+End reaches the last loaded card from the same anchor.
|
||||
await page.keyboard.press('ArrowRight');
|
||||
await expect(selectedCount(page)).toHaveText('3 selected');
|
||||
await page.keyboard.press('Shift+End');
|
||||
await expect(selectedCount(page)).toHaveText('40 selected');
|
||||
await bulkbar(page).getByRole('button', { name: 'Clear', exact: true }).focus();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
await expect(page.locator('#workspace-main h1')).toBeFocused();
|
||||
});
|
||||
|
||||
test('a touch long-press starts selection mode, taps then toggle, and an empty selection ends it', async ({ page }, testInfo) => {
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
const [first, second] = [cards(page).first(), cards(page).last()];
|
||||
const open = card => card.getByRole('button', { name: 'Open same.png', exact: true });
|
||||
await touchDown(open(first));
|
||||
// The context menu of a held press belongs to the long press.
|
||||
expect(await open(first).evaluate(node => !node.dispatchEvent(new MouseEvent('contextmenu', { bubbles: true, cancelable: true })))).toBe(true);
|
||||
// Held: the long press selects the card once its delay passed.
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
await touchUpAndClick(open(first));
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
await expect(checkbox(first, 'same.png')).toBeChecked();
|
||||
// Taps now toggle instead of opening.
|
||||
if (testInfo.project.name === 'touch') await open(second).tap();
|
||||
else await touchTap(open(second));
|
||||
await expect(selectedCount(page)).toHaveText('2 selected');
|
||||
// A keyboard Enter keeps its meaning in touch selection mode: the image opens.
|
||||
await open(first).focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(page.locator('#viewer-dialog[open]')).toHaveCount(1);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(page.locator('#viewer-dialog[open]')).toHaveCount(0);
|
||||
await expect(selectedCount(page)).toHaveText('2 selected');
|
||||
await touchTap(open(first));
|
||||
await touchTap(open(second));
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
// The selection is empty, so touch selection mode ended: a tap opens the image again.
|
||||
await touchTap(open(first));
|
||||
await expect(page.locator('#viewer-dialog[open]')).toHaveCount(1);
|
||||
});
|
||||
|
||||
test('all matching sets the category through the listing filter and survives a return visit as the loaded images', async ({ page, request }) => {
|
||||
await bulkReset(page, request);
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }).click();
|
||||
// The offer left with its press: focus moves to the bar's first action.
|
||||
await expect(bulkbar(page).getByRole('button', { name: 'Add tags', exact: true })).toBeFocused();
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// Export metadata waits for loaded images: a press does nothing.
|
||||
const exportButton = bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true });
|
||||
await expect(exportButton).toHaveAttribute('title', 'Export metadata reads each image in the browser, so it works on loaded images only.');
|
||||
// No match linked (a stubbed answer): the nothing-linked sentence, and the selection stays.
|
||||
await page.route('**/prompt_manager/bulk/category', route => route.fulfill({ json: { success: true, updated_count: 0, prompt_count: 0, skipped: 202 } }), { times: 1 });
|
||||
const categorize = page.getByRole('dialog', { name: 'Categorize selected images', exact: true });
|
||||
await bulkbar(page).getByRole('button', { name: 'Set category', exact: true }).click();
|
||||
await categorize.getByRole('textbox', { name: 'Category', exact: true }).fill('Bulk392');
|
||||
await categorize.getByRole('button', { name: 'Apply', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText("None of the selected images is linked to a saved prompt, so there is no prompt category to set.");
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// The real write: one request with the filter, every matching image's positive changes.
|
||||
await bulkbar(page).getByRole('button', { name: 'Set category', exact: true }).click();
|
||||
await categorize.getByRole('textbox', { name: 'Category', exact: true }).fill('Bulk392');
|
||||
const posted = page.waitForRequest(item => item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/bulk/category');
|
||||
await categorize.getByRole('button', { name: 'Apply', exact: true }).click();
|
||||
expect((await posted).postDataJSON()).toEqual({ images_filter: { sort: 'newest' }, category: 'Bulk392' });
|
||||
await expect(toast(page)).toHaveText(/^Category set on \d+ prompts?\.$/);
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
const filtered = await (await request.get('/prompt_manager/images/output?mode=workspace&limit=1&category=Bulk392')).json();
|
||||
expect(filtered.total).toBe(202);
|
||||
// A return visit keeps "all matching" as the loaded images, and the offer shows again.
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }).click();
|
||||
await page.evaluate(() => { window.location.hash = '/overview'; });
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Overview');
|
||||
await page.evaluate(() => { window.location.hash = '/images'; });
|
||||
await expect(selectedCount(page)).toHaveText('40 selected');
|
||||
await expect(bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true })).toBeVisible();
|
||||
// Escape in the bar with nothing selected does nothing; with a selection it clears.
|
||||
await bulkbar(page).getByRole('button', { name: 'Clear', exact: true }).focus();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
});
|
||||
|
||||
test('all matching over 10,000 images says so before any dialog or request', async ({ page, request }) => {
|
||||
await page.route(url => url.pathname === '/prompt_manager/images/output' && url.searchParams.get('limit') !== '1', async route => {
|
||||
const response = await route.fetch();
|
||||
const body = await response.json();
|
||||
await route.fulfill({ response, json: { ...body, total: 20002 } });
|
||||
});
|
||||
await bulkReset(page, request);
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await bulkbar(page).getByRole('button', { name: 'Select all 20,002 matching', exact: true }).click();
|
||||
await expect(selectedCount(page)).toHaveText('All 20,002 matching selected');
|
||||
let posted = 0;
|
||||
page.on('request', item => { if (item.method() === 'POST') posted += 1; });
|
||||
for (const action of ['Set category', 'Add tags', 'Auto-tag selected']) {
|
||||
await bulkbar(page).getByRole('button', { name: action, exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Narrow the view to at most 10,000 images.');
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
}
|
||||
expect(posted).toBe(0);
|
||||
});
|
||||
|
||||
@@ -1,590 +0,0 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
|
||||
// System › Diagnostics & logs (#249 System S12, #329) on the concept-shaped fixture, with the logger writing into this
|
||||
// server's own root (workspace_diagnostics_fixture.py): the meta-grid's live values, Run checks, the report, Inspect and
|
||||
// the test link (D110); the log viewer, statistics, downloads and runtime drafts (D109); and the reviewed rotated-log
|
||||
// cleanup (D100) and Clear all logs, which delete only inside that temporary log folder.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const REPORT_URL = '/prompt_manager/system/diagnostics';
|
||||
const LOGS_URL = '/prompt_manager/logs';
|
||||
const CONFIG_URL = '/prompt_manager/logs/config';
|
||||
const PREVIEW_URL = '/prompt_manager/maintenance/preview';
|
||||
const APPLY_URL = '/prompt_manager/maintenance/apply';
|
||||
const TEST_LINK_URL = '/prompt_manager/diagnostics/test-link';
|
||||
const CHANGED = 'The log files changed since the review; nothing was deleted. Review again.';
|
||||
const KEY = 'sk-s12-browser-SECRET-0123456789';
|
||||
const LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'];
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/integrations/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(await request.post('/__test__/logs/reset')).toBeOK();
|
||||
page.systemErrors = [];
|
||||
page.on('pageerror', error => page.systemErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/release');
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/logs/reset');
|
||||
await request.post('/__test__/integrations/reset');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.systemErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
const layoutOf = page => page.locator('#workspace-main .settings-layout');
|
||||
const dialog = page => page.locator('dialog#modal[open]');
|
||||
const surfaceOf = (page, title) => layoutOf(page).locator(':scope > section.surface').filter({
|
||||
has: page.getByRole('heading', { level: 2, name: title, exact: true }) });
|
||||
const cell = (page, term) => surfaceOf(page, 'Diagnostics').locator('.meta-grid > div').filter({
|
||||
has: page.locator(`dt:text-is("${term}")`) }).locator('dd');
|
||||
const lines = page => page.locator('#log-lines > code');
|
||||
const operation = (page, title) => layoutOf(page).locator('.operation').filter({ has: page.locator(`h3:text-is("${title}")`) });
|
||||
const settingOf = (page, label) => layoutOf(page).locator('.setting-row').filter({ has: page.locator(`label:text-is("${label}")`) });
|
||||
const status = page => page.locator('#settings-status');
|
||||
const save = page => page.getByRole('button', { name: 'Save preferences', exact: true });
|
||||
|
||||
async function serverJson(request, url) {
|
||||
const response = await request.get(url);
|
||||
await expect(response).toBeOK();
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function seedLogs(request, data = {}) {
|
||||
await expect(await request.post('/__test__/logs/seed', { data })).toBeOK();
|
||||
}
|
||||
|
||||
const names = async request => (await serverJson(request, '/__test__/logs/state')).entries.map(entry => entry.name);
|
||||
|
||||
async function openDiagnostics(page) {
|
||||
if (page.url().endsWith('#/system/diagnostics')) await page.reload();
|
||||
else await page.goto(`${workspace}#/system/diagnostics`);
|
||||
await expect(layoutOf(page).locator(':scope > section.surface')).toHaveCount(2);
|
||||
await expect(lines(page)).not.toHaveText('Loading…');
|
||||
await expect(cell(page, 'Database')).not.toHaveText('Loading…');
|
||||
// The renderer line settles first: its text can move the rows below.
|
||||
await expect(cell(page, 'Host renderers')).not.toHaveText('Loading…');
|
||||
await expect(page.locator('#log-stats')).not.toHaveText('Loading…');
|
||||
return layoutOf(page);
|
||||
}
|
||||
|
||||
async function openFiles(page) {
|
||||
const details = surfaceOf(page, 'Logs').locator('details');
|
||||
if (!(await details.evaluate(node => node.open))) await details.locator('summary').click();
|
||||
await expect(details.locator(':scope > p.small').first()).not.toHaveText('Loading…');
|
||||
return details;
|
||||
}
|
||||
|
||||
/** The levels of the viewer's lines, in order. */
|
||||
const shownLevels = async page => (await lines(page).textContent()).split('\n').filter(Boolean)
|
||||
.map(line => line.split(/ +/)[1]);
|
||||
|
||||
async function ended(request, id) {
|
||||
await expect.poll(async () => (await serverJson(request, `/prompt_manager/jobs/${id}`)).job.state).not.toBe('running');
|
||||
return (await serverJson(request, `/prompt_manager/jobs/${id}`)).job;
|
||||
}
|
||||
|
||||
/** A second page of this origin that serves a fake ComfyUI window on the send bridge. */
|
||||
async function fakeComfyWindow(context, renderer) {
|
||||
const host = await context.newPage();
|
||||
await host.goto(`${workspace}#/overview`);
|
||||
await host.evaluate(async drawn => {
|
||||
const { createBridgeHost } = await import('/prompt_manager/js/shared/toolkit/prompt-bridge.js');
|
||||
window.fakeHost = createBridgeHost({ namespace: '/', origin: location.origin, label: 'PromptManager — ComfyUI',
|
||||
adapter: { listTargets: () => [{ id: '7', label: 'Prompt Manager', type: 'PromptManager', roles: ['positive'],
|
||||
renderer: drawn }], preview() { throw new Error('unused'); }, apply() { throw new Error('unused'); } } });
|
||||
}, renderer);
|
||||
return host;
|
||||
}
|
||||
|
||||
async function readBlob(download) {
|
||||
const chunks = await (await download.createReadStream()).toArray();
|
||||
return Buffer.concat(chunks).toString('utf8');
|
||||
}
|
||||
|
||||
// ── Task 4: the Diagnostics surface ─────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test('meta-grid shows real database and source status, the pinned Viewer.js version and the host renderers',
|
||||
async ({ page, request, context }) => {
|
||||
const mutations = [];
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) mutations.push(item.url()); });
|
||||
await openDiagnostics(page);
|
||||
const report = await serverJson(request, REPORT_URL);
|
||||
const surface = surfaceOf(page, 'Diagnostics');
|
||||
await expect(surface.locator(':scope > .panel-heading > h2')).toHaveText('Diagnostics');
|
||||
await expect(surface.locator(':scope > .panel-heading > button')).toHaveText('Run checks');
|
||||
await expect(surface.locator('.meta-grid dt')).toHaveText(['Database', 'Sources', 'Viewer.js', 'Host renderers']);
|
||||
const images = report.checks.database.image_count;
|
||||
await expect(cell(page, 'Database')).toHaveText(new RegExp(`^Readable · [\\d.]+ (bytes|KB|MB|GB) · ${images.toLocaleString('en-US')} images?$`));
|
||||
// The size is the database's real size (`database_size_bytes`), never a missing key read as 0.
|
||||
expect(report.database_info.database_size_bytes).toBeGreaterThan(0);
|
||||
await expect(cell(page, 'Database')).not.toContainText('· 0 bytes ·');
|
||||
const { configured, watching } = report.sources;
|
||||
await expect(cell(page, 'Sources')).toHaveText(configured
|
||||
? `${configured} source${configured === 1 ? '' : 's'} · ${watching} watching`
|
||||
: `ComfyUI output folder · ${watching ? 'watching' : 'manual scan'}`);
|
||||
await expect(cell(page, 'Viewer.js')).toHaveText('1.15.2 · pinned vendor');
|
||||
await expect(cell(page, 'Host renderers')).toHaveText('No ComfyUI window is open in this browser, so its renderer is not known.');
|
||||
// A ComfyUI window with a Classic node: the line Integrations shows (D106).
|
||||
const host = await fakeComfyWindow(context, 'classic');
|
||||
await openDiagnostics(page);
|
||||
await expect(cell(page, 'Host renderers')).toHaveText('Detected: Classic (1 ComfyUI window).');
|
||||
await host.close();
|
||||
// A refused read names the running job (D57's naming).
|
||||
await page.route(`**${REPORT_URL}`, route => route.fulfill({ status: 409, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, code: 'busy', error: 'busy', running: { id: 'a'.repeat(32), title: 'Restore' } }) }));
|
||||
await openDiagnostics(page);
|
||||
await expect(cell(page, 'Database')).toHaveText('Busy: Restore is running.');
|
||||
await page.unroute(`**${REPORT_URL}`);
|
||||
await expect(await request.post('/__test__/system/fail', { data: { path: REPORT_URL, times: 1 } })).toBeOK();
|
||||
await openDiagnostics(page);
|
||||
await expect(cell(page, 'Sources')).toHaveText('Could not be read.');
|
||||
expect(mutations).toEqual([]);
|
||||
});
|
||||
|
||||
test('Run checks opens the shared Health report', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
await openDiagnostics(page);
|
||||
const run = surfaceOf(page, 'Diagnostics').getByRole('button', { name: 'Run checks', exact: true });
|
||||
await run.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Health report');
|
||||
await expect(run).toHaveText('Run checks');
|
||||
await dialog(page).getByRole('button', { name: 'Close', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(run).toBeFocused();
|
||||
// The same report Overview reads: Overview's metric names it.
|
||||
expect((await serverJson(request, '/prompt_manager/system/health')).report).toBeTruthy();
|
||||
expect((await serverJson(request, '/__test__/system/digest')).rows).toEqual(before.rows);
|
||||
});
|
||||
|
||||
test('report download goes through the api client and holds no secret or absolute path', async ({ page, request }) => {
|
||||
const { revision } = await serverJson(request, '/prompt_manager/settings');
|
||||
await expect(await request.post('/prompt_manager/lora/enable', { data: { enabled: false, trigger_words_enabled: false, path: '',
|
||||
civitai_api_key: KEY, expected_revision: revision } })).toBeOK();
|
||||
await openDiagnostics(page);
|
||||
const reads = [];
|
||||
page.on('request', item => { if (item.url().endsWith(REPORT_URL)) reads.push(item.url()); });
|
||||
const download = page.waitForEvent('download');
|
||||
await surfaceOf(page, 'Diagnostics').getByRole('button', { name: 'Download diagnostic report', exact: true }).click();
|
||||
const file = await download;
|
||||
expect(file.suggestedFilename()).toMatch(/^prompt-manager-diagnostics-\d{8}-\d{4}\.json$/);
|
||||
await expect(toast(page)).toHaveText('Diagnostic report downloaded.');
|
||||
expect(reads).toHaveLength(1);
|
||||
const text = await readBlob(file);
|
||||
const report = JSON.parse(text);
|
||||
expect(report.host_renderers).toBe('No ComfyUI window is open in this browser, so its renderer is not known.');
|
||||
expect(report.config.integrations.has_civitai_api_key).toBe(true);
|
||||
expect(report.redacted).toBe(0);
|
||||
expect(report.viewerjs).toBe('1.15.2');
|
||||
expect(Object.keys(report.checks)).toEqual(['database', 'dependencies', 'comfyui_output', 'image_monitor']);
|
||||
expect(text).not.toContain(KEY);
|
||||
expect(text).not.toContain('pm-browser-');
|
||||
const strings = [];
|
||||
const walk = value => {
|
||||
if (typeof value === 'string') strings.push(value);
|
||||
else if (value && typeof value === 'object') Object.values(value).forEach(walk);
|
||||
};
|
||||
walk(report);
|
||||
expect(strings.filter(value => /^(\/|[A-Za-z]:[\\/]|\\\\)/.test(value))).toEqual([]);
|
||||
// A refused report names the job and saves nothing.
|
||||
await page.route(`**${REPORT_URL}`, route => route.fulfill({ status: 409, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, code: 'busy', error: 'busy', running: { id: 'a'.repeat(32), title: 'Restore' } }) }));
|
||||
let saved = false;
|
||||
page.on('download', () => { saved = true; });
|
||||
await surfaceOf(page, 'Diagnostics').getByRole('button', { name: 'Download diagnostic report', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Busy: Restore is running. Try again when it ends.');
|
||||
await page.unroute(`**${REPORT_URL}`);
|
||||
await page.route(`**${REPORT_URL}`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'The diagnostic report could not be built.' }) }));
|
||||
await surfaceOf(page, 'Diagnostics').getByRole('button', { name: 'Download diagnostic report', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The diagnostic report could not be downloaded. Nothing was saved to this device.');
|
||||
expect(saved).toBe(false);
|
||||
});
|
||||
|
||||
test('Inspect opens the Inspect a file page and writes nothing', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
const mutations = [];
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) mutations.push(item.url()); });
|
||||
await openDiagnostics(page);
|
||||
const row = operation(page, 'Image-link diagnostics');
|
||||
await expect(row.locator('p')).toHaveText('Check a file-to-prompt association without changing it.');
|
||||
await row.getByRole('button', { name: 'Inspect', exact: true }).click();
|
||||
await expect(page).toHaveURL(`${workspace}#/inspect`);
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Inspect a file');
|
||||
expect(mutations).toEqual([]);
|
||||
expect(await serverJson(request, '/__test__/system/digest')).toEqual(before);
|
||||
});
|
||||
|
||||
test('Create test link confirms first and names the prompt', async ({ page, request }) => {
|
||||
const [prompt] = (await serverJson(request, '/prompt_manager/recent?limit=1')).results;
|
||||
const posts = [];
|
||||
page.on('request', item => { if (item.url().endsWith(TEST_LINK_URL)) posts.push(item.postDataJSON()); });
|
||||
await openDiagnostics(page);
|
||||
const row = operation(page, 'Test image link');
|
||||
await expect(row.locator('p')).toHaveText('Write a placeholder image record to check that linking works.');
|
||||
const create = row.getByRole('button', { name: 'Create test link…', exact: true });
|
||||
await expect(create).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await create.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Create test link');
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText(new RegExp('^Writes a placeholder image record '
|
||||
+ `\\(prompt_manager_test_link\\.png\\) linked to prompt #${prompt.id} “.+”\\. Maintenance › Missing-file links removes it later\\.$`));
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(posts).toEqual([]);
|
||||
await create.click();
|
||||
await dialog(page).getByRole('button', { name: 'Create', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText(/^Test link created \(image record \d+\)\.$/);
|
||||
expect(posts).toEqual([{ prompt_id: prompt.id }]);
|
||||
// A failed write says nothing was changed.
|
||||
await page.route(`**${TEST_LINK_URL}`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'Test link failed' }) }));
|
||||
await create.click();
|
||||
await dialog(page).getByRole('button', { name: 'Create', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The test link could not be created. Nothing was changed.');
|
||||
// With no prompt to link, the button is unavailable and says why.
|
||||
await page.route('**/prompt_manager/recent?limit=1', route => route.fulfill({ status: 200, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: true, results: [] }) }));
|
||||
await openDiagnostics(page);
|
||||
await expect(operation(page, 'Test image link').locator('p'))
|
||||
.toHaveText('Write a placeholder image record to check that linking works. Add a prompt first.');
|
||||
await expect(operation(page, 'Test image link').getByRole('button')).toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
|
||||
// ── Task 5: the log viewer, statistics and runtime settings ─────────────────────────────────────────────────────────
|
||||
|
||||
test('level filter is minimum-level over all six choices', async ({ page, request }) => {
|
||||
await seedLogs(request);
|
||||
const urls = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === LOGS_URL) urls.push(new URL(item.url()).search); });
|
||||
await openDiagnostics(page);
|
||||
const level = page.locator('#log-level');
|
||||
await expect(level.locator('option')).toHaveText(['All levels', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']);
|
||||
await expect(page.locator('label[for="log-level"]')).toHaveText('Level');
|
||||
// All levels: the concept's four lines first, newest first, in its format.
|
||||
await expect(lines(page)).toContainText('09:42:18 INFO Scan completed: 12 enriched, 0 duplicate links\n'
|
||||
+ '09:42:18 WARNING 3 images have ambiguous prompt metadata\n09:39:04 INFO Database backup verified\n'
|
||||
+ '09:38:52 INFO Source monitor started: output');
|
||||
for (const line of (await lines(page).textContent()).split('\n')) expect(line).toMatch(/^\d\d:\d\d:\d\d [A-Z]+ +\S/);
|
||||
expect(urls.at(-1)).toBe('?limit=100');
|
||||
for (const minimum of LEVELS) {
|
||||
await level.selectOption(minimum);
|
||||
await expect.poll(() => urls.at(-1)).toBe(`?limit=100&level=${minimum}`);
|
||||
await expect(lines(page)).toContainText('CRITICAL The database could not be opened');
|
||||
const shown = await shownLevels(page);
|
||||
expect(shown.every(name => LEVELS.indexOf(name) >= LEVELS.indexOf(minimum)), `${minimum}: ${shown}`).toBe(true);
|
||||
if (minimum === 'DEBUG') await expect(lines(page)).toContainText('DEBUG Thumbnail cache lookup');
|
||||
if (minimum === 'WARNING') {
|
||||
await expect(lines(page)).not.toContainText('Database backup verified');
|
||||
await expect(lines(page)).toContainText('ERROR A source folder could not be read');
|
||||
}
|
||||
}
|
||||
await level.selectOption('');
|
||||
await expect.poll(() => urls.at(-1)).toBe('?limit=100');
|
||||
});
|
||||
|
||||
test('Lines accepts 1 to 500 and sends nothing when invalid', async ({ page, request }) => {
|
||||
await seedLogs(request, { filler: 30 });
|
||||
const urls = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === LOGS_URL) urls.push(new URL(item.url()).search); });
|
||||
await openDiagnostics(page);
|
||||
const limit = page.locator('#log-limit');
|
||||
await expect(page.locator('label[for="log-limit"]')).toHaveText('Lines');
|
||||
await expect(limit).toHaveValue('100');
|
||||
expect(await limit.evaluate(node => [node.type, node.min, node.max, node.title]))
|
||||
.toEqual(['number', '1', '500', 'Lines to show: a whole number from 1 to 500.']);
|
||||
await limit.fill('20');
|
||||
await limit.dispatchEvent('change');
|
||||
await expect.poll(() => urls.at(-1)).toBe('?limit=20');
|
||||
await expect.poll(async () => (await lines(page).textContent()).split('\n').length).toBe(20);
|
||||
const sent = urls.length;
|
||||
for (const value of ['0', '501', '2.5', '']) {
|
||||
await limit.fill(value);
|
||||
await limit.dispatchEvent('change');
|
||||
await expect(limit).toHaveAttribute('aria-invalid', 'true');
|
||||
}
|
||||
// A Level change while Lines is invalid sends nothing either.
|
||||
await page.locator('#log-level').selectOption('WARNING');
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
expect(urls.length).toBe(sent);
|
||||
await limit.fill('500');
|
||||
await limit.dispatchEvent('change');
|
||||
await expect(limit).not.toHaveAttribute('aria-invalid');
|
||||
await expect.poll(() => urls.at(-1)).toBe('?limit=500&level=WARNING');
|
||||
});
|
||||
|
||||
test('logs refresh handles empty, error and stale answers', async ({ page, request }) => {
|
||||
await seedLogs(request);
|
||||
await openDiagnostics(page);
|
||||
const level = page.locator('#log-level');
|
||||
const refresh = surfaceOf(page, 'Logs').getByRole('button', { name: 'Refresh', exact: true });
|
||||
// Empty: nothing at this level once the buffer is cleared.
|
||||
await expect(await request.post('/__test__/logs/reset')).toBeOK();
|
||||
await level.selectOption('CRITICAL');
|
||||
await expect(lines(page)).toHaveText('No log entries at this level.');
|
||||
// Error.
|
||||
await page.route(`**${LOGS_URL}?*`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'boom', logs: [] }) }));
|
||||
await refresh.click();
|
||||
await expect(lines(page)).toHaveText('The log could not be read.');
|
||||
await page.unroute(`**${LOGS_URL}?*`);
|
||||
// Stale: an older answer that arrives after a newer request never paints.
|
||||
await seedLogs(request);
|
||||
await expect(await request.post('/__test__/hold', { data: { path: LOGS_URL } })).toBeOK();
|
||||
await level.selectOption('DEBUG');
|
||||
await expect.poll(async () => (await serverJson(request, '/__test__/hold')).held[LOGS_URL] ?? 0).toBe(1);
|
||||
await level.selectOption('ERROR');
|
||||
await expect.poll(async () => (await serverJson(request, '/__test__/hold')).held[LOGS_URL] ?? 0).toBe(2);
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await expect(lines(page)).toContainText('CRITICAL The database could not be opened');
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
expect((await shownLevels(page)).every(name => ['ERROR', 'CRITICAL'].includes(name))).toBe(true);
|
||||
});
|
||||
|
||||
test('log statistics show entries, errors, warnings, files and size', async ({ page, request }) => {
|
||||
await seedLogs(request, { rotated: [2048, 1024], other: true, link: true, active: 512 });
|
||||
await openDiagnostics(page);
|
||||
const stats = page.getByRole('status', { name: 'Log statistics', exact: true });
|
||||
await expect(stats).toHaveAttribute('id', 'log-stats');
|
||||
const text = await stats.textContent();
|
||||
const match = /^([\d,]+) entr(?:y|ies) in memory · (\d+) errors? · (\d+) warnings? · (\d+) files? · (.+) on disk$/.exec(text);
|
||||
expect(match, text).not.toBeNull();
|
||||
const [, entries, errors, warnings, files, size] = match;
|
||||
expect(Number(entries.replace(/,/g, ''))).toBeGreaterThanOrEqual(7);
|
||||
expect(Number(errors)).toBeGreaterThanOrEqual(2);
|
||||
expect(Number(warnings)).toBeGreaterThanOrEqual(1);
|
||||
// The active log, two rotated copies and the .bak; the link is not counted (Rule R).
|
||||
expect(Number(files)).toBe(4);
|
||||
expect(size).toMatch(/^[\d.]+ (bytes|KB|MB)$/);
|
||||
});
|
||||
|
||||
test('Download log saves the active log', async ({ page, request }) => {
|
||||
await seedLogs(request, { active: 300 });
|
||||
await openDiagnostics(page);
|
||||
const download = page.waitForEvent('download');
|
||||
await surfaceOf(page, 'Logs').getByRole('button', { name: 'Download log', exact: true }).click();
|
||||
const file = await download;
|
||||
expect(file.suggestedFilename()).toBe('prompt_manager.log');
|
||||
expect((await readBlob(file)).length).toBeGreaterThanOrEqual(300);
|
||||
await page.route('**/prompt_manager/logs/download/**', route => route.fulfill({ status: 404, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'Log file not found' }) }));
|
||||
await surfaceOf(page, 'Logs').getByRole('button', { name: 'Download log', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The log could not be downloaded.');
|
||||
});
|
||||
|
||||
test('runtime level (including CRITICAL) and console logging save and say they apply until the server restarts',
|
||||
async ({ page, request }) => {
|
||||
const posts = [];
|
||||
page.on('request', item => { if (item.method() === 'POST' && item.url().endsWith(CONFIG_URL)) posts.push(item.postDataJSON()); });
|
||||
await openDiagnostics(page);
|
||||
const level = settingOf(page, 'Runtime log level');
|
||||
await expect(level.locator('p')).toHaveText('How much the server writes to the log. Applied until the server restarts.');
|
||||
await expect(level.locator('option')).toHaveText(['INFO', 'WARNING', 'DEBUG', 'ERROR', 'CRITICAL']);
|
||||
await expect(level.locator('select')).toHaveValue('INFO');
|
||||
const consoleRow = settingOf(page, 'Console logging');
|
||||
await expect(consoleRow.locator('p')).toHaveText('Print Prompt Manager’s warnings and errors in the ComfyUI console '
|
||||
+ '(everything at DEBUG). Applied until the server restarts.');
|
||||
const toggle = consoleRow.locator('input[type="checkbox"]');
|
||||
await expect(toggle).toBeChecked();
|
||||
await level.locator('select').selectOption('CRITICAL');
|
||||
await toggle.uncheck();
|
||||
await expect(status(page)).toHaveText('Unsaved changes.');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Applied until the server restarts.');
|
||||
expect(posts).toEqual([{ level: 'CRITICAL', console_logging: false }]);
|
||||
const { config } = await serverJson(request, CONFIG_URL);
|
||||
expect([config.level, config.console_logging]).toEqual(['CRITICAL', false]);
|
||||
// Read again on a fresh page: the server's values.
|
||||
await openDiagnostics(page);
|
||||
await expect(settingOf(page, 'Runtime log level').locator('select')).toHaveValue('CRITICAL');
|
||||
await expect(settingOf(page, 'Console logging').locator('input[type="checkbox"]')).not.toBeChecked();
|
||||
// Only the changed key is sent.
|
||||
await settingOf(page, 'Runtime log level').locator('select').selectOption('DEBUG');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Applied until the server restarts.');
|
||||
expect(posts.at(-1)).toEqual({ level: 'DEBUG' });
|
||||
});
|
||||
|
||||
test('a refused runtime value is marked invalid', async ({ page }) => {
|
||||
await page.route(`**${CONFIG_URL}`, route => (route.request().method() === 'POST'
|
||||
? route.fulfill({ status: 400, contentType: 'application/json', body: JSON.stringify({ success: false, error: 'Invalid log level' }) })
|
||||
: route.continue()));
|
||||
await openDiagnostics(page);
|
||||
const select = settingOf(page, 'Runtime log level').locator('select');
|
||||
await select.selectOption('WARNING');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Some changes were not saved. Review the highlighted settings.');
|
||||
await expect(select).toHaveAttribute('aria-invalid', 'true');
|
||||
await expect(settingOf(page, 'Console logging').locator('input[type="checkbox"]')).not.toHaveAttribute('aria-invalid');
|
||||
// The draft stays: an edit clears the mark.
|
||||
await expect(select).toHaveValue('WARNING');
|
||||
await select.selectOption('ERROR');
|
||||
await expect(select).not.toHaveAttribute('aria-invalid');
|
||||
});
|
||||
|
||||
test('runtime values that cannot be read stay disabled and say so', async ({ page }) => {
|
||||
await page.route(`**${CONFIG_URL}`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'boom' }) }));
|
||||
await openDiagnostics(page);
|
||||
for (const label of ['Runtime log level', 'Console logging']) {
|
||||
const row = settingOf(page, label);
|
||||
await expect(row.locator('p.small')).toHaveText('The current value could not be read.');
|
||||
await expect(row.locator('select, input[type="checkbox"]')).toBeDisabled();
|
||||
}
|
||||
});
|
||||
|
||||
// ── Task 6: log files, the reviewed cleanup and Clear all logs ─────────────────────────────────────────────────────
|
||||
|
||||
const LOG_FOLDER = { rotated: [3000, 2048, 1024], other: true, link: true, active: 100 };
|
||||
const LINKS = process.platform === 'win32' ? 0 : 1;
|
||||
|
||||
test('Review log cleanup lists only rotated copies and keeps the rest', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
const before = await names(request);
|
||||
const previews = [];
|
||||
page.on('request', item => { if (item.url().endsWith(PREVIEW_URL)) previews.push(item.postDataJSON()); });
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/^prompt_manager\.log · [\d.]+ (bytes|KB) · rotated copies: 3$/);
|
||||
await details.getByRole('button', { name: 'Review log cleanup', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review maintenance plan');
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Rotated log copies');
|
||||
expect(previews).toEqual([{ operation: 'rotated_logs', options: {} }]);
|
||||
const grid = await dialog(page).locator('dl.meta-grid > div').evaluateAll(items => items.map(item =>
|
||||
[item.querySelector('dt').textContent, item.querySelector('dd').textContent]));
|
||||
expect(grid.map(([term]) => term)).toEqual(['Scope', 'Protected', 'Backup', 'Media on disk']);
|
||||
expect(grid[0][1]).toMatch(/^3 rotated log copies · 5\.9 KB$/);
|
||||
expect(grid[1][1]).toBe(['Kept: the active log', ...(LINKS ? ['Kept: 1 link'] : []), 'Kept: 1 other file'].join(' · '));
|
||||
expect(grid[2][1]).toBe('Not needed: no library record changes');
|
||||
expect(grid[3][1]).toBe('Deletes only the listed log files. No image, video or library record changes.');
|
||||
await expect(dialog(page).locator('details > summary')).toHaveText('Listed files (3)');
|
||||
await expect(dialog(page).locator('details pre')).toHaveText(/^prompt_manager\.log\.1 · 2\.9 KB · .+\nprompt_manager\.log\.2 · 2\.0 KB · .+\nprompt_manager\.log\.3 · 1\.0 KB · .+$/);
|
||||
// No D55 backup box: nothing in the library changes.
|
||||
await expect(dialog(page).locator('.notice')).toHaveCount(0);
|
||||
await expect(dialog(page).getByRole('button', { name: 'Back up now' })).toHaveCount(0);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(await names(request)).toEqual(before);
|
||||
});
|
||||
|
||||
test('Apply deletes exactly the listed files and Activity lists them', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await details.getByRole('button', { name: 'Review log cleanup', exact: true }).click();
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
const job = await ended(request, page.url().split('/').pop());
|
||||
expect([job.kind, job.state, job.title, job.counters.changed]).toEqual(['maintenance', 'finished', 'Rotated log copies', 3]);
|
||||
await expect(toast(page)).toHaveText(/Rotated log copies (started|finished · 3 changed)/);
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(main.locator('h1')).toHaveText('Rotated log copies');
|
||||
for (const name of ['prompt_manager.log.1', 'prompt_manager.log.2', 'prompt_manager.log.3']) await expect(main).toContainText(name);
|
||||
// Only those three are gone: the active log, the user's .bak and the link are kept.
|
||||
expect(await names(request)).toEqual(['prompt_manager.log', ...(LINKS ? ['prompt_manager.log.4'] : []), 'prompt_manager.log.bak']);
|
||||
});
|
||||
|
||||
test('a rollover after the review deletes nothing and Activity offers Review again', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await details.getByRole('button', { name: 'Review log cleanup', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Rotated log copies');
|
||||
await expect(await request.post('/__test__/logs/rotate')).toBeOK();
|
||||
const rolled = await serverJson(request, '/__test__/logs/state');
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
const job = await ended(request, page.url().split('/').pop());
|
||||
expect([job.state, job.scope.failure]).toEqual(['failed', 'plan-changed']);
|
||||
await expect(page.locator('#workspace-main')).toContainText(CHANGED);
|
||||
expect((await serverJson(request, '/__test__/logs/state')).entries.map(entry => entry.name))
|
||||
.toEqual(rolled.entries.map(entry => entry.name));
|
||||
const previews = [];
|
||||
page.on('request', item => { if (item.url().endsWith(PREVIEW_URL)) previews.push(item.postDataJSON()); });
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Review again', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Rotated log copies');
|
||||
expect(previews).toEqual([{ operation: 'rotated_logs', options: {} }]);
|
||||
});
|
||||
|
||||
test('Clear all logs states both counts, confirms first and reads everything again', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
const truncates = [];
|
||||
page.on('request', item => { if (item.url().endsWith('/prompt_manager/logs/truncate')) truncates.push(item.url()); });
|
||||
await openDiagnostics(page);
|
||||
await expect(lines(page)).toContainText('Database backup verified');
|
||||
const details = await openFiles(page);
|
||||
const clearRow = operation(page, 'Clear all logs');
|
||||
await expect(clearRow.locator('p')).toHaveText('Empty the active log and delete its rotated copies.');
|
||||
const clear = clearRow.getByRole('button', { name: 'Clear all logs…', exact: true });
|
||||
await expect(clear).toHaveClass(/\bdanger\b/);
|
||||
await clear.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Clear all logs');
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText('Empties prompt_manager.log, deletes 3 rotated copies (5.9 KB) '
|
||||
+ 'and clears the lines shown here. The other files in the log folder are kept. This cannot be undone.');
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(truncates).toEqual([]);
|
||||
await clear.click();
|
||||
await dialog(page).getByRole('button', { name: 'Clear', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Logs cleared · 3 rotated copies deleted.');
|
||||
expect(truncates).toHaveLength(1);
|
||||
// Read again: the viewer, the file line and the list.
|
||||
await expect(lines(page)).not.toContainText('Database backup verified');
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/rotated copies: 0$/);
|
||||
const kept = ['prompt_manager.log', 'prompt_manager.log.bak', ...(LINKS ? ['prompt_manager.log.4'] : [])];
|
||||
// The server lists newest first: compare as a set.
|
||||
await expect.poll(async () => (await details.locator('.operation-list h3').allTextContents()).sort()).toEqual([...kept].sort());
|
||||
expect(await names(request)).toEqual(['prompt_manager.log', ...(LINKS ? ['prompt_manager.log.4'] : []), 'prompt_manager.log.bak']);
|
||||
});
|
||||
|
||||
test('Clear all logs reads the files again before it states what it deletes', async ({ page, request }) => {
|
||||
await seedLogs(request, { rotated: [1024], active: 100 });
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/rotated copies: 1$/);
|
||||
// The log rolls over while the page stays open: Clear names what it deletes now, not the count shown.
|
||||
await seedLogs(request, { records: false, rotated: [1024, 1024, 1024] });
|
||||
await operation(page, 'Clear all logs').getByRole('button', { name: 'Clear all logs…', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText(/^Empties prompt_manager\.log, deletes 3 rotated copies \(3\.0 KB\) /);
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/rotated copies: 3$/);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
// A list that cannot be read gives the general sentence, never an old count.
|
||||
await page.route('**/prompt_manager/logs/files', route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'boom', files: [] }) }));
|
||||
await operation(page, 'Clear all logs').getByRole('button', { name: 'Clear all logs…', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText(/^Empties prompt_manager\.log, deletes its rotated copies and clears/);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
expect(await names(request)).toEqual(['prompt_manager.log', 'prompt_manager.log.1', 'prompt_manager.log.2', 'prompt_manager.log.3']);
|
||||
});
|
||||
|
||||
test('each listed file downloads by its returned name and a link has no Download', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await expect(details.locator(':scope > p.small').nth(1))
|
||||
.toHaveText('Rotates at 10 MB · keeps 5 rotated copies · file logging on · memory buffer 1,000 entries');
|
||||
const list = details.locator('.operation-list');
|
||||
expect((await list.locator('h3').allTextContents()).sort()).toEqual(['prompt_manager.log', 'prompt_manager.log.1',
|
||||
'prompt_manager.log.2', 'prompt_manager.log.3', ...(LINKS ? ['prompt_manager.log.4'] : []), 'prompt_manager.log.bak'].sort());
|
||||
const item = name => list.locator('.operation').filter({ has: page.locator(`h3:text-is("${name}")`) });
|
||||
await expect(item('prompt_manager.log.1').locator('p')).toHaveText(/^2\.9 KB · \d{4}-\d\d-\d\d \d\d:\d\d · Rotated copy$/);
|
||||
await expect(item('prompt_manager.log.bak').locator('p')).toHaveText(/^0 bytes · .+ · Other file$/);
|
||||
if (LINKS) {
|
||||
await expect(item('prompt_manager.log.4').locator('p')).toHaveText(/ · Link, not downloaded$/);
|
||||
await expect(item('prompt_manager.log.4').getByRole('button')).toHaveCount(0);
|
||||
}
|
||||
for (const name of ['prompt_manager.log.1', 'prompt_manager.log.bak']) {
|
||||
const download = page.waitForEvent('download');
|
||||
await item(name).getByRole('button', { name: `Download ${name}`, exact: true }).click();
|
||||
const file = await download;
|
||||
expect(file.suggestedFilename()).toBe(name);
|
||||
if (name === 'prompt_manager.log.1') expect((await readBlob(file)).length).toBe(3000);
|
||||
}
|
||||
});
|
||||
@@ -315,28 +315,22 @@ test('Preview tagging review without a selection toasts the D115 sentence', asyn
|
||||
await expect(tagDialog(page)).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('Tag output images reviews the images, tags each one and never a prompt (#379)', async ({ page, request }) => {
|
||||
test('Tag output images reviews the scope and starts a batch job', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/__test__/tagging/snapshot');
|
||||
await openTagging(page);
|
||||
// The saved policy is Skip: no image is auto-tagged yet, so every listed still image is to tag.
|
||||
// The fixture's one prompt is already tagged: under Merge it is tagged again with what its images give.
|
||||
await page.locator('#tag-existing').selectOption('merge');
|
||||
const previewed = page.waitForResponse(item => new URL(item.url()).pathname === PREVIEW_URL);
|
||||
await reviewButton(page).click();
|
||||
const answer = await previewed;
|
||||
expect(answer.request().postDataJSON()).toEqual({ model_type: 'wd14-swinv2', existing: 'skip' });
|
||||
expect(answer.request().postDataJSON()).toEqual({ model_type: 'wd14-swinv2', existing: 'merge' });
|
||||
const plan = await answer.json();
|
||||
// Four still images (two linked, a copy, an unlinked one); the video is not a target.
|
||||
expect([plan.images, plan.already_tagged, plan.to_tag]).toEqual([4, 0, 4]);
|
||||
expect(plan.prompts).toBeUndefined();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review tag output images');
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Tag output images');
|
||||
expect(await gridOf(page)).toEqual([['Model', 'WD14 SwinV2 · ~400 MB'], ['Existing tags', 'Skip already tagged'],
|
||||
['Images', '4'], ['Already auto-tagged', '0'], ['Images to tag', 'about 4'],
|
||||
['Time', 'about 1 min on a processor; a graphics card can be faster.'],
|
||||
['Settings', `General 0.35 · Character 0.85 · up to ${plan.suggestion_limit} tags per image`],
|
||||
expect(await gridOf(page)).toEqual([['Model', 'WD14 SwinV2 · ~400 MB'], ['Existing tags', 'Merge with existing'],
|
||||
['Images linked', String(plan.images)], ['Prompts', String(plan.prompts)], ['Already tagged', String(plan.already_tagged)],
|
||||
['Prompts to tag', `up to ${plan.to_tag}`], ['Settings', 'General 0.35 · Character 0.85'],
|
||||
['Backup', 'Create and verify before changes']]);
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Nothing is written until you start. The number to '
|
||||
+ 'tag is an estimate: images changed since they were tagged are tagged again, and missing files are skipped. '
|
||||
+ 'Activity reports what was tagged.');
|
||||
await expect(dialog(page).locator('#modal-body > .notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
// The review wrote nothing.
|
||||
expect(await serverJson(request, '/__test__/tagging/snapshot')).toEqual(before);
|
||||
@@ -344,23 +338,15 @@ test('Tag output images reviews the images, tags each one and never a prompt (#3
|
||||
const started = page.waitForResponse(item => new URL(item.url()).pathname === '/prompt_manager/jobs/autotag');
|
||||
await dialog(page).getByRole('button', { name: 'Start tagging', exact: true }).click();
|
||||
const start = await started;
|
||||
expect(start.request().postDataJSON()).toEqual({ model_type: 'wd14-swinv2', existing: 'skip',
|
||||
expect(start.request().postDataJSON()).toEqual({ model_type: 'wd14-swinv2', existing: 'merge',
|
||||
options: { general_threshold: 0.35, character_threshold: 0.85 } });
|
||||
const { job } = await start.json();
|
||||
await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`);
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Tag output images');
|
||||
const finished = await ended(request, job.id);
|
||||
expect(finished.state).toBe('finished');
|
||||
expect(finished.status_line).toBe('Tagged 4 images');
|
||||
// Each image got its own tags; no prompt, pair or link row changed.
|
||||
expect(await serverJson(request, '/__test__/tagging/snapshot')).toEqual(before);
|
||||
|
||||
// Under Skip every image is now auto-tagged: nothing to start.
|
||||
await openTagging(page);
|
||||
await reviewButton(page).click();
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Every output image is already auto-tagged.');
|
||||
expect((await gridOf(page)).slice(2, 5)).toEqual([['Images', '4'], ['Already auto-tagged', '4'], ['Images to tag', 'about 0']]);
|
||||
await expect(dialog(page).locator('.dialog-actions button')).toHaveText(['Cancel']);
|
||||
const after = await serverJson(request, '/__test__/tagging/snapshot');
|
||||
expect(after.prompt_tags.length).toBeGreaterThan(before.prompt_tags.length);
|
||||
});
|
||||
|
||||
test('Replace after review shows only Cancel with the reason', async ({ page, request }) => {
|
||||
@@ -369,13 +355,18 @@ test('Replace after review shows only Cancel with the reason', async ({ page, re
|
||||
await page.locator('#tag-existing').selectOption('replace_after_review');
|
||||
await reviewButton(page).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review tag output images');
|
||||
expect((await gridOf(page)).map(([term]) => term)).toEqual(['Model', 'Existing tags', 'Images', 'Already auto-tagged',
|
||||
'Images to tag', 'Time', 'Settings', 'Backup']);
|
||||
expect((await gridOf(page)).map(([term]) => term)).toEqual(['Model', 'Existing tags', 'Images linked', 'Prompts', 'Already tagged',
|
||||
'Prompts to tag', 'Settings', 'Backup']);
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Replacing tags needs a review of each image. '
|
||||
+ 'Use Preview tagging review, or choose Skip or Merge for a batch run.');
|
||||
await expect(dialog(page).locator('.dialog-actions button')).toHaveText(['Cancel']);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
// Under Skip the fixture's only prompt is already tagged: nothing to start either.
|
||||
await page.locator('#tag-existing').selectOption('skip');
|
||||
await reviewButton(page).click();
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Every prompt with an output image is already tagged.');
|
||||
await expect(dialog(page).locator('.dialog-actions button')).toHaveText(['Cancel']);
|
||||
expect(await serverJson(request, '/__test__/tagging/snapshot')).toEqual(before);
|
||||
expect((await serverJson(request, '/prompt_manager/jobs')).jobs.filter(item => item.kind === 'autotag')).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
import { readCoverageTable, parseControl, CONTROL_EXEMPTIONS } from '../../tools/vendor/legacy-coverage.mjs';
|
||||
|
||||
// System › Overview (#249 System S2 Task 6) on the concept-shaped real fixture: the metrics, rows, Statistics dialog and
|
||||
// Health report read the server, never sample values (D93, D95, D96). The hooks are workspace_system_fixture.py's.
|
||||
@@ -413,16 +412,15 @@ test('out-of-order overview responses never repaint a newer section', async ({ p
|
||||
await nav.getByRole('button', { name: 'Appearance', exact: true }).click();
|
||||
await nav.getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect.poll(held).toBe(2);
|
||||
// Diagnostics & logs is native (S12): its body is its two surfaces.
|
||||
// Diagnostics still hands off (S12), so its body is one known panel.
|
||||
await nav.getByRole('button', { name: 'Diagnostics & logs', exact: true }).click();
|
||||
await expect(layout.locator(':scope > section.surface')).toHaveCount(2);
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await expect.poll(held).toBe(0);
|
||||
// Both late answers reached a section that had left: Diagnostics keeps its own body.
|
||||
await expect(layout.locator(':scope > section.handoff')).toBeVisible();
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
await expect(layout.locator('.summary-metrics')).toHaveCount(0);
|
||||
expect(await layout.evaluate(node => [...node.children].map(child => `${child.localName}.${child.className}`)))
|
||||
.toEqual(['section.surface', 'section.surface', 'div.save-row']);
|
||||
expect(await layout.evaluate(node => [...node.children].map(child => child.className))).toEqual(['surface handoff', 'save-row']);
|
||||
// Back on Overview, one fresh read paints one set of metrics.
|
||||
await nav.getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect(layout.locator('.summary-metrics')).toHaveCount(1);
|
||||
@@ -795,45 +793,3 @@ test('reset restores defaults without touching other keys', async ({ page }) =>
|
||||
expect(stored).toEqual({ gridColumns: 5, defaultLimit: 200, thumbnailPromptDismissed: 'kept', workspaceLayout: 'grid',
|
||||
videoMute: true, showImageInfo: true });
|
||||
});
|
||||
|
||||
// TESTING.md "Legacy System capability coverage" (#249 System S12, #329): each `covered` row's native control is on its
|
||||
// System section, found by role and accessible name (a `term` by its text, a disclosure by its summary). Collapsed
|
||||
// `details` are opened first, so a control inside one counts once it is shown. tests/vendor/legacy-coverage.test.mjs
|
||||
// checks the rest of each row.
|
||||
test('every covered legacy System capability has its native control', async ({ page, request }) => {
|
||||
// Download is offered for a model that is not on disk ("Download again" otherwise): the selected default's file is
|
||||
// missing here.
|
||||
await expect(await request.post('/__test__/tagging/models', { data: { unavailable: ['wd14-swinv2'] } })).toBeOK();
|
||||
try {
|
||||
await findControls(page);
|
||||
} finally {
|
||||
await request.post('/__test__/tagging/reset', { data: {} });
|
||||
}
|
||||
});
|
||||
|
||||
async function findControls(page) {
|
||||
const covered = readCoverageTable().filter(row => row.Status === 'covered')
|
||||
.map(row => ({ capability: row.Capability, ...parseControl(row['Native control']) }));
|
||||
// A covered row without a control is one of the recorded exemptions, never silently skipped.
|
||||
expect(covered.filter(row => !row.section).map(row => row.capability)).toEqual(Object.keys(CONTROL_EXEMPTIONS));
|
||||
const controls = covered.filter(row => row.section);
|
||||
expect(controls.length).toBeGreaterThan(40);
|
||||
const sections = [...new Set(controls.map(row => row.section))];
|
||||
for (const section of sections) {
|
||||
await page.goto(`${workspace}#/system/${section}`);
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(layoutOf(page).locator(':scope > *').first()).toBeVisible();
|
||||
for (const details of await main.locator('details:not([open]) > summary').all()) await details.click();
|
||||
for (const { capability, role, name, row } of controls.filter(item => item.section === section)) {
|
||||
// A name several rows share is looked up inside the row the table names.
|
||||
const within = row ? main.locator('.operation').filter({ has: page.locator(`h3:text-is("${row}")`) }) : main;
|
||||
const exact = new RegExp(`^${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`);
|
||||
// A `dt` (term) and a `details` disclosure (group) have no name from content: the term's text, the summary's.
|
||||
const control = role === 'term' ? within.getByRole('term').filter({ hasText: exact })
|
||||
: role === 'group' ? within.locator('details').filter({ has: page.locator(':scope > summary', { hasText: exact }) })
|
||||
.or(within.getByRole('group', { name, exact: true }))
|
||||
: within.getByRole(role, { name, exact: true });
|
||||
await expect(control.first(), `${capability}: ${section} · ${role} · "${name}"${row ? ` in "${row}"` : ''}`).toBeVisible();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -513,41 +513,6 @@ test('search finds an image by its image tag', async ({ page, request }) => {
|
||||
await expect(card(page, UNLINKED)).toBeVisible();
|
||||
});
|
||||
|
||||
// #379 CUJ 8: the Tags filter finds images by their own image tags through the tag index; no prompt carries them, and
|
||||
// selecting a match shows its prompt. A linked prompt's tag still combines with the image tags.
|
||||
test('the Tags filter finds images by image tags only, and selecting one shows its prompt (#379)', async ({ page, request }) => {
|
||||
await saveImageTags(request, FIRST, ['beach', '1girl', 'waves']);
|
||||
await saveImageTags(request, SECOND, ['beach', 'waves']);
|
||||
await saveImageTags(request, UNLINKED, ['beach', '1girl', 'waves', 'gulls']);
|
||||
await page.goto(images);
|
||||
await expect(page.locator('.gallery-body article[data-image-key]')).toHaveCount(manifest.files.length);
|
||||
const filters = page.locator('#workspace-main search.toolbar').getByRole('button', { name: /^Filters(?: · \d+)?$/ });
|
||||
const refine = page.getByRole('dialog', { name: 'Refine this view', exact: true });
|
||||
const applyTags = async value => {
|
||||
await filters.click();
|
||||
await refine.getByLabel('Tags', { exact: true }).fill(value);
|
||||
const listing = page.waitForRequest(item => item.url().includes('/images/output?')
|
||||
&& new URL(item.url()).searchParams.get('tags') !== null);
|
||||
await refine.getByRole('button', { name: 'Apply filters', exact: true }).click();
|
||||
await listing;
|
||||
};
|
||||
await applyTags('beach, 1girl, waves');
|
||||
// Every listed image whose image tags hold all three; SECOND lacks 1girl.
|
||||
await expect(page.locator('.gallery-body article[data-image-key]')).toHaveCount(2);
|
||||
await expect(card(page, FIRST)).toBeVisible();
|
||||
await expect(card(page, UNLINKED)).toBeVisible();
|
||||
await expect(card(page, SECOND)).toHaveCount(0);
|
||||
// The newest match is selected; the inspector shows its linked prompt.
|
||||
await infoButton(page, FIRST).click();
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
await expect(panel(page).locator('.panel-heading h2')).toHaveText(manifest.title);
|
||||
// The linked positive's own tag combines: SECOND has beach and waves itself and minimal through its prompt.
|
||||
await applyTags(`beach, waves, ${manifest.prompt_tags[0]}`);
|
||||
await expect(page.locator('.gallery-body article[data-image-key]')).toHaveCount(2);
|
||||
await expect(card(page, FIRST)).toBeVisible();
|
||||
await expect(card(page, SECOND)).toBeVisible();
|
||||
});
|
||||
|
||||
// Task C4 rulings: a save re-reads the selected file's details without a loading flash, and under a Tags filter or a
|
||||
// search the loaded listing is re-read in place (scroll and selection kept while the file still matches).
|
||||
const cards = page => page.locator('.gallery-body article[data-image-key]');
|
||||
|
||||
@@ -18,7 +18,7 @@ const routes = [
|
||||
['tags', 'Tags & categories', null],
|
||||
// #249 local Inspect: the native Inspect a file page; Overview's tool card still links the legacy metadata page.
|
||||
['inspect', 'Inspect a file', null],
|
||||
// #249 System S2: native System; every section native since S12 (D94 retired, asserted below).
|
||||
// #249 System S2: native System; its undelivered sections hand off per section (D94, asserted below).
|
||||
['system', 'System', null],
|
||||
// #249 D51: SAVED VIEWS' Favorites, Images filtered to favorites.
|
||||
['favorites', 'Favorites', null],
|
||||
@@ -395,7 +395,7 @@ const SURFACE = {
|
||||
intro: { fontSize: '12px', marginTop: '7px' },
|
||||
};
|
||||
|
||||
for (const width of [1600, 390]) test(`overview renders with concept surfaces and no System section hands off ${width}`, async ({ page }) => {
|
||||
for (const width of [1600, 390]) test(`overview and handoff render with concept surfaces ${width}`, async ({ page }) => {
|
||||
await page.setViewportSize({ width, height: 900 });
|
||||
await page.goto(`${workspace}#/overview`);
|
||||
const main = page.locator('#workspace-main');
|
||||
@@ -412,23 +412,23 @@ for (const width of [1600, 390]) test(`overview renders with concept surfaces an
|
||||
expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true);
|
||||
|
||||
await sidebarLink(page, 'System').click();
|
||||
// Every System section is native (S12 retired D94): the last one, Diagnostics & logs, shows concept surfaces.
|
||||
// System's Overview is native (S2 Task 6); a section not delivered yet still hands off (D94).
|
||||
await main.locator('.section-nav').getByRole('button', { name: 'Diagnostics & logs', exact: true }).click();
|
||||
const logs = main.locator('.settings-layout > section.surface').last();
|
||||
await expect(logs.locator(':scope > h2')).toHaveText('Logs');
|
||||
expect(await logs.evaluate(surfaceStyle)).toMatchObject(SURFACE);
|
||||
await expect(main.locator('.handoff')).toHaveCount(0);
|
||||
const handoff = main.locator('section.handoff');
|
||||
await expect(handoff).toHaveClass(/\bsurface\b/);
|
||||
expect(await handoff.evaluate(surfaceStyle)).toMatchObject(SURFACE);
|
||||
await expect(handoff.locator('a[href="/prompt_manager/admin"]')).toBeVisible();
|
||||
await expect(main.locator('.panel')).toHaveCount(0);
|
||||
expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true);
|
||||
});
|
||||
|
||||
// #249 System S2 Task 5: the concept's renderSystem shell (app.js 698–706) at `#/system/<section>`, every section native
|
||||
// (S12 retired the D94 hand-off), and the Save row's truthful status (D92).
|
||||
// #249 System S2 Task 5: the concept's renderSystem shell (app.js 698–706) at `#/system/<section>`, every section a D94
|
||||
// hand-off until its PR, and the Save row's truthful status (D92).
|
||||
const SYSTEM_SECTION_LABELS = ['Overview', 'Appearance', 'Sources & scans', 'Integrations', 'Auto-tagging', 'Maintenance',
|
||||
'Backup & export', 'Diagnostics & logs'];
|
||||
const SYSTEM_HASHES = ['overview', 'appearance', 'sources', 'integrations', 'tagging', 'maintenance', 'backups', 'diagnostics'];
|
||||
|
||||
test('System is native: the concept shell, section navigation, sections and the Save row', async ({ page }) => {
|
||||
test('System is native: the concept shell, section navigation, sections, hand-offs and the Save row', async ({ page }) => {
|
||||
const mutations = [];
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) mutations.push(item.url()); });
|
||||
await page.goto(`${workspace}#/system`);
|
||||
@@ -466,8 +466,7 @@ test('System is native: the concept shell, section navigation, sections and the
|
||||
// (S4 Task 11, workspace-system-sources.spec.mjs, its update notice shown in a fresh browser), Integrations (S10
|
||||
// Task 27, workspace-system-integrations.spec.mjs), Maintenance (S7 Task 19,
|
||||
// workspace-system-maintenance.spec.mjs), Backup & export (S9 Task 24, workspace-system-backups.spec.mjs) and
|
||||
// Auto-tagging (S11 Task 6, workspace-system-tagging.spec.mjs) and Diagnostics & logs (S12,
|
||||
// workspace-system-diagnostics.spec.mjs): none hands off.
|
||||
// Auto-tagging (S11 Task 6, workspace-system-tagging.spec.mjs); every other section hands off.
|
||||
if (hash === 'tagging') await expect(layout.locator('#tag-model')).toBeVisible();
|
||||
if (hash === 'overview') await expect(layout.locator(':scope > .summary-metrics')).toBeVisible();
|
||||
if (hash === 'sources') await expect(layout.locator('#source-list .operation')).toHaveCount(2);
|
||||
@@ -477,9 +476,13 @@ test('System is native: the concept shell, section navigation, sections and the
|
||||
integrations: ['section.surface', 'section.surface', 'div.save-row'],
|
||||
maintenance: ['div.notice', 'section.surface', 'section.surface', 'div.save-row'],
|
||||
backups: ['div.two-col', 'section.surface', 'div.save-row'],
|
||||
tagging: ['section.surface', 'div.save-row'],
|
||||
diagnostics: ['section.surface', 'section.surface', 'div.save-row'] }[hash]);
|
||||
await expect(layout.locator('.handoff')).toHaveCount(0);
|
||||
tagging: ['section.surface', 'div.save-row'] }[hash]
|
||||
?? ['section.surface handoff', 'div.save-row']);
|
||||
if (['overview', 'appearance', 'sources', 'integrations', 'tagging', 'maintenance', 'backups'].includes(hash)) continue;
|
||||
const handoff = layout.locator('section.handoff');
|
||||
await expect(handoff.locator('h2')).toHaveText('Your current tools are ready');
|
||||
expect(await handoff.locator('a.workspace-link').evaluateAll(links => links.map(link => link.getAttribute('href'))))
|
||||
.toEqual(['/prompt_manager/admin']);
|
||||
await expect(layout.locator('#settings-status')).toHaveText('No unsaved changes.');
|
||||
await expect(page.locator('aside.sidebar a[href="#/system"]')).toHaveAttribute('aria-current', 'page');
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -33,7 +34,6 @@ from utils.execution_contexts import (
|
||||
from utils.execution_history_events import make_finish, make_start
|
||||
from utils.execution_outputs import OutputBundle, OutputReference, snapshot_output_roots
|
||||
from utils.pair_execution import EXECUTION_KEY
|
||||
from tests.browser.folder_trash import replace_folder
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
@@ -300,7 +300,8 @@ def _clear(api: PromptManagerAPI, root: Path) -> None:
|
||||
for table in _TABLES:
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for name in ("output", "input"):
|
||||
replace_folder(root / name)
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
|
||||
|
||||
def _record_run(
|
||||
|
||||
@@ -1,267 +0,0 @@
|
||||
"""System › Diagnostics & logs test hooks for the browser tests (#249 System S12).
|
||||
|
||||
The logger writes into ``<server root>/logs`` (``repoint_logs``, called by
|
||||
``create_app`` before any record is written), so no test reads or deletes a
|
||||
real log folder. Every route refuses a log folder outside the server root.
|
||||
|
||||
- ``POST /__test__/logs/reset`` restores the default logging configuration,
|
||||
empties the active log, removes every other entry of the log folder (a
|
||||
link is removed, never followed) and clears the in-memory buffer. The
|
||||
global ``/__test__/reset`` calls it too;
|
||||
- ``POST /__test__/logs/seed`` ``{records, filler, rotated, active, link,
|
||||
other}`` writes buffer records with fixed times (``records``: the concept's
|
||||
four lines, then one DEBUG, ERROR and CRITICAL record, older; ``filler``
|
||||
more INFO records older still), ``active`` bytes into the active log,
|
||||
rotated copies ``prompt_manager.log.<n>`` of the listed sizes (fixed
|
||||
times), on POSIX a symlink ``prompt_manager.log.<n+1>`` to a file outside
|
||||
the folder (``link``) and a user's ``prompt_manager.log.bak`` (``other``).
|
||||
``{mode: "concept"}`` reproduces the concept's Logs sample (app.js 695)
|
||||
for the comparison: the buffer holds only its four lines, the active log
|
||||
is 1.4 MB with three rotated copies ("prompt_manager.log · 1.4 MB · rotated
|
||||
copies: 3") and console logging is off, as the concept's toggle;
|
||||
- ``POST /__test__/logs/rotate`` rolls the copies over as the handler does
|
||||
(``.2`` → ``.3``, ``.1`` → ``.2``, a new ``.1``), so a reviewed plan no
|
||||
longer matches;
|
||||
- ``GET /__test__/logs/state`` answers the folder's entries (name, size, link)
|
||||
and the buffer's length.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
ACTIVE = "prompt_manager.log"
|
||||
SEED_DAY = datetime.datetime(2026, 10, 26)
|
||||
# The concept's four lines (app.js 695), oldest first: the viewer shows them
|
||||
# newest first.
|
||||
CONCEPT_RECORDS = (
|
||||
("09:38:52", logging.INFO, "Source monitor started: output"),
|
||||
("09:39:04", logging.INFO, "Database backup verified"),
|
||||
("09:42:18", logging.WARNING, "3 images have ambiguous prompt metadata"),
|
||||
("09:42:18", logging.INFO, "Scan completed: 12 enriched, 0 duplicate links"),
|
||||
)
|
||||
# Older than the concept's lines, so they come after them: the levels the
|
||||
# concept does not show.
|
||||
LEVEL_RECORDS = (
|
||||
("09:30:01", logging.DEBUG, "Thumbnail cache lookup"),
|
||||
("09:30:02", logging.ERROR, "A source folder could not be read"),
|
||||
("09:30:03", logging.CRITICAL, "The database could not be opened"),
|
||||
)
|
||||
ROTATED_TIME = datetime.datetime(2026, 10, 25, 14, 2)
|
||||
# The concept's "prompt_manager.log · 1.4 MB · rotated copies: 3": the active
|
||||
# log stays under 1.45 MB while the server writes its own lines.
|
||||
CONCEPT_ACTIVE = int(1.4 * 1024 * 1024)
|
||||
CONCEPT_ROTATED = (3072, 2048, 1024)
|
||||
MAX_SEED_BYTES = 64 * 1024 * 1024
|
||||
DEFAULT_CONFIG = {
|
||||
"level": "INFO",
|
||||
"max_file_size": 10 * 1024 * 1024,
|
||||
"backup_count": 5,
|
||||
"console_logging": True,
|
||||
"file_logging": True,
|
||||
"buffer_size": 1000,
|
||||
}
|
||||
|
||||
|
||||
def _manager():
|
||||
from utils.logging_config import get_logger_manager
|
||||
|
||||
return get_logger_manager()
|
||||
|
||||
|
||||
def repoint_logs(root: Path) -> Path:
|
||||
"""Point the logger at ``<root>/logs`` before anything is logged.
|
||||
|
||||
The logger is built while ``logs_dir`` answers that folder, so its first
|
||||
handler never opens a log file anywhere else (a node folder with a legacy
|
||||
database would otherwise name a real log folder).
|
||||
"""
|
||||
from utils import library_location
|
||||
|
||||
folder = Path(root) / "logs"
|
||||
folder.mkdir(exist_ok=True)
|
||||
original = library_location.logs_dir
|
||||
library_location.logs_dir = lambda **_kwargs: str(folder)
|
||||
try:
|
||||
manager = _manager()
|
||||
finally:
|
||||
library_location.logs_dir = original
|
||||
with manager._files_lock:
|
||||
manager.log_dir = folder
|
||||
manager._setup_loggers()
|
||||
return folder
|
||||
|
||||
|
||||
def _require_inside(root: Path, folder: Path) -> None:
|
||||
resolved = Path(os.path.realpath(folder))
|
||||
if Path(os.path.realpath(root)) not in resolved.parents:
|
||||
raise web.HTTPForbidden(text="The log folder is outside the test server root")
|
||||
|
||||
|
||||
def _record(time_text, level, message, index):
|
||||
record = logging.LogRecord(
|
||||
"prompt_manager.test", level, __file__, index, message, None, None
|
||||
)
|
||||
hours, minutes, seconds = (int(part) for part in time_text.split(":"))
|
||||
moment = SEED_DAY.replace(hour=hours, minute=minutes, second=seconds)
|
||||
# Equal times keep their order: each later record is a microsecond newer.
|
||||
record.created = moment.timestamp() + index / 1e6
|
||||
return record
|
||||
|
||||
|
||||
def _touch(path: Path, size: int, moment: datetime.datetime) -> None:
|
||||
# A link at this name is replaced, never written through.
|
||||
if path.is_symlink():
|
||||
path.unlink()
|
||||
with open(path, "wb") as handle:
|
||||
handle.write(b"x" * size)
|
||||
stamp = moment.timestamp()
|
||||
os.utime(path, (stamp, stamp))
|
||||
|
||||
|
||||
def _bounded(value, name, top=MAX_SEED_BYTES):
|
||||
if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= top:
|
||||
raise web.HTTPBadRequest(text=f"{name} must be an integer 0..{top}")
|
||||
return value
|
||||
|
||||
|
||||
def register_diagnostics_commands(app, routes, api, root: Path):
|
||||
"""Register the hooks; returns ``reset()`` for the global reset."""
|
||||
|
||||
def folder() -> Path:
|
||||
path = Path(_manager().log_dir)
|
||||
_require_inside(root, path)
|
||||
return path
|
||||
|
||||
def reset() -> None:
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
if manager.get_config() != DEFAULT_CONFIG:
|
||||
manager.update_config(dict(DEFAULT_CONFIG))
|
||||
with manager._files_lock:
|
||||
held = manager._acquire_file_handlers()
|
||||
try:
|
||||
with os.scandir(log_dir) as entries:
|
||||
found = list(entries)
|
||||
for entry in found:
|
||||
path = Path(entry.path)
|
||||
if entry.name == ACTIVE and entry.is_file(follow_symlinks=False):
|
||||
manager._truncate_main_log(path)
|
||||
elif entry.is_dir(follow_symlinks=False):
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
os.unlink(path)
|
||||
finally:
|
||||
manager._release_handlers(held)
|
||||
with manager._buffer_lock:
|
||||
manager._log_buffer.clear()
|
||||
|
||||
def seed_concept() -> None:
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
manager.update_config({"console_logging": False})
|
||||
with manager._buffer_lock:
|
||||
manager._log_buffer.clear()
|
||||
for index, (time_text, level, message) in enumerate(CONCEPT_RECORDS):
|
||||
record = _record(time_text, level, message, index)
|
||||
manager.add_to_buffer(record, record.getMessage())
|
||||
with open(log_dir / ACTIVE, "ab") as handle:
|
||||
handle.write(b"x" * CONCEPT_ACTIVE)
|
||||
for number, size in enumerate(CONCEPT_ROTATED, start=1):
|
||||
moment = ROTATED_TIME - datetime.timedelta(hours=number)
|
||||
_touch(log_dir / f"{ACTIVE}.{number}", size, moment)
|
||||
|
||||
def seed(data) -> None:
|
||||
if data.get("mode") == "concept":
|
||||
seed_concept()
|
||||
return
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
with_records = data.get("records", True) is not False
|
||||
filler = _bounded(data.get("filler", 0), "filler", 900)
|
||||
older = [
|
||||
(f"08:{index // 60:02}:{index % 60:02}", logging.INFO, f"Filler {index}")
|
||||
for index in range(filler)
|
||||
]
|
||||
named = list(LEVEL_RECORDS) + list(CONCEPT_RECORDS) if with_records else []
|
||||
ordered = older + named
|
||||
for index, (time_text, level, message) in enumerate(ordered):
|
||||
record = _record(time_text, level, message, index)
|
||||
manager.add_to_buffer(record, record.getMessage())
|
||||
active = _bounded(data.get("active", 0), "active")
|
||||
if active:
|
||||
with open(log_dir / ACTIVE, "ab") as handle:
|
||||
handle.write(b"x" * active)
|
||||
rotated = data.get("rotated", [])
|
||||
if not isinstance(rotated, list) or len(rotated) > 20:
|
||||
raise web.HTTPBadRequest(text="rotated must be a list of sizes")
|
||||
for number, size in enumerate(rotated, start=1):
|
||||
moment = ROTATED_TIME - datetime.timedelta(hours=number)
|
||||
_touch(log_dir / f"{ACTIVE}.{number}", _bounded(size, "size"), moment)
|
||||
if data.get("link") and os.name != "nt":
|
||||
target = Path(root) / "outside-log-target.txt"
|
||||
target.write_bytes(b"not a log of this folder")
|
||||
os.symlink(target, log_dir / f"{ACTIVE}.{len(rotated) + 1}")
|
||||
if data.get("other"):
|
||||
_touch(log_dir / f"{ACTIVE}.bak", 0, ROTATED_TIME)
|
||||
|
||||
def rotate() -> None:
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
with manager._files_lock:
|
||||
held = manager._acquire_file_handlers()
|
||||
try:
|
||||
for number in (2, 1):
|
||||
source = log_dir / f"{ACTIVE}.{number}"
|
||||
if source.is_file() and not source.is_symlink():
|
||||
os.replace(source, log_dir / f"{ACTIVE}.{number + 1}")
|
||||
_touch(log_dir / f"{ACTIVE}.1", 4096, datetime.datetime.now())
|
||||
finally:
|
||||
manager._release_handlers(held)
|
||||
|
||||
def state():
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
entries = []
|
||||
with os.scandir(log_dir) as listing:
|
||||
found = sorted(listing, key=lambda item: item.name)
|
||||
for entry in found:
|
||||
info = entry.stat(follow_symlinks=False)
|
||||
entries.append(
|
||||
{"name": entry.name, "size": info.st_size, "link": entry.is_symlink()}
|
||||
)
|
||||
with manager._buffer_lock:
|
||||
buffered = len(manager._log_buffer)
|
||||
return {"entries": entries, "buffer": buffered, "config": manager.get_config()}
|
||||
|
||||
async def body(request):
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
if not isinstance(data, dict):
|
||||
raise web.HTTPBadRequest()
|
||||
return data
|
||||
|
||||
@routes.post("/__test__/logs/reset")
|
||||
async def logs_reset(request: web.Request) -> web.Response:
|
||||
await api._run_in_executor(reset)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/logs/seed")
|
||||
async def logs_seed(request: web.Request) -> web.Response:
|
||||
data = await body(request)
|
||||
await api._run_in_executor(seed, data)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/logs/rotate")
|
||||
async def logs_rotate(request: web.Request) -> web.Response:
|
||||
await api._run_in_executor(rotate)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.get("/__test__/logs/state")
|
||||
async def logs_state(request: web.Request) -> web.Response:
|
||||
return web.json_response(await api._run_in_executor(state))
|
||||
|
||||
return reset
|
||||
@@ -3,11 +3,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from aiohttp import web
|
||||
from PIL import Image
|
||||
from tests.browser.folder_trash import replace_folder
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
@@ -51,7 +51,8 @@ def seed_library_editor(api: PromptManagerAPI, root: Path) -> dict:
|
||||
for table in _TABLES:
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for name in ("output", "input"):
|
||||
replace_folder(root / name)
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
epoch = db.get_execution_epoch()
|
||||
receipt = db.create_library_recipe(
|
||||
EDITOR_POSITIVE,
|
||||
|
||||
@@ -52,7 +52,6 @@ from typing import TYPE_CHECKING
|
||||
from aiohttp import web
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
from tests.browser.folder_trash import replace_folder
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
@@ -226,7 +225,8 @@ def seed_review(
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
api.close_annotation_captures()
|
||||
for name in ("output", "input"):
|
||||
replace_folder(root / name)
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
shutil.rmtree(root / "temp" / "review", ignore_errors=True)
|
||||
output = root / "output"
|
||||
files = []
|
||||
|
||||
@@ -19,7 +19,6 @@ from typing import TYPE_CHECKING
|
||||
from aiohttp import web
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
from tests.browser.folder_trash import replace_folder
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
@@ -99,7 +98,8 @@ def seed_tagging(api: PromptManagerAPI, root: Path, extra_images: int = 0) -> di
|
||||
for table in _TABLES:
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for name in ("output", "input"):
|
||||
replace_folder(root / name)
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
output = root / "output"
|
||||
_png(output / FILES[0][0], (196, 140, 92))
|
||||
_png(output / FILES[1][0], (150, 110, 80))
|
||||
|
||||
@@ -5,13 +5,13 @@ from __future__ import annotations
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from tests.browser.folder_trash import replace_folder
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
@@ -129,7 +129,8 @@ def seed_workspace_visual(api: PromptManagerAPI, root: Path) -> None:
|
||||
):
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for name in ("output", "input"):
|
||||
replace_folder(root / name)
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
for parent_index, (positive, negatives) in enumerate(zip(_POSITIVES, _NEGATIVES)):
|
||||
category = ("Architecture", "Landscape", "Abstract")[parent_index % 3]
|
||||
tags = ["geometric", ("warm", "cool")[parent_index % 2]]
|
||||
|
||||
@@ -7,10 +7,10 @@ in concept order, so tag creation order and category first use match the mock.
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from aiohttp import web
|
||||
from tests.browser.folder_trash import replace_folder
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
@@ -120,7 +120,8 @@ def seed_vocabulary(
|
||||
for table in _TABLES:
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for name in ("output", "input"):
|
||||
replace_folder(root / name)
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
epoch = db.get_execution_epoch()
|
||||
recipes = []
|
||||
for recipe in CONCEPT_RECIPES:
|
||||
|
||||
Vendored
-16
@@ -113,19 +113,3 @@ by their own suites. The expectations start as the observed baseline. A change
|
||||
that alters an outcome edits the vector in the same commit.
|
||||
`tests/js/legacy-metadata-characterization.test.js` pins what
|
||||
the legacy viewers show on the same files.
|
||||
|
||||
# Legacy contract snapshot
|
||||
|
||||
`legacy_contract_snapshot.json` records what the legacy admin and gallery pages
|
||||
talk to while System S13 (#330) moves their tools to native System: every
|
||||
registered route as `"<METHOD> <path>"` (`routes`) and the key paths of
|
||||
`GET /prompt_manager/settings` to depth 2 (`settings_keys`; list items give the
|
||||
union of their keys as `name[].key`). `tests/test_legacy_contract_snapshot.py`
|
||||
compares a temp-database API against it. Record it again only for an intended
|
||||
change, and say why in the PR body:
|
||||
|
||||
```sh
|
||||
PM_RECORD_LEGACY_SNAPSHOT=1 python3 -m unittest tests.test_legacy_contract_snapshot
|
||||
```
|
||||
|
||||
Cutover (#334) removes the test and this fixture.
|
||||
|
||||
-200
@@ -1,200 +0,0 @@
|
||||
{
|
||||
"routes": [
|
||||
"DELETE /prompt_manager/delete/{prompt_id}",
|
||||
"DELETE /prompt_manager/image-annotations/capture/{token}",
|
||||
"DELETE /prompt_manager/images/favorite",
|
||||
"DELETE /prompt_manager/images/{image_id}",
|
||||
"DELETE /prompt_manager/pairs/{pair_id}",
|
||||
"DELETE /prompt_manager/prompts/{prompt_id}/tags",
|
||||
"DELETE /prompt_manager/restore/preview/{token}",
|
||||
"DELETE /prompt_manager/saved-views/{view_id}",
|
||||
"DELETE /prompt_manager/tags/{tag_name}",
|
||||
"GET /prompt_manager/admin",
|
||||
"GET /prompt_manager/autotag/models",
|
||||
"GET /prompt_manager/backup",
|
||||
"GET /prompt_manager/backup/status",
|
||||
"GET /prompt_manager/categories",
|
||||
"GET /prompt_manager/diagnostics",
|
||||
"GET /prompt_manager/export",
|
||||
"GET /prompt_manager/gallery",
|
||||
"GET /prompt_manager/gallery.html",
|
||||
"GET /prompt_manager/gallery/subfolders",
|
||||
"GET /prompt_manager/history/runs/{run_id}",
|
||||
"GET /prompt_manager/history/runs/{run_id}/output/current/inspection",
|
||||
"GET /prompt_manager/history/runs/{run_id}/output/current/media",
|
||||
"GET /prompt_manager/history/runs/{run_id}/output/current/thumbnail",
|
||||
"GET /prompt_manager/history/runs/{run_id}/outputs",
|
||||
"GET /prompt_manager/image-annotations/capture/{token}",
|
||||
"GET /prompt_manager/images/all",
|
||||
"GET /prompt_manager/images/inspect/{image_path}",
|
||||
"GET /prompt_manager/images/output",
|
||||
"GET /prompt_manager/images/prompt/{image_path}",
|
||||
"GET /prompt_manager/images/recent",
|
||||
"GET /prompt_manager/images/search",
|
||||
"GET /prompt_manager/images/serve/{filepath}",
|
||||
"GET /prompt_manager/images/thumbnails/plan",
|
||||
"GET /prompt_manager/images/thumbnails/status",
|
||||
"GET /prompt_manager/images/{image_id}/file",
|
||||
"GET /prompt_manager/jobs",
|
||||
"GET /prompt_manager/jobs/{job_id}",
|
||||
"GET /prompt_manager/jobs/{job_id}/events",
|
||||
"GET /prompt_manager/jobs/{job_id}/report",
|
||||
"GET /prompt_manager/js/{filepath}",
|
||||
"GET /prompt_manager/lib/{filepath}",
|
||||
"GET /prompt_manager/library",
|
||||
"GET /prompt_manager/library/export",
|
||||
"GET /prompt_manager/library/options",
|
||||
"GET /prompt_manager/library/{positive_id}",
|
||||
"GET /prompt_manager/library/{positive_id}/history",
|
||||
"GET /prompt_manager/library/{positive_id}/history/pairs/{pair_id}/newest",
|
||||
"GET /prompt_manager/library/{positive_id}/images",
|
||||
"GET /prompt_manager/logs",
|
||||
"GET /prompt_manager/logs/config",
|
||||
"GET /prompt_manager/logs/download/{filename}",
|
||||
"GET /prompt_manager/logs/files",
|
||||
"GET /prompt_manager/logs/stats",
|
||||
"GET /prompt_manager/lora/detect",
|
||||
"GET /prompt_manager/lora/status",
|
||||
"GET /prompt_manager/lora/trigger-words",
|
||||
"GET /prompt_manager/maintenance/duplicates/{job_id}",
|
||||
"GET /prompt_manager/pairs",
|
||||
"GET /prompt_manager/pairs/export",
|
||||
"GET /prompt_manager/pairs/{pair_id}",
|
||||
"GET /prompt_manager/prompts/{prompt_id}/images",
|
||||
"GET /prompt_manager/recent",
|
||||
"GET /prompt_manager/review/events",
|
||||
"GET /prompt_manager/review/items",
|
||||
"GET /prompt_manager/review/items/{item_id}",
|
||||
"GET /prompt_manager/saved-views",
|
||||
"GET /prompt_manager/scan/status",
|
||||
"GET /prompt_manager/scan_duplicates",
|
||||
"GET /prompt_manager/scan_duplicates/status",
|
||||
"GET /prompt_manager/scan_output_dir",
|
||||
"GET /prompt_manager/search",
|
||||
"GET /prompt_manager/settings",
|
||||
"GET /prompt_manager/stats",
|
||||
"GET /prompt_manager/subfolders",
|
||||
"GET /prompt_manager/system/diagnostics",
|
||||
"GET /prompt_manager/system/health",
|
||||
"GET /prompt_manager/system/library",
|
||||
"GET /prompt_manager/system/overview",
|
||||
"GET /prompt_manager/system/statistics",
|
||||
"GET /prompt_manager/tags",
|
||||
"GET /prompt_manager/tags/filter",
|
||||
"GET /prompt_manager/tags/stats",
|
||||
"GET /prompt_manager/tags/suggest",
|
||||
"GET /prompt_manager/tags/{tag_name}/prompts",
|
||||
"GET /prompt_manager/test",
|
||||
"GET /prompt_manager/vocabulary/categories",
|
||||
"GET /prompt_manager/vocabulary/tags",
|
||||
"GET /prompt_manager/vocabulary/tags/{tag_id}/impact",
|
||||
"GET /prompt_manager/web",
|
||||
"GET /prompt_manager/workspace",
|
||||
"PATCH /prompt_manager/library/{positive_id}",
|
||||
"PATCH /prompt_manager/pairs/{pair_id}",
|
||||
"PATCH /prompt_manager/saved-views/{view_id}",
|
||||
"PATCH /prompt_manager/vocabulary/tags/{tag_id}",
|
||||
"POST /prompt_manager/autotag/apply",
|
||||
"POST /prompt_manager/autotag/batch/preview",
|
||||
"POST /prompt_manager/autotag/download/{model_type}",
|
||||
"POST /prompt_manager/autotag/single",
|
||||
"POST /prompt_manager/autotag/start",
|
||||
"POST /prompt_manager/autotag/unload",
|
||||
"POST /prompt_manager/bulk/category",
|
||||
"POST /prompt_manager/bulk/delete",
|
||||
"POST /prompt_manager/bulk/tags",
|
||||
"POST /prompt_manager/cleanup",
|
||||
"POST /prompt_manager/delete_duplicate_images",
|
||||
"POST /prompt_manager/diagnostics/test-link",
|
||||
"POST /prompt_manager/gallery/rescan",
|
||||
"POST /prompt_manager/image-annotations/observe",
|
||||
"POST /prompt_manager/image-annotations/suggest",
|
||||
"POST /prompt_manager/images/clear-thumbnails",
|
||||
"POST /prompt_manager/images/generate-thumbnails",
|
||||
"POST /prompt_manager/images/generate-thumbnails/progress",
|
||||
"POST /prompt_manager/images/link",
|
||||
"POST /prompt_manager/images/reveal",
|
||||
"POST /prompt_manager/jobs/autotag",
|
||||
"POST /prompt_manager/jobs/duplicates",
|
||||
"POST /prompt_manager/jobs/image-tags",
|
||||
"POST /prompt_manager/jobs/model-download",
|
||||
"POST /prompt_manager/jobs/scan",
|
||||
"POST /prompt_manager/jobs/thumbnails",
|
||||
"POST /prompt_manager/jobs/{job_id}/cancel",
|
||||
"POST /prompt_manager/library",
|
||||
"POST /prompt_manager/library/{positive_id}/variants",
|
||||
"POST /prompt_manager/logs/config",
|
||||
"POST /prompt_manager/logs/truncate",
|
||||
"POST /prompt_manager/lora/enable",
|
||||
"POST /prompt_manager/lora/import/apply",
|
||||
"POST /prompt_manager/lora/import/preview",
|
||||
"POST /prompt_manager/lora/refresh-cache",
|
||||
"POST /prompt_manager/lora/scan",
|
||||
"POST /prompt_manager/maintenance",
|
||||
"POST /prompt_manager/maintenance/apply",
|
||||
"POST /prompt_manager/maintenance/preview",
|
||||
"POST /prompt_manager/pairs",
|
||||
"POST /prompt_manager/prompts/tags",
|
||||
"POST /prompt_manager/prompts/{prompt_id}/tags",
|
||||
"POST /prompt_manager/restore",
|
||||
"POST /prompt_manager/restore/apply",
|
||||
"POST /prompt_manager/restore/preview",
|
||||
"POST /prompt_manager/review/items/{item_id}/decision",
|
||||
"POST /prompt_manager/save",
|
||||
"POST /prompt_manager/saved-views",
|
||||
"POST /prompt_manager/scan",
|
||||
"POST /prompt_manager/scan/cancel",
|
||||
"POST /prompt_manager/scan_duplicates/stream",
|
||||
"POST /prompt_manager/settings",
|
||||
"POST /prompt_manager/settings/validate-source",
|
||||
"POST /prompt_manager/system/health",
|
||||
"POST /prompt_manager/system/library/keep",
|
||||
"POST /prompt_manager/system/library/move",
|
||||
"POST /prompt_manager/tags/merge",
|
||||
"POST /prompt_manager/vocabulary/categories/apply",
|
||||
"POST /prompt_manager/vocabulary/categories/preview",
|
||||
"POST /prompt_manager/vocabulary/tags",
|
||||
"POST /prompt_manager/vocabulary/tags/{tag_id}/delete",
|
||||
"PUT /prompt_manager/image-annotations/tags",
|
||||
"PUT /prompt_manager/images/favorite",
|
||||
"PUT /prompt_manager/prompts/{prompt_id}",
|
||||
"PUT /prompt_manager/prompts/{prompt_id}/rating",
|
||||
"PUT /prompt_manager/tags/{tag_name}"
|
||||
],
|
||||
"settings_keys": [
|
||||
"monitor",
|
||||
"monitor.observer_alive",
|
||||
"monitor.present",
|
||||
"monitor.scheduled",
|
||||
"revision",
|
||||
"settings",
|
||||
"settings.autotag_character_threshold",
|
||||
"settings.autotag_existing",
|
||||
"settings.autotag_general_threshold",
|
||||
"settings.autotag_model_type",
|
||||
"settings.autotag_suggestion_limit",
|
||||
"settings.autotag_template",
|
||||
"settings.cpu_count",
|
||||
"settings.gallery_root_path",
|
||||
"settings.gallery_root_paths",
|
||||
"settings.infinite_scroll",
|
||||
"settings.monitored_directories",
|
||||
"settings.monitoring_enabled",
|
||||
"settings.result_timeout",
|
||||
"settings.safety_backup",
|
||||
"settings.thumbnail_edge",
|
||||
"settings.thumbnail_format",
|
||||
"settings.thumbnail_quality",
|
||||
"settings.webui_display_mode",
|
||||
"settings.worker_threads",
|
||||
"sources",
|
||||
"sources.auto_detected",
|
||||
"sources.configured",
|
||||
"sources.effective",
|
||||
"success",
|
||||
"thumbnail_encoders",
|
||||
"thumbnail_encoders.jpeg",
|
||||
"thumbnail_encoders.jxl",
|
||||
"thumbnail_encoders.webp"
|
||||
]
|
||||
}
|
||||
@@ -36,48 +36,33 @@ function fixture() {
|
||||
const reply = (index, data) => requests[index].resolve({ json: async () => data });
|
||||
return { manager, requests, reply, notifications };
|
||||
}
|
||||
const SHA = 'a'.repeat(64);
|
||||
test('gallery review retains root identity and applies the tags to the image by its sha256 (#379)', async () => {
|
||||
test('gallery review retains root identity and applies tags with the returned prompt ID', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
assert.equal(requests[0].body.root_index, 1);
|
||||
assert.equal(requests[0].body.existing, 'merge');
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'], sha256: SHA, image_tags: [] });
|
||||
await load;
|
||||
manager.loadNextReviewImage = () => {};
|
||||
const apply = manager.applyReviewTags();
|
||||
assert.equal(requests[1].url, '/prompt_manager/autotag/apply');
|
||||
assert.deepEqual(requests[1].body, ImageHelpers.autotagApplyBody(
|
||||
{ id: 'digest', path: 'same.png', root_index: 1 }, SHA, ['smile']));
|
||||
assert.equal('prompt_id' in requests[1].body, false);
|
||||
reply(1, { success: true });
|
||||
await apply;
|
||||
});
|
||||
test('an unlinked image applies to the image like a linked one', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
reply(0, { success: true, prompt_id: null, tags: ['smile'], sha256: SHA, image_tags: [] });
|
||||
await load;
|
||||
manager.loadNextReviewImage = () => {};
|
||||
const apply = manager.applyReviewTags();
|
||||
assert.equal(requests[1].body.sha256, SHA);
|
||||
assert.deepEqual(requests[1].body.tags, ['smile']);
|
||||
reply(1, { success: true });
|
||||
await apply;
|
||||
});
|
||||
test('an answer without a captured sha256 never sends an apply', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'] });
|
||||
await load;
|
||||
manager.loadNextReviewImage = () => {};
|
||||
await manager.applyReviewTags();
|
||||
assert.equal(requests.filter(r => r.url === '/prompt_manager/autotag/apply').length, 0);
|
||||
const apply = manager.applyReviewTags();
|
||||
assert.deepEqual(requests[1].body, { prompt_id: 42, tags: ['smile'] });
|
||||
reply(1, { success: true });
|
||||
await apply;
|
||||
});
|
||||
test('unlinked images do not submit an invalid apply request or advance', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
reply(0, { success: true, prompt_id: null, tags: ['smile'] });
|
||||
await load;
|
||||
const apply = manager.applyReviewTags();
|
||||
if (requests[1]) reply(1, { success: false, error: "Unexpected apply" });
|
||||
await apply;
|
||||
assert.equal(requests.length, 1);
|
||||
assert.equal(manager.autoTagState.reviewIndex, 0);
|
||||
});
|
||||
test('a failed apply keeps the current image available for retry', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'], sha256: SHA, image_tags: [] });
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'] });
|
||||
await load;
|
||||
const apply = manager.applyReviewTags();
|
||||
reply(1, { success: false, error: 'Try again' });
|
||||
|
||||
@@ -5,8 +5,6 @@ const {
|
||||
formatImageCaption,
|
||||
workflowDocumentHtml,
|
||||
autotagSingleBody,
|
||||
autotagApplyBody,
|
||||
autotagCompleteMessage,
|
||||
fallbackImageSource,
|
||||
} = require("../../web/js/image-helpers.js");
|
||||
|
||||
@@ -123,27 +121,3 @@ test('metadata lookup preserves encoded filename characters and works behind a p
|
||||
assert.equal(metadataLookupUrl('/prompt_manager/images/serve/a.png?v=123'), '/prompt_manager/images/prompt/a.png');
|
||||
assert.equal(metadataLookupUrl('/elsewhere/a.png'), null);
|
||||
});
|
||||
|
||||
test("autotagSingleBody asks to skip auto-tagged images only when told (#379)", () => {
|
||||
assert.equal(autotagSingleBody({ id: 1 }, WD14).existing, "merge");
|
||||
assert.equal(autotagSingleBody({ id: 1 }, { ...WD14, existing: "skip" }).existing, "skip");
|
||||
assert.equal(autotagSingleBody({ id: 1 }, { ...WD14, existing: "odd" }).existing, "merge");
|
||||
});
|
||||
|
||||
test("autotagApplyBody names the image, its bytes and the tags, never a prompt (#379)", () => {
|
||||
const sha = "a".repeat(64);
|
||||
assert.deepEqual(autotagApplyBody({ id: 4, prompt_id: 7 }, sha, ["bird"]), { image_id: 4, sha256: sha, tags: ["bird"] });
|
||||
assert.deepEqual(autotagApplyBody({ id: "digest", path: "sub/a.png", root_index: 1 }, sha, []),
|
||||
{ path: "sub/a.png", root_index: 1, sha256: sha, tags: [] });
|
||||
assert.deepEqual(autotagApplyBody({ id: 2 }, undefined, null), { image_id: 2, sha256: "", tags: [] });
|
||||
const tags = ["x"];
|
||||
autotagApplyBody({ id: 1 }, sha, tags).tags.push("y");
|
||||
assert.deepEqual(tags, ["x"]);
|
||||
});
|
||||
|
||||
test("autotagCompleteMessage counts images (#379)", () => {
|
||||
assert.equal(autotagCompleteMessage(2), "Auto tagging complete! Tagged 2 images.");
|
||||
assert.equal(autotagCompleteMessage(1), "Auto tagging complete! Tagged 1 image.");
|
||||
assert.equal(autotagCompleteMessage(0), "Auto tagging complete! Tagged 0 images.");
|
||||
assert.equal(autotagCompleteMessage(undefined), "Auto tagging complete! Tagged 0 images.");
|
||||
});
|
||||
|
||||
@@ -97,30 +97,6 @@ test("connect reports a non-2xx response through onerror and never onmessage", a
|
||||
assert.equal(source.readyState, source.CLOSED);
|
||||
});
|
||||
|
||||
test("a non-2xx JSON answer's error sentence reaches onerror as serverMessage", async () => {
|
||||
let error = null;
|
||||
const fetchImpl = async () => new Response(
|
||||
JSON.stringify({ success: false, error: "Auto-tagging is already running" }),
|
||||
{ status: 409, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
const source = connect("/x", { fetchImpl });
|
||||
source.onerror = (err) => { error = err; };
|
||||
await until(() => error !== null);
|
||||
assert.match(error.message, /409/);
|
||||
assert.equal(error.status, 409);
|
||||
assert.equal(error.serverMessage, "Auto-tagging is already running");
|
||||
});
|
||||
|
||||
test("a non-2xx answer without a JSON error has no serverMessage", async () => {
|
||||
let error = null;
|
||||
const fetchImpl = async () => new Response("nope", { status: 500 });
|
||||
const source = connect("/x", { fetchImpl });
|
||||
source.onerror = (err) => { error = err; };
|
||||
await until(() => error !== null);
|
||||
assert.equal(error.status, 500);
|
||||
assert.equal(error.serverMessage, undefined);
|
||||
});
|
||||
|
||||
test("connect reports a network failure through onerror", async () => {
|
||||
let error = null;
|
||||
const source = connect("/x", { fetchImpl: async () => { throw new TypeError("Failed to fetch"); } });
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const source = import('../../web/js/workspace/images-multiselect.mjs');
|
||||
|
||||
const KEYS = Object.freeze(['0:a', '0:b', '0:c', '0:d', '1:e']);
|
||||
|
||||
test('a range runs from the anchor to the target in display order, either way', async () => {
|
||||
const { rangeKeys } = await source;
|
||||
assert.deepEqual(rangeKeys(KEYS, '0:b', '0:d'), ['0:b', '0:c', '0:d']);
|
||||
assert.deepEqual(rangeKeys(KEYS, '1:e', '0:c'), ['0:c', '0:d', '1:e']);
|
||||
assert.deepEqual(rangeKeys(KEYS, '0:c', '0:c'), ['0:c']);
|
||||
});
|
||||
|
||||
test('a range without a listed anchor is the target alone, and an unlisted target is empty', async () => {
|
||||
const { rangeKeys } = await source;
|
||||
assert.deepEqual(rangeKeys(KEYS, null, '0:c'), ['0:c']);
|
||||
assert.deepEqual(rangeKeys(KEYS, '9:gone', '0:c'), ['0:c']);
|
||||
assert.deepEqual(rangeKeys(KEYS, '0:a', '9:gone'), []);
|
||||
});
|
||||
|
||||
test('applying keys adds or removes them and keeps the same frozen set when nothing changes', async () => {
|
||||
const { applyKeys } = await source;
|
||||
const start = Object.freeze(new Set(['0:a']));
|
||||
const added = applyKeys(start, ['0:b', '0:c'], true);
|
||||
assert.deepEqual([...added], ['0:a', '0:b', '0:c']);
|
||||
assert.ok(Object.isFrozen(added));
|
||||
assert.deepEqual([...start], ['0:a']);
|
||||
const removed = applyKeys(added, ['0:a', '0:b'], false);
|
||||
assert.deepEqual([...removed], ['0:c']);
|
||||
assert.equal(applyKeys(removed, ['0:c'], true), removed);
|
||||
assert.equal(applyKeys(removed, ['0:a'], false), removed);
|
||||
});
|
||||
|
||||
test('every listed key is checked only when each one is in the set', async () => {
|
||||
const { everyChecked } = await source;
|
||||
assert.equal(everyChecked(new Set(KEYS), KEYS), true);
|
||||
assert.equal(everyChecked(new Set(KEYS.slice(1)), KEYS), false);
|
||||
assert.equal(everyChecked(new Set(['x', ...KEYS.slice(1)]), KEYS), false);
|
||||
assert.equal(everyChecked(new Set(), []), false);
|
||||
});
|
||||
|
||||
test('the select-all-matching offer needs every loaded image checked and more to load', async () => {
|
||||
const { offersAllMatching } = await source;
|
||||
const all = new Set(KEYS);
|
||||
assert.equal(offersAllMatching(all, KEYS, { total: 9, hasMore: true }), true);
|
||||
assert.equal(offersAllMatching(all, KEYS, { total: 5, hasMore: false }), false);
|
||||
assert.equal(offersAllMatching(new Set(KEYS.slice(1)), KEYS, { total: 9, hasMore: true }), false);
|
||||
assert.equal(offersAllMatching(new Set(), [], { total: 9, hasMore: true }), false);
|
||||
});
|
||||
|
||||
test('the counts read "N selected" and "All N matching selected", grouped', async () => {
|
||||
const { matchingLabel, matchingOffer } = await source;
|
||||
assert.equal(matchingLabel(12345), 'All 12,345 matching selected');
|
||||
assert.equal(matchingOffer(12345), 'Select all 12,345 matching');
|
||||
});
|
||||
|
||||
function fakeTimers() {
|
||||
const timers = new Map();
|
||||
let next = 1;
|
||||
return {
|
||||
set: (callback, delay) => { timers.set(next, { callback, delay }); return next++; },
|
||||
clear: id => timers.delete(id),
|
||||
fire: () => { for (const [id, { callback }] of [...timers]) { timers.delete(id); callback(); } },
|
||||
pending: () => timers.size,
|
||||
delays: () => [...timers.values()].map(timer => timer.delay),
|
||||
};
|
||||
}
|
||||
|
||||
test('a held press fires once after the delay and swallows the click that follows', async () => {
|
||||
const { createLongPress, LONG_PRESS_MS } = await source;
|
||||
const timers = fakeTimers(), fired = [];
|
||||
const press = createLongPress({ onLongPress: target => fired.push(target), setTimer: timers.set, clearTimer: timers.clear });
|
||||
press.down(10, 10, 'card-a');
|
||||
assert.deepEqual(timers.delays(), [LONG_PRESS_MS]);
|
||||
assert.equal(LONG_PRESS_MS, 500);
|
||||
press.move(14, 13);
|
||||
timers.fire();
|
||||
assert.deepEqual(fired, ['card-a']);
|
||||
press.up();
|
||||
assert.equal(press.consumeClick(), true);
|
||||
assert.equal(press.consumeClick(), false);
|
||||
});
|
||||
|
||||
test('moving past the slop, lifting early or cancelling never fires', async () => {
|
||||
const { createLongPress, LONG_PRESS_SLOP } = await source;
|
||||
assert.equal(LONG_PRESS_SLOP, 10);
|
||||
const timers = fakeTimers(), fired = [];
|
||||
const press = createLongPress({ onLongPress: target => fired.push(target), setTimer: timers.set, clearTimer: timers.clear });
|
||||
press.down(0, 0, 'a'); press.move(0, 11);
|
||||
assert.equal(timers.pending(), 0);
|
||||
press.down(0, 0, 'b'); press.up();
|
||||
assert.equal(timers.pending(), 0);
|
||||
press.down(0, 0, 'c');
|
||||
assert.equal(press.pending(), true);
|
||||
press.cancel();
|
||||
assert.equal(press.pending(), false);
|
||||
assert.equal(timers.pending(), 0);
|
||||
press.move(0, 0); press.up(); press.cancel();
|
||||
assert.deepEqual(fired, []);
|
||||
assert.equal(press.consumeClick(), false);
|
||||
// A new press drops a long press whose click never came.
|
||||
press.down(0, 0, 'd'); timers.fire();
|
||||
press.down(0, 0, 'e');
|
||||
assert.equal(press.consumeClick(), false);
|
||||
timers.fire();
|
||||
assert.deepEqual(fired, ['d', 'e']);
|
||||
// A cancel (a mouse press on a hybrid device) forgets a fired press whose click never came.
|
||||
press.cancel();
|
||||
assert.equal(press.consumeClick(), false);
|
||||
});
|
||||
@@ -494,11 +494,3 @@ test('defaults apply to a new visit but not to the current view or a saved view'
|
||||
assert.deepEqual(initialImagesView({}), { layout: 'grid', density: 'comfortable' });
|
||||
assert.deepEqual(initialImagesView({ preferences: {} }), { layout: 'grid', density: 'comfortable' });
|
||||
});
|
||||
|
||||
test('listingFilter is the listing query as an object, without paging (#392)', async () => {
|
||||
const { listingFilter } = await source;
|
||||
assert.deepEqual(listingFilter({q:'cat', root:'1', subfolder:'a/b', category:'Animals', tags:[' x ','y'], media:'video', sort:'oldest', favorites:true}),
|
||||
{q:'cat', root:'1', subfolder:'a/b', category:'Animals', tags:'x,y', media:'video', sort:'oldest', favorites:'1'});
|
||||
assert.deepEqual(listingFilter({root:'all', q:'', tags:[], media:'all'}), {sort:'newest'});
|
||||
assert.throws(() => listingFilter({sort:'sideways'}), TypeError);
|
||||
});
|
||||
|
||||
@@ -196,29 +196,3 @@ test('a preset starts the Tag image dialog on the chosen model and options', asy
|
||||
assert.throws(() => createImageTaggingController({ api, target, preset: { model: 'gguf', options: { instructions: '' } } }), TypeError);
|
||||
assert.throws(() => createImageTaggingController({ api, target, preset: { model: 'gguf', options: { nope: 1 } } }), TypeError);
|
||||
});
|
||||
|
||||
test('all matching sends the listing filter instead of targets, up to 10,000 images (#392)', async () => {
|
||||
const { openAddTags, openAutoTag, MAX_FILTER_IMAGES, TOO_MANY_MATCHING } = module();
|
||||
assert.equal(MAX_FILTER_IMAGES, 10000);
|
||||
assert.equal(TOO_MANY_MATCHING, 'Narrow the view to at most 10,000 images.');
|
||||
const toasts = [], posted = [], tagged = [];
|
||||
let ended = null;
|
||||
const store = { start: async (kind, body) => { posted.push([kind, body]); return { job: { id: JOB_ID, state: 'running' } }; },
|
||||
find: () => ended, subscribe: () => () => {} };
|
||||
const filter = { q: 'cat', sort: 'newest' };
|
||||
// The loaded images can be more than 500: the filter, not the identities, is sent.
|
||||
const loaded = Array.from({ length: 600 }, (_, index) => image(`bulk/${index}.png`));
|
||||
let dialog = null;
|
||||
openAddTags({ scope: pageScope(), store, images: loaded, filter, count: 1234, notify: text => toasts.push(text),
|
||||
onTagged: keys => tagged.push(keys.size), openInput: args => { dialog = args; } });
|
||||
assert.ok(dialog, 'the Add tags dialog opens');
|
||||
ended = { id: JOB_ID, state: 'finished', counters: { tagged: 3 } };
|
||||
await dialog.onApply('sunset');
|
||||
assert.deepEqual(posted, [['image-tags', { mode: 'add', filter, tags: ['sunset'] }]]);
|
||||
// The loaded files are read again when the job tagged any.
|
||||
assert.deepEqual(tagged, [600]);
|
||||
openAddTags({ scope: pageScope(), store, images: loaded, filter, count: 10001, notify: text => toasts.push(text),
|
||||
openInput: () => assert.fail('no dialog over the cap') });
|
||||
openAutoTag({ api: {}, scope: pageScope(), store, images: loaded, filter, count: 10001, notify: text => toasts.push(text) });
|
||||
assert.deepEqual(toasts, [TOO_MANY_MATCHING, TOO_MANY_MATCHING]);
|
||||
});
|
||||
|
||||
@@ -23,7 +23,7 @@ test('tags is a native route titled Tags; only unbuilt routes hand off to legacy
|
||||
assert.equal(routeTitle(route), route.label);
|
||||
}
|
||||
assert.equal(routeTitle(resolveRoute('#/library/7')), 'Prompt library');
|
||||
// System is native from S2 (#249 System), and every section of it from S12 (D94 retired), not the route.
|
||||
// System is native from S2 (#249 System): its undelivered sections hand off per section (D94), not the route.
|
||||
for (const id of ['images', 'library', 'tags', 'inspect', 'overview', 'activity', 'review', 'system']) assert.equal(isHandoff(resolveRoute(`#/${id}`)), false, id);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
// System › Diagnostics & logs (#249 System S12, #329): the concept's two surfaces (app.js 694–695) with live values: the
|
||||
// meta-grid texts, the report's name and merge (D110), the log viewer's lines, Lines validation, statistics, retention
|
||||
// and file rows, and the Clear all logs copy (D109).
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const diagnostics = import('../../web/js/workspace/system-diagnostics.mjs');
|
||||
|
||||
const CONCEPT = fs.readFileSync(path.join(__dirname, '../../docs/concepts/workspace/app.js'), 'utf8');
|
||||
const SOURCE = fs.readFileSync(path.join(__dirname, '../../web/js/workspace/system-diagnostics.mjs'), 'utf8');
|
||||
const SYSTEM = fs.readFileSync(path.join(__dirname, '../../web/js/workspace/system.mjs'), 'utf8');
|
||||
|
||||
const REPORT = {
|
||||
checks: { database: { status: 'ok', prompt_count: 4210, has_images_table: true, image_count: 19956 } },
|
||||
database_info: { database_path: 'user/__promptmanager/prompts.db', database_size_bytes: 1717986918 },
|
||||
sources: { configured: 2, effective: 2, watching: 2, unavailable: 0 },
|
||||
viewerjs: '1.15.2',
|
||||
};
|
||||
|
||||
test('the section keeps the concept surfaces, terms, rows and labels; no sample copy ships', () => {
|
||||
const start = CONCEPT.indexOf('case "diagnostics":');
|
||||
const body = CONCEPT.slice(start, CONCEPT.indexOf('break;', start));
|
||||
const terms = [...body.matchAll(/<dt>([^<]+)<\/dt>/g)].map(([, term]) => term);
|
||||
assert.deepEqual(terms, ['Database', 'Sources', 'Viewer.js', 'Host renderers']);
|
||||
for (const text of ["'Diagnostics'", "'Logs'", "'Run checks'", "'Refresh'", "'Level'", "'Log files & retention'",
|
||||
"'Review log cleanup'", "'Runtime log level'", "'Console logging'", "'Image-link diagnostics'"]) {
|
||||
assert.ok(SOURCE.includes(text), text);
|
||||
}
|
||||
for (const id of ['runtime-level', 'console-log']) assert.ok(body.includes(`setting("${id}"`) && SOURCE.includes(`id: '${id}'`), id);
|
||||
assert.ok(SOURCE.includes("'toolbar spaced-16'") && body.includes('class="toolbar spaced-16"'));
|
||||
// Prototype copy is never shipped.
|
||||
for (const sample of ['sample', 'Sample']) assert.ok(!SOURCE.includes(`'${sample}`) && !/Download sample|Inspect sample/.test(SOURCE), sample);
|
||||
});
|
||||
|
||||
test('D94 is retired: Diagnostics & logs is a delivered section and nothing hands off', () => {
|
||||
assert.ok(SYSTEM.includes('diagnostics: mountDiagnostics'));
|
||||
assert.ok(!SYSTEM.includes('SECTION_HANDOFFS'));
|
||||
assert.ok(!SYSTEM.includes('handoff('));
|
||||
assert.ok(!SYSTEM.includes('NOT_DELIVERED'));
|
||||
});
|
||||
|
||||
test('database text covers readable, not readable, busy and loading', async () => {
|
||||
const { databaseText, LOADING, UNREADABLE } = await diagnostics;
|
||||
assert.equal(databaseText({ report: REPORT }), 'Readable · 1.6 GB · 19,956 images');
|
||||
assert.equal(databaseText({ report: { ...REPORT, checks: { database: { status: 'ok', image_count: 1 } },
|
||||
database_info: { database_size_bytes: 2048 } } }), 'Readable · 2.0 KB · 1 image');
|
||||
assert.equal(databaseText({ report: { checks: { database: { status: 'error', message: 'Database error: locked' } } } }),
|
||||
'Not readable: Database error: locked');
|
||||
assert.equal(databaseText({ loading: true }), LOADING);
|
||||
assert.equal(LOADING, 'Loading…');
|
||||
assert.equal(databaseText({ error: { code: 'busy', running: { id: 'j1', title: 'Restore' } } }), 'Busy: Restore is running.');
|
||||
assert.equal(databaseText({ error: { code: 'busy' } }), 'Busy: another job is running.');
|
||||
assert.equal(databaseText({ error: new Error('offline') }), UNREADABLE);
|
||||
assert.equal(UNREADABLE, 'Could not be read.');
|
||||
});
|
||||
|
||||
test('sources text follows the precedence table', async () => {
|
||||
const { sourcesText } = await diagnostics;
|
||||
const of = sources => sourcesText({ report: { sources } });
|
||||
assert.equal(of({ configured: 0, effective: 1, watching: 1, unavailable: 0 }), 'ComfyUI output folder · watching');
|
||||
assert.equal(of({ configured: 0, effective: 1, watching: 0, unavailable: 0 }), 'ComfyUI output folder · manual scan');
|
||||
// Unavailable wins over watching.
|
||||
assert.equal(of({ configured: 3, effective: 2, watching: 2, unavailable: 1 }), '3 sources · 1 not available');
|
||||
assert.equal(of({ configured: 2, effective: 2, watching: 2, unavailable: 0 }), '2 sources · 2 watching');
|
||||
assert.equal(of({ configured: 1, effective: 1, watching: 0, unavailable: 0 }), '1 source · 0 watching');
|
||||
assert.equal(of(undefined), 'ComfyUI output folder · manual scan');
|
||||
});
|
||||
|
||||
test('Viewer.js reads the pinned vendor version', async () => {
|
||||
const { viewerText } = await diagnostics;
|
||||
assert.equal(viewerText({ report: REPORT }), '1.15.2 · pinned vendor');
|
||||
assert.equal(viewerText({ report: {} }), 'unknown · pinned vendor');
|
||||
});
|
||||
|
||||
test('report filename uses local time', async () => {
|
||||
const { reportFilename } = await diagnostics;
|
||||
assert.equal(reportFilename(new Date(2026, 9, 6, 7, 5)), 'prompt-manager-diagnostics-20261006-0705.json');
|
||||
assert.equal(reportFilename(new Date(2026, 11, 31, 23, 59)), 'prompt-manager-diagnostics-20261231-2359.json');
|
||||
});
|
||||
|
||||
test('merged report adds only host_renderers', async () => {
|
||||
const { mergeReport } = await diagnostics;
|
||||
const merged = mergeReport(REPORT, 'Detected: Classic (1 ComfyUI window).');
|
||||
assert.deepEqual(Object.keys(merged).sort(), [...Object.keys(REPORT), 'host_renderers'].sort());
|
||||
assert.equal(merged.host_renderers, 'Detected: Classic (1 ComfyUI window).');
|
||||
assert.equal(mergeReport(REPORT, null).host_renderers, null);
|
||||
assert.equal(mergeReport(REPORT, undefined).host_renderers, null);
|
||||
// The server's fields are not changed.
|
||||
assert.equal(merged.sources, REPORT.sources);
|
||||
});
|
||||
|
||||
test('sizes read as the server labels them (1 KB = 1024 bytes)', async () => {
|
||||
const { sizeText } = await diagnostics;
|
||||
assert.equal(sizeText(0), '0 bytes');
|
||||
assert.equal(sizeText(1), '1 byte');
|
||||
assert.equal(sizeText(1000), '1,000 bytes');
|
||||
assert.equal(sizeText(2048), '2.0 KB');
|
||||
assert.equal(sizeText(10 * 2 ** 20), '10.0 MB');
|
||||
assert.equal(sizeText(1717986918), '1.6 GB');
|
||||
assert.equal(sizeText(undefined), '0 bytes');
|
||||
});
|
||||
|
||||
test("log line uses the concept's format", async () => {
|
||||
const { logLine } = await diagnostics;
|
||||
assert.equal(logLine({ timestamp: '2026-10-26T09:42:18.123456', level: 'INFO', message: 'Database backup verified' }),
|
||||
'09:42:18 INFO Database backup verified');
|
||||
assert.equal(logLine({ timestamp: '2026-10-26T09:42:18', level: 'WARNING', message: '3 images have ambiguous prompt metadata' }),
|
||||
'09:42:18 WARNING 3 images have ambiguous prompt metadata');
|
||||
assert.match(logLine({ timestamp: '2026-10-26T09:42:18', level: 'ERROR', message: 'x' }), /^\d\d:\d\d:\d\d [A-Z]+ +\S/);
|
||||
// The concept's sample lines read the same way.
|
||||
assert.ok(CONCEPT.includes('09:42:18 INFO Scan completed'));
|
||||
});
|
||||
|
||||
test('Lines accepts whole numbers 1 to 500 only', async () => {
|
||||
const { validLimit, logsUrl } = await diagnostics;
|
||||
for (const [value, expected] of [['1', 1], ['500', 500], [' 20 ', 20], ['100', 100]]) assert.equal(validLimit(value), expected, value);
|
||||
for (const value of ['0', '501', '2.5', '', ' ', '-1', '1e2', 'abc', undefined]) assert.equal(validLimit(value), null, String(value));
|
||||
assert.equal(logsUrl('', 20), '/prompt_manager/logs?limit=20');
|
||||
assert.equal(logsUrl('WARNING', 20), '/prompt_manager/logs?limit=20&level=WARNING');
|
||||
});
|
||||
|
||||
test('the level filter offers all six minimum levels; the runtime level adds CRITICAL after the concept four', async () => {
|
||||
const { LEVEL_OPTIONS, RUNTIME_LEVELS } = await diagnostics;
|
||||
assert.deepEqual(LEVEL_OPTIONS.map(([, label]) => label), ['All levels', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']);
|
||||
assert.equal(LEVEL_OPTIONS[0][0], '');
|
||||
assert.deepEqual(RUNTIME_LEVELS, ['INFO', 'WARNING', 'DEBUG', 'ERROR', 'CRITICAL']);
|
||||
assert.ok(CONCEPT.includes('["INFO", "WARNING", "DEBUG", "ERROR"]'));
|
||||
});
|
||||
|
||||
test('statistics text uses singular forms for one', async () => {
|
||||
const { statsText } = await diagnostics;
|
||||
assert.equal(statsText({ buffer_count: 1000, level_counts: { INFO: 980, WARNING: 12, ERROR: 2, CRITICAL: 1 },
|
||||
log_files_count: 5, total_log_size: 43201024 }), '1,000 entries in memory · 3 errors · 12 warnings · 5 files · 41.2 MB on disk');
|
||||
assert.equal(statsText({ buffer_count: 1, level_counts: { WARNING: 1, CRITICAL: 1 }, log_files_count: 1, total_log_size: 1 }),
|
||||
'1 entry in memory · 1 error · 1 warning · 1 file · 1 byte on disk');
|
||||
assert.equal(statsText({}), '0 entries in memory · 0 errors · 0 warnings · 0 files · 0 bytes on disk');
|
||||
});
|
||||
|
||||
test('retention, file line and file rows read the server values', async () => {
|
||||
const { retentionText, fileLine, fileDetail } = await diagnostics;
|
||||
assert.equal(retentionText({ max_file_size: 10485760, backup_count: 5, file_logging: true, buffer_size: 1000 }),
|
||||
'Rotates at 10 MB · keeps 5 rotated copies · file logging on · memory buffer 1,000 entries');
|
||||
assert.equal(retentionText({ max_file_size: 1572864, backup_count: 1, file_logging: false, buffer_size: 1 }),
|
||||
'Rotates at 1.5 MB · keeps 1 rotated copy · file logging off · memory buffer 1 entry');
|
||||
const files = [
|
||||
{ filename: 'prompt_manager.log', kind: 'active', size: 1468006, modified: '2026-10-26T09:42:18' },
|
||||
{ filename: 'prompt_manager.log.1', kind: 'rotated', size: 10485760, modified: '2026-10-25T14:02:00' },
|
||||
{ filename: 'prompt_manager.log.4', kind: 'link', size: 20, modified: '2026-10-25T14:02:00' },
|
||||
{ filename: 'prompt_manager.log.bak', kind: 'other', size: 0, modified: '2026-10-24T08:00:00' },
|
||||
];
|
||||
assert.equal(fileLine(files), 'prompt_manager.log · 1.4 MB · rotated copies: 1');
|
||||
assert.equal(fileLine([]), 'prompt_manager.log · not written yet · rotated copies: 0');
|
||||
assert.equal(fileDetail(files[1]), '10.0 MB · 2026-10-25 14:02 · Rotated copy');
|
||||
assert.equal(fileDetail(files[2]), '20 bytes · 2026-10-25 14:02 · Link, not downloaded');
|
||||
assert.equal(fileDetail(files[3]), '0 bytes · 2026-10-24 08:00 · Other file');
|
||||
assert.equal(fileDetail(files[0]), '1.4 MB · 2026-10-26 09:42 · Active log');
|
||||
});
|
||||
|
||||
test('Clear all logs states both counts and its toast says what was done', async () => {
|
||||
const { clearSentence, clearToast } = await diagnostics;
|
||||
const rotated = size => ({ kind: 'rotated', size });
|
||||
const files = [{ kind: 'active', size: 10 }, rotated(10 * 2 ** 20), rotated(10 * 2 ** 20), rotated(8.4 * 2 ** 20), { kind: 'other', size: 5 }];
|
||||
assert.equal(clearSentence(files), 'Empties prompt_manager.log, deletes 3 rotated copies (28.4 MB) and clears the lines shown '
|
||||
+ 'here. The other files in the log folder are kept. This cannot be undone.');
|
||||
assert.equal(clearSentence([{ kind: 'active', size: 1 }]), 'Empties prompt_manager.log and clears the lines shown here. '
|
||||
+ 'The other files in the log folder are kept. This cannot be undone.');
|
||||
assert.match(clearSentence(null), /^Empties prompt_manager.log, deletes its rotated copies and clears/);
|
||||
assert.equal(clearToast({ truncated: ['prompt_manager.log', 'prompt_manager.log.1', 'prompt_manager.log.2', 'prompt_manager.log.3'],
|
||||
errors: [] }), 'Logs cleared · 3 rotated copies deleted.');
|
||||
assert.equal(clearToast({ truncated: ['prompt_manager.log', 'prompt_manager.log.1', 'prompt_manager.log.2'],
|
||||
errors: ['Failed to truncate prompt_manager.log.3: in use'] }), 'Logs cleared · 2 rotated copies deleted · 1 could not be deleted.');
|
||||
assert.equal(clearToast({ truncated: ['prompt_manager.log', 'prompt_manager.log.1'], errors: [] }), 'Logs cleared · 1 rotated copy deleted.');
|
||||
assert.equal(clearToast(undefined), 'Logs cleared · 0 rotated copies deleted.');
|
||||
});
|
||||
|
||||
test('the test-link dialog names the prompt and how Missing-file links removes the record', async () => {
|
||||
const { testLinkSentence, promptName } = await diagnostics;
|
||||
assert.equal(testLinkSentence({ id: 812, text: 'Misty harbor' }), 'Writes a placeholder image record '
|
||||
+ '(prompt_manager_test_link.png) linked to prompt #812 “Misty harbor”. Maintenance › Missing-file links removes it later.');
|
||||
assert.equal(promptName({ text: ' a\n b ' }), 'a b');
|
||||
assert.equal(promptName({ text: 'x'.repeat(80) }).length, 60);
|
||||
});
|
||||
|
||||
test('runtime values and body map config.level to runtime_level and back', async () => {
|
||||
const { runtimeValues, runtimeBody } = await diagnostics;
|
||||
assert.deepEqual(runtimeValues({ level: 'INFO', console_logging: true, max_file_size: 1 }), { runtime_level: 'INFO', console_logging: true });
|
||||
assert.deepEqual(runtimeBody({ runtime_level: 'CRITICAL' }), { level: 'CRITICAL' });
|
||||
assert.deepEqual(runtimeBody({ console_logging: false }), { console_logging: false });
|
||||
assert.deepEqual(runtimeBody({ runtime_level: 'DEBUG', console_logging: true }), { level: 'DEBUG', console_logging: true });
|
||||
assert.deepEqual(runtimeBody({}), {});
|
||||
});
|
||||
@@ -169,44 +169,3 @@ test('a plan with nothing to do offers no Apply (S7 live test)', async () => {
|
||||
assert.equal(hasWork({ ...PLAN, scope: {} }), true);
|
||||
assert.equal(NOTHING_TO_APPLY, 'There is nothing to apply; nothing was changed.');
|
||||
});
|
||||
|
||||
// S12 (#329): the rotated-log cleanup plan changes no library record: its Backup reads "Not needed", the review shows no
|
||||
// D55 backup box, and its details are files.
|
||||
const LOG_PLAN = { operation: 'rotated_logs', token: 't', expires_at: '2026-10-26T23:15:00',
|
||||
scope: { count: 3, label: '3 rotated log copies · 28.4 MB' },
|
||||
protected: { count: 3, reasons: ['Kept: the active log', 'Kept: 1 link', 'Kept: 1 other file'] },
|
||||
backup: { policy: 'none', destination_label: null, expected_bytes: null },
|
||||
media: { deletes_files: true, label: 'Deletes only the listed log files. No image, video or library record changes.' },
|
||||
details: [{ label: 'prompt_manager.log.1', detail: '10.0 MB · 2026-10-25 14:02' }] };
|
||||
|
||||
test('a plan without a database change reads Not needed and shows no backup box', async () => {
|
||||
const { planGrid, backupText, backupBox, BACKUP_COPY, OPERATION_TITLES } = await review;
|
||||
assert.equal(backupText({ policy: 'none' }), 'Not needed: no library record changes');
|
||||
assert.deepEqual(planGrid(LOG_PLAN), [
|
||||
['Scope', '3 rotated log copies · 28.4 MB'],
|
||||
['Protected', 'Kept: the active log · Kept: 1 link · Kept: 1 other file'],
|
||||
['Backup', 'Not needed: no library record changes'],
|
||||
['Media on disk', 'Deletes only the listed log files. No image, video or library record changes.'],
|
||||
]);
|
||||
const onBackup = () => {};
|
||||
assert.equal(backupBox(LOG_PLAN, onBackup), null);
|
||||
// Every database plan keeps its D55 box.
|
||||
assert.deepEqual(backupBox({ ...LOG_PLAN, operation: 'optimize', backup: { policy: 'required' } }, onBackup), { copy: BACKUP_COPY, onBackup });
|
||||
assert.deepEqual(backupBox({ operation: 'missing_links' }, onBackup), { copy: BACKUP_COPY, onBackup });
|
||||
// Its title heads the dialog and lets Activity offer Review again.
|
||||
assert.equal(OPERATION_TITLES.rotated_logs, 'Rotated log copies');
|
||||
});
|
||||
|
||||
test('rotated_logs details read Listed files', async () => {
|
||||
const { detailsSummary, detailLines } = await review;
|
||||
assert.equal(detailsSummary(LOG_PLAN, 3), 'Listed files (3)');
|
||||
assert.equal(detailsSummary({ ...LOG_PLAN, scope: { count: 60 } }, 50), 'Listed files (50 of 60)');
|
||||
assert.equal(detailsSummary({ operation: 'missing_links', scope: { count: 2 } }, 2), 'Listed records (2)');
|
||||
assert.deepEqual(detailLines(LOG_PLAN), ['prompt_manager.log.1 · 10.0 MB · 2026-10-25 14:02']);
|
||||
});
|
||||
|
||||
test('a rotated-log cleanup the log overtook offers Review again (S12)', async () => {
|
||||
const { reviewAgainOf } = await jobView;
|
||||
assert.deepEqual(reviewAgainOf({ kind: 'maintenance', state: 'failed', scope: { operation: 'rotated_logs', failure: 'plan-changed' } }),
|
||||
{ operation: 'rotated_logs' });
|
||||
});
|
||||
|
||||
@@ -420,12 +420,3 @@ test('a partial host save keeps what saved, adopts its revision and fails only t
|
||||
assert.equal((await drafts.save()).host, 'failed');
|
||||
assert.equal(drafts.statusText(), SETTINGS_STATUS.partial);
|
||||
});
|
||||
|
||||
// S12 (#329): System's runtime domain is the running server's log level and console setting (`/logs/config`).
|
||||
test('runtime adapter maps config.level to runtime_level and back', async () => {
|
||||
const { runtimeValues, runtimeBody } = await import('../../web/js/workspace/system-diagnostics.mjs');
|
||||
assert.deepEqual(runtimeValues({ level: 'WARNING', console_logging: false, backup_count: 5 }),
|
||||
{ runtime_level: 'WARNING', console_logging: false });
|
||||
assert.deepEqual(runtimeBody({ runtime_level: 'WARNING' }), { level: 'WARNING' });
|
||||
assert.deepEqual(runtimeBody({ runtime_level: 'CRITICAL', console_logging: true }), { level: 'CRITICAL', console_logging: true });
|
||||
});
|
||||
|
||||
@@ -360,23 +360,19 @@ test('Preview tagging review opens the Tag image dialog with the section’s val
|
||||
} finally { scope.dispose(); dom.restore(); }
|
||||
});
|
||||
|
||||
const PLAN = { images: 3, already_tagged: 1, to_tag: 2, estimate_seconds: 3.2, estimate_device: 'cpu', suggestion_limit: 10,
|
||||
const PLAN = { images: 3, prompts: 2, already_tagged: 1, to_tag: 1,
|
||||
model: { type: 'wd14-swinv2', label: 'WD14 SwinV2', size_label: '~400 MB', downloaded: true }, existing: 'skip',
|
||||
review_required: false, backup: { policy: 'on' }, disk: { free_bytes: 10 * 2 ** 30, required_bytes: 2 ** 20, sufficient: true } };
|
||||
|
||||
test('the batch review grid, body and Cancel-only reasons (D134, #379, CUJ 4–5)', async () => {
|
||||
const { batchGrid, batchBody, batchCloseNote, existingLabel, REPLACE_NOTE, NOTHING_TO_TAG, NO_IMAGES } = await tagging;
|
||||
test('the batch review grid, body and Cancel-only reasons (D134, CUJ 4–5)', async () => {
|
||||
const { batchGrid, batchBody, batchCloseNote, existingLabel, REPLACE_NOTE, NOTHING_TO_TAG, NO_LINKED_IMAGES } = await tagging;
|
||||
assert.deepEqual(batchGrid(PLAN, HOST), [['Model', 'WD14 SwinV2 · ~400 MB'], ['Existing tags', 'Skip already tagged'],
|
||||
['Images', '3'], ['Already auto-tagged', '1'], ['Images to tag', 'about 2'],
|
||||
['Time', 'about 1 min on a processor; a graphics card can be faster.'],
|
||||
['Settings', 'General 0.35 · Character 0.85 · up to 10 tags per image'], ['Backup', 'Create and verify before changes']]);
|
||||
['Images linked', '3'], ['Prompts', '2'], ['Already tagged', '1'], ['Prompts to tag', 'up to 1'],
|
||||
['Settings', 'General 0.35 · Character 0.85'], ['Backup', 'Create and verify before changes']]);
|
||||
const llm = { ...HOST, autotag_model_type: 'gguf' };
|
||||
assert.deepEqual(batchGrid({ ...PLAN, model: { type: 'gguf' }, backup: { policy: 'off' }, estimate_seconds: null,
|
||||
estimate_device: null, suggestion_limit: 4 }, llm).slice(-3),
|
||||
[['Time', 'Language-model engines take longer per image. Activity shows the time left once tagging starts.'],
|
||||
['Settings', 'Template of 27 characters · up to 4 tags per image'], ['Backup', 'No safety backup (the policy is off)']]);
|
||||
assert.deepEqual(batchGrid({ ...PLAN, model: { type: 'gguf' }, backup: { policy: 'off' } }, llm).slice(-2),
|
||||
[['Settings', 'Template of 27 characters'], ['Backup', 'No safety backup (the policy is off)']]);
|
||||
assert.equal(batchGrid({}, HOST)[2][1], '0');
|
||||
assert.equal(batchGrid({}, HOST)[6][1], 'General 0.35 · Character 0.85');
|
||||
assert.equal(existingLabel('odd'), 'odd');
|
||||
assert.deepEqual(batchBody(HOST), { model_type: 'wd14-swinv2', existing: 'skip',
|
||||
options: { general_threshold: 0.35, character_threshold: 0.85 } });
|
||||
@@ -389,31 +385,9 @@ test('the batch review grid, body and Cancel-only reasons (D134, #379, CUJ 4–5
|
||||
assert.equal(batchCloseNote({ ...PLAN, model: { ...PLAN.model, downloaded: false } }), 'WD14 SwinV2 is not downloaded. Download it first.');
|
||||
assert.equal(batchCloseNote({ ...PLAN, disk: { free_bytes: 2 ** 20, required_bytes: 2 ** 21, sufficient: false } }),
|
||||
'Not enough free disk space: the safety backup needs about 2.0 MB and 1.0 MB is free. Nothing is written until there is room.');
|
||||
assert.equal(batchCloseNote({ ...PLAN, images: 0, to_tag: 0 }), NO_IMAGES);
|
||||
assert.equal(batchCloseNote({ ...PLAN, images: undefined }), NO_IMAGES);
|
||||
assert.equal(NO_IMAGES, 'The output folders contain no images to tag.');
|
||||
assert.equal(batchCloseNote({ ...PLAN, prompts: 0, to_tag: 0 }), NO_LINKED_IMAGES);
|
||||
assert.equal(batchCloseNote({ ...PLAN, to_tag: 0 }), NOTHING_TO_TAG);
|
||||
assert.equal(NOTHING_TO_TAG, 'Every output image is already auto-tagged.');
|
||||
});
|
||||
|
||||
test('the review Time row (DEC11, #379): minutes, hours and the engine it runs on', async () => {
|
||||
const { durationText, timeText, TIME_UNKNOWN } = await tagging;
|
||||
assert.equal(durationText(1), 'about 1 min');
|
||||
assert.equal(durationText(0), 'about 1 min');
|
||||
assert.equal(durationText(89), 'about 89 min');
|
||||
assert.equal(durationText(90), 'about 1 h 30 min');
|
||||
assert.equal(durationText(120), 'about 2 h');
|
||||
assert.equal(durationText(631), 'about 10 h 31 min');
|
||||
assert.equal(durationText(Number.NaN), 'about 1 min');
|
||||
// 23,645 images at 1.6 s on a processor (the design's CUJ 1).
|
||||
assert.equal(timeText({ estimate_seconds: 37832, estimate_device: 'cpu' }),
|
||||
'about 10 h 31 min on a processor; a graphics card can be faster.');
|
||||
assert.equal(timeText({ estimate_seconds: 59, estimate_device: 'gpu' }), 'about 1 min on the graphics card.');
|
||||
assert.equal(timeText({ estimate_seconds: 0, estimate_device: 'cpu' }), 'about 1 min on a processor; a graphics card can be faster.');
|
||||
for (const plan of [{ estimate_seconds: null }, {}, { estimate_seconds: -1 }, { estimate_seconds: Infinity }, null]) {
|
||||
assert.equal(timeText(plan), TIME_UNKNOWN);
|
||||
}
|
||||
assert.equal(TIME_UNKNOWN, 'Language-model engines take longer per image. Activity shows the time left once tagging starts.');
|
||||
assert.equal(NOTHING_TO_TAG, 'Every prompt with an output image is already tagged.');
|
||||
});
|
||||
|
||||
test('showBatchPlan opens the review with the backup box, or the Cancel-only review', async () => {
|
||||
|
||||
@@ -35,8 +35,6 @@ REVIEW_TABLES = (
|
||||
"media_review_items",
|
||||
"media_review_metadata",
|
||||
)
|
||||
# The auto-tag marks (#379) cite annotation bytes too; an old backup lacks them.
|
||||
CITING_TABLES = REVIEW_TABLES + ("media_image_autotags",)
|
||||
# The exact DDL the plan pins; the module must produce precisely these objects.
|
||||
DDL = {
|
||||
METADATA: (
|
||||
@@ -216,7 +214,7 @@ class AnnotationSchemaTests(unittest.TestCase):
|
||||
"""
|
||||
with closing(sqlite3.connect(path)) as raw, raw:
|
||||
raw.execute("PRAGMA foreign_keys=OFF")
|
||||
for table in CITING_TABLES + tuple(reversed(TABLES)):
|
||||
for table in REVIEW_TABLES + tuple(reversed(TABLES)):
|
||||
raw.execute(f"DROP TABLE {table}")
|
||||
|
||||
def test_fresh_install_and_reopen_is_idempotent(self):
|
||||
|
||||
+7
-157
@@ -430,9 +430,7 @@ class ReplaceImageTagsTests(AnnotationTestCase):
|
||||
self.assertTrue(
|
||||
self.db.refresh_image_fingerprint(loc, DIGEST_A, SIZE, MTIME + 1)
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.image_tag_locators(self.db.image_tag_ids(["img"], "")), {loc}
|
||||
)
|
||||
list(self.db.iter_image_tag_records())
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["prompt-tag"])
|
||||
|
||||
|
||||
@@ -553,162 +551,14 @@ class CurrentImageTagsTests(AnnotationTestCase):
|
||||
self.assertEqual(current, {locators[0]: ["z", "y"], locators[502]: ["x"]})
|
||||
lookups = [s for s in statements if "media_annotation_files" in s]
|
||||
self.assertEqual(len(lookups), 2)
|
||||
# The tag search finds every record's file by tag id, and the empty
|
||||
# set's file never.
|
||||
self.assertEqual(
|
||||
self.db.image_tag_locators(self.db.image_tag_ids(["z", "x"], "")),
|
||||
{locators[0], locators[502]},
|
||||
list(self.db.iter_image_tag_records()),
|
||||
[
|
||||
(locators[0], SIZE, MTIME, ["z", "y"]),
|
||||
(locators[250], SIZE, MTIME + 1, []),
|
||||
(locators[502], SIZE, MTIME + 2, ["x"]),
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({locators[250]: (SIZE, MTIME + 1)}),
|
||||
{locators[250]: []},
|
||||
)
|
||||
|
||||
def test_tag_locators_cover_more_ids_than_one_chunk(self):
|
||||
# A broad text search can match more than 500 names: every id's file
|
||||
# comes back across the chunks.
|
||||
locators = [self.locator(f"t{i:04}.png") for i in range(503)]
|
||||
for i, loc in enumerate(locators):
|
||||
self.replace(loc, tags=[f"name_{i:04}"])
|
||||
ids = self.db.image_tag_ids([], "name_")
|
||||
self.assertEqual(len(ids), 503)
|
||||
self.assertEqual(self.db.image_tag_locators(ids), set(locators))
|
||||
|
||||
|
||||
class AutotagMarkTests(AnnotationTestCase):
|
||||
"""The auto-tagged mark (#379): per byte record, written with the tags."""
|
||||
|
||||
def marked(self):
|
||||
conn = self.db.model.get_connection()
|
||||
return sorted(
|
||||
row[0] for row in conn.execute("SELECT * FROM media_image_autotags")
|
||||
)
|
||||
|
||||
def byte_id(self, loc, digest=DIGEST_A):
|
||||
conn = self.db.model.get_connection()
|
||||
return conn.execute(
|
||||
"SELECT b.id FROM media_annotation_bytes b JOIN media_annotation_files f "
|
||||
"ON f.id=b.file_id WHERE f.locator=? AND b.sha256=?",
|
||||
(loc, digest),
|
||||
).fetchone()[0]
|
||||
|
||||
def test_hand_tags_never_mark(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=["keeper"])
|
||||
self.add(loc, tags=["more"])
|
||||
self.assertEqual(self.marked(), [])
|
||||
self.assertEqual(self.db.autotag_state({loc: (SIZE, MTIME)}), {loc: False})
|
||||
self.assertEqual(self.db.autotagged_locators(), set())
|
||||
|
||||
def test_model_union_marks_including_replayed_and_empty_results(self):
|
||||
loc = self.locator("a.png")
|
||||
receipt = self.add(loc, tags=["bird"], mark_autotagged=True)
|
||||
self.assertEqual((receipt["added"], receipt["replayed"]), (1, False))
|
||||
self.assertEqual(self.marked(), [self.byte_id(loc)])
|
||||
# A replayed union and an empty result still mark (idempotently).
|
||||
other = self.locator("b.png")
|
||||
self.replace(other, DIGEST_B, ["keeper"])
|
||||
replayed = self.add(other, DIGEST_B, ["KEEPER"], mark_autotagged=True)
|
||||
self.assertTrue(replayed["replayed"])
|
||||
empty = self.locator("c.png")
|
||||
nothing = self.add(empty, tags=[], mark_autotagged=True)
|
||||
self.assertEqual((nothing["tags"], nothing["added"]), ([], 0))
|
||||
self.add(empty, tags=[], mark_autotagged=True)
|
||||
self.assertEqual(
|
||||
self.marked(),
|
||||
sorted(
|
||||
[self.byte_id(loc), self.byte_id(other, DIGEST_B), self.byte_id(empty)]
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.autotag_state(
|
||||
{loc: (SIZE, MTIME), other: (SIZE, MTIME), empty: (SIZE, MTIME)}
|
||||
),
|
||||
{loc: True, other: True, empty: True},
|
||||
)
|
||||
self.schema_clean()
|
||||
|
||||
def test_empty_union_without_mark_is_refused(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.add(self.locator("a.png"), tags=[])
|
||||
self.assertEqual(self.counts()["media_annotation_files"], 0)
|
||||
|
||||
def test_tag_limit_union_writes_no_tag_and_no_mark(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=[f"t{i}" for i in range(100)])
|
||||
before = self.rows(*ANNOTATION_TABLES, "tags")
|
||||
with self.assertRaises(AnnotationCommandError) as ctx:
|
||||
self.add(loc, tags=["one more"], mark_autotagged=True)
|
||||
self.assertEqual(ctx.exception.code, "tag-limit")
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES, "tags"), before)
|
||||
self.assertEqual(self.marked(), [])
|
||||
|
||||
def test_replace_marks_on_saved_and_replayed_receipts_not_on_conflict(self):
|
||||
loc = self.locator("a.png")
|
||||
saved = self.replace(loc, tags=["a"], mark_autotagged=True)
|
||||
self.assertFalse(saved["replayed"])
|
||||
self.assertEqual(self.marked(), [self.byte_id(loc)])
|
||||
other = self.locator("b.png")
|
||||
self.replace(other, DIGEST_B, ["x"])
|
||||
replayed = self.replace(
|
||||
other, DIGEST_B, ["x"], revision=1, mark_autotagged=True
|
||||
)
|
||||
self.assertTrue(replayed["replayed"])
|
||||
third = self.locator("c.png")
|
||||
self.replace(third, tags=["y"])
|
||||
with self.assertRaises(AnnotationCommandError):
|
||||
self.replace(third, tags=["z"], revision=5, mark_autotagged=True)
|
||||
self.assertEqual(
|
||||
self.marked(), sorted([self.byte_id(loc), self.byte_id(other, DIGEST_B)])
|
||||
)
|
||||
|
||||
def test_mark_follows_the_bytes_not_the_file(self):
|
||||
loc = self.locator("a.png")
|
||||
self.add(loc, tags=["bird"], mark_autotagged=True)
|
||||
# The file's bytes changed: the new record is current and unmarked.
|
||||
self.replace(loc, DIGEST_B, ["hand"], mtime_ns=MTIME + 1)
|
||||
self.assertEqual(self.db.autotag_state({loc: (SIZE, MTIME + 1)}), {loc: False})
|
||||
self.assertEqual(self.db.autotag_state({loc: (SIZE, MTIME)}), {loc: True})
|
||||
# The review's estimate counts any marked record of the locator.
|
||||
self.assertEqual(self.db.autotagged_locators(), {loc})
|
||||
missing = self.locator("none.png")
|
||||
self.assertEqual(self.db.autotag_state({missing: (1, 1)}), {missing: False})
|
||||
|
||||
def test_state_and_tags_choose_the_same_current_record(self):
|
||||
loc = self.locator("a.png")
|
||||
# Two records with the same verified fingerprint: the newest wins.
|
||||
self.replace(loc, DIGEST_A, ["old bytes"])
|
||||
with patch.object(
|
||||
annotations_module, "utc_now_iso", return_value="2099-01-01T00:00:00"
|
||||
):
|
||||
self.add(loc, DIGEST_B, ["new bytes"], mark_autotagged=True)
|
||||
stats = {loc: (SIZE, MTIME)}
|
||||
self.assertEqual(self.db.current_image_tags(stats), {loc: ["new bytes"]})
|
||||
self.assertEqual(self.db.autotag_state(stats), {loc: True})
|
||||
with patch.object(
|
||||
annotations_module, "utc_now_iso", return_value="2199-01-01T00:00:00"
|
||||
):
|
||||
self.add(loc, DIGEST_A, ["older bytes again"])
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags(stats), {loc: ["old bytes", "older bytes again"]}
|
||||
)
|
||||
self.assertEqual(self.db.autotag_state(stats), {loc: False})
|
||||
|
||||
def test_marks_never_touch_prompt_rows(self):
|
||||
conn = self.db.model.get_connection()
|
||||
prompt_id = self.db.save_prompt("a beach", tags=["beach"])
|
||||
before = [
|
||||
[tuple(row) for row in conn.execute(f"SELECT * FROM {table} ORDER BY 1")]
|
||||
for table in ("prompts", "prompt_tags")
|
||||
]
|
||||
self.add(self.locator("a.png"), tags=["bird"], mark_autotagged=True)
|
||||
self.replace(self.locator("b.png"), DIGEST_B, ["waves"], mark_autotagged=True)
|
||||
after = [
|
||||
[tuple(row) for row in conn.execute(f"SELECT * FROM {table} ORDER BY 1")]
|
||||
for table in ("prompts", "prompt_tags")
|
||||
]
|
||||
self.assertIsNotNone(prompt_id)
|
||||
self.assertEqual(after, before)
|
||||
|
||||
|
||||
class AnnotationIsolationTests(AnnotationTestCase):
|
||||
|
||||
+21
-13
@@ -846,14 +846,6 @@ class TestSettingsReadMonitorThroughModule(AdminAPITestCase):
|
||||
self.assertEqual(data["settings"]["gallery_root_paths"], ["output"])
|
||||
|
||||
|
||||
def _wired_prompt(text):
|
||||
"""A sampler-less graph whose one encoder feeds a guider (#252)."""
|
||||
return {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": text}},
|
||||
"2": {"class_type": "BasicGuider", "inputs": {"conditioning": ["1", 0]}},
|
||||
}
|
||||
|
||||
|
||||
def _raise(*args, **kwargs):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
@@ -2194,7 +2186,11 @@ class TestOutputScanInternals(AdminAPITestCase):
|
||||
|
||||
def test_ingest_outcomes(self):
|
||||
f = Path("gen.png")
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a cat"))}
|
||||
meta = {
|
||||
"prompt": json.dumps(
|
||||
{"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat"}}}
|
||||
)
|
||||
}
|
||||
self.assertIsNone(self.api._ingest_scanned_file(f, {}))
|
||||
self.assertIsNone(self.api._ingest_scanned_file(f, {"prompt": "{}"}))
|
||||
self.assertEqual(
|
||||
@@ -2206,19 +2202,31 @@ class TestOutputScanInternals(AdminAPITestCase):
|
||||
|
||||
def test_ingest_link_failures_are_tolerated(self):
|
||||
f = Path("gen.png")
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a dog"))}
|
||||
meta = {
|
||||
"prompt": json.dumps(
|
||||
{"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a dog"}}}
|
||||
)
|
||||
}
|
||||
self.api.db.link_image_to_prompt = _raise
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "added")
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "found")
|
||||
|
||||
def test_ingest_save_failure_counts_as_found(self):
|
||||
f = Path("gen.png")
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a bird"))}
|
||||
meta = {
|
||||
"prompt": json.dumps(
|
||||
{"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a bird"}}}
|
||||
)
|
||||
}
|
||||
self.api.db.save_prompt = lambda *a, **k: None
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "found")
|
||||
|
||||
def test_batch_counts_and_per_file_errors(self):
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a fish"))}
|
||||
meta = {
|
||||
"prompt": json.dumps(
|
||||
{"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a fish"}}}
|
||||
)
|
||||
}
|
||||
self.api.db.get_prompt_by_hash = _raise
|
||||
counts = self.api._ingest_scan_batch_sync(
|
||||
[(Path("a.png"), meta), (Path("b.png"), {})]
|
||||
@@ -2907,7 +2915,7 @@ class TestScanReadsVideos(AdminAPITestCase):
|
||||
shutil.which("ffmpeg") and shutil.which("ffprobe"), "ffmpeg not installed"
|
||||
)
|
||||
async def test_scan_adds_prompts_found_in_videos(self):
|
||||
graph = _wired_prompt("a cat")
|
||||
graph = {"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat"}}}
|
||||
_write_mp4(
|
||||
self.output_dir / "clip.mp4", json.dumps({"prompt": json.dumps(graph)})
|
||||
)
|
||||
|
||||
@@ -363,20 +363,23 @@ class TestModelGate(SuggestTestCase):
|
||||
prompt_id = self._save_prompt("batch prompt")
|
||||
self.api.db.link_image_to_prompt(prompt_id, str(self.output_dir / "sub/a.png"))
|
||||
started, proceed = threading.Event(), threading.Event()
|
||||
# The legacy batch is the autotag job (#379): it suggests per image.
|
||||
self.tagger.block = (started, proceed)
|
||||
|
||||
def slow_generate(image_path, **kwargs):
|
||||
started.set()
|
||||
proceed.wait(WAIT)
|
||||
return ["batch"]
|
||||
|
||||
self.tagger.generate_tags = slow_generate
|
||||
batch = asyncio.ensure_future(
|
||||
self.client.post("/prompt_manager/autotag/start?model_type=wd14-vit")
|
||||
)
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, started.wait, WAIT)
|
||||
calls = len(self.tagger.calls)
|
||||
status, body = await self.suggest(self.suggest_body(observed, "gguf"))
|
||||
self.assertEqual((status, body["code"]), (409, "model-busy"))
|
||||
self.assertEqual(len(self.tagger.calls), calls)
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
self.assertEqual(self.tagger.loaded, "wd14-vit") # not swapped
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
response = await batch
|
||||
await response.read()
|
||||
|
||||
+574
-78
@@ -5,7 +5,6 @@ no network request is made.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
@@ -35,7 +34,6 @@ from aiohttp.test_utils import AioHTTPTestCase
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
from py.api import PromptManagerAPI
|
||||
from py.api.autotag_routes import MODEL_BUSY_ERROR
|
||||
from py.api.jobs import JobBusy
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
@@ -72,7 +70,6 @@ class FakeAutotagService:
|
||||
self.status_calls = 0
|
||||
self.status_threads = []
|
||||
self.status_paths = {}
|
||||
self.suggest_calls = 0
|
||||
|
||||
def get_models_status(self):
|
||||
self.status_calls += 1
|
||||
@@ -118,25 +115,6 @@ class FakeAutotagService:
|
||||
raise self.generate_error
|
||||
return list(self.tags)
|
||||
|
||||
def generate_tag_suggestions(
|
||||
self,
|
||||
image_path,
|
||||
*,
|
||||
limit,
|
||||
prompt=None,
|
||||
general_threshold=None,
|
||||
character_threshold=None,
|
||||
):
|
||||
"""The suggestion entry point (#379): the fixed tags, at most ``limit``."""
|
||||
self.suggest_calls += 1
|
||||
tags = self.generate_tags(
|
||||
image_path,
|
||||
prompt=prompt,
|
||||
general_threshold=general_threshold,
|
||||
character_threshold=character_threshold,
|
||||
)
|
||||
return [(name, None) for name in tags[:limit]]
|
||||
|
||||
|
||||
class AutotagAPITestCase(AioHTTPTestCase):
|
||||
"""Stands up the PromptManager routes with a temp DB, temp output dir and a
|
||||
@@ -197,17 +175,8 @@ class AutotagAPITestCase(AioHTTPTestCase):
|
||||
)
|
||||
|
||||
def _make_image(self, directory, name="img.png"):
|
||||
"""A small, valid image (the legacy review captures and probes it)."""
|
||||
from PIL import Image
|
||||
|
||||
path = directory / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shade = sum(name.encode()) % 256
|
||||
fmt = {".jpg": "JPEG", ".jpeg": "JPEG", ".webp": "WEBP"}.get(
|
||||
path.suffix.lower(), "PNG"
|
||||
)
|
||||
with Image.new("RGB", (4, 4), (shade, 64, 128)) as image:
|
||||
image.save(path, fmt)
|
||||
path.write_bytes(b"\x89PNG\r\n\x1a\n")
|
||||
return path
|
||||
|
||||
def _link_image(self, prompt_id, path):
|
||||
@@ -247,9 +216,10 @@ class TestAutotagSingleContainment(AutotagAPITestCase):
|
||||
self.assertEqual(data["tags"], ["1girl", "smile"])
|
||||
self.assertEqual(data["prompt_id"], prompt_id)
|
||||
self.assertEqual(len(self.service.generate_calls), 1)
|
||||
# #379: the model reads a private capture of the image, never the file.
|
||||
self.assertNotIn(str(self.output_dir), self.service.generate_calls[0][0])
|
||||
self.assertEqual(data["sha256"], hashlib.sha256(image.read_bytes()).hexdigest())
|
||||
self.assertEqual(
|
||||
os.path.realpath(self.service.generate_calls[0][0]),
|
||||
os.path.realpath(str(image)),
|
||||
)
|
||||
|
||||
async def test_path_inside_output_dir_is_tagged(self):
|
||||
image = self._make_image(self.output_dir)
|
||||
@@ -479,6 +449,238 @@ class TestAutotagSideEffectRoutesArePostOnly(AutotagAPITestCase):
|
||||
self.assertTrue((await resp.json())["success"])
|
||||
|
||||
|
||||
class TestAutotagBatch(AutotagAPITestCase):
|
||||
"""Batch tagging over the DB image list with SSE progress reporting."""
|
||||
|
||||
def _seed(self):
|
||||
"""Three prompts: A (one image), B (two images), C (missing file)."""
|
||||
a = self._save_prompt("prompt a")
|
||||
b = self._save_prompt("prompt b")
|
||||
c = self._save_prompt("prompt c")
|
||||
self._link_image(a, self._make_image(self.output_dir, "a1.png"))
|
||||
self._link_image(b, self._make_image(self.output_dir, "b1.png"))
|
||||
self._link_image(b, self._make_image(self.output_dir, "b2.png"))
|
||||
self._link_image(c, self.output_dir / "missing.png")
|
||||
return a, b, c
|
||||
|
||||
async def _start(self, query="model_type=gguf", body=None):
|
||||
if body is None:
|
||||
resp = await self.client.request(
|
||||
"POST", f"/prompt_manager/autotag/start?{query}"
|
||||
)
|
||||
else:
|
||||
resp = await self._post_json(f"/prompt_manager/autotag/start?{query}", body)
|
||||
return resp, self._sse_events(await resp.text())
|
||||
|
||||
async def test_batch_tags_each_prompt_once_and_reports_counts(self):
|
||||
a, b, c = self._seed()
|
||||
|
||||
resp, events = await self._start()
|
||||
|
||||
self.assertEqual(resp.status, 200)
|
||||
done = events[-1]
|
||||
self.assertEqual(done["type"], "complete")
|
||||
self.assertEqual(done["processed"], 2)
|
||||
self.assertEqual(done["tagged"], 2)
|
||||
self.assertEqual(done["skipped"], 2)
|
||||
self.assertEqual(done["errors"], 0)
|
||||
self.assertEqual(len(self.service.generate_calls), 2)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(a)["tags"], ["1girl", "smile"])
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(b)["tags"], ["1girl", "smile"])
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(c)["tags"], [])
|
||||
progress = [e for e in events if e["type"] == "progress"]
|
||||
self.assertTrue(all(0 <= e["progress"] <= 100 for e in progress))
|
||||
self.assertTrue(any("processed" in e for e in progress))
|
||||
|
||||
async def test_rows_outside_output_dirs_are_skipped_and_counted(self):
|
||||
inside = self._save_prompt("inside")
|
||||
outside = self._save_prompt("outside")
|
||||
self._link_image(inside, self._make_image(self.output_dir, "in.png"))
|
||||
secret = self._make_image(self.secret_dir, "password.png")
|
||||
self._link_image(outside, secret)
|
||||
|
||||
_, events = await self._start()
|
||||
|
||||
done = events[-1]
|
||||
self.assertEqual(done["type"], "complete")
|
||||
self.assertEqual(done["processed"], 1)
|
||||
self.assertEqual(done["tagged"], 1)
|
||||
self.assertEqual(done["skipped"], 1)
|
||||
self.assertEqual(done["skipped_outside_roots"], 1)
|
||||
self.assertEqual(len(self.service.generate_calls), 1)
|
||||
self.assertEqual(
|
||||
os.path.realpath(self.service.generate_calls[0][0]),
|
||||
os.path.realpath(str(self.output_dir / "in.png")),
|
||||
)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(outside)["tags"], [])
|
||||
progress = [e for e in events if e["type"] == "progress" and "skipped" in e]
|
||||
self.assertTrue(progress)
|
||||
self.assertTrue(all("skipped_outside_roots" in e for e in progress))
|
||||
body = json.dumps(events)
|
||||
self.assertNotIn(str(self.secret_dir), body)
|
||||
|
||||
async def test_second_start_while_running_is_409(self):
|
||||
self._seed()
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_generate(image_path, **kwargs):
|
||||
started.set()
|
||||
release.wait(5)
|
||||
return ["slow"]
|
||||
|
||||
self.service.generate_tags = slow_generate
|
||||
first = asyncio.ensure_future(self._start())
|
||||
try:
|
||||
await asyncio.get_event_loop().run_in_executor(None, started.wait, 5)
|
||||
self.assertTrue(started.is_set())
|
||||
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?model_type=gguf"
|
||||
)
|
||||
|
||||
self.assertEqual(resp.status, 409)
|
||||
self.assertEqual(
|
||||
await resp.json(),
|
||||
{"success": False, "error": "Auto-tagging is already running"},
|
||||
)
|
||||
finally:
|
||||
release.set()
|
||||
resp1, events = await first
|
||||
self.assertEqual(resp1.status, 200)
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
|
||||
async def test_lock_is_released_after_a_run(self):
|
||||
self._seed()
|
||||
|
||||
resp1, _ = await self._start()
|
||||
resp2, events = await self._start()
|
||||
|
||||
self.assertEqual(resp1.status, 200)
|
||||
self.assertEqual(resp2.status, 200)
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
|
||||
async def test_lock_is_released_when_the_run_fails(self):
|
||||
self._seed()
|
||||
self.service.load_error = RuntimeError("no model")
|
||||
|
||||
_, events = await self._start()
|
||||
self.assertEqual(events[-1]["type"], "error")
|
||||
|
||||
self.service.load_error = None
|
||||
resp, events = await self._start()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
|
||||
async def test_already_tagged_prompts_are_skipped_unless_asked(self):
|
||||
p = self._save_prompt("tagged", tags=["cat"])
|
||||
self._link_image(p, self._make_image(self.output_dir, "t.png"))
|
||||
|
||||
_, events = await self._start("model_type=gguf&skip_tagged=true")
|
||||
self.assertEqual(events[-1]["skipped"], 1)
|
||||
self.assertEqual(self.service.generate_calls, [])
|
||||
|
||||
_, events = await self._start("model_type=gguf&skip_tagged=false")
|
||||
self.assertEqual(events[-1]["tagged"], 1)
|
||||
self.assertEqual(
|
||||
self.api.db.get_prompt_by_id(p)["tags"], ["cat", "1girl", "smile"]
|
||||
)
|
||||
|
||||
async def test_prompt_already_holding_the_tags_counts_as_skipped(self):
|
||||
p = self._save_prompt("same", tags=["1girl", "smile"])
|
||||
self._link_image(p, self._make_image(self.output_dir, "s.png"))
|
||||
|
||||
_, events = await self._start("model_type=gguf&skip_tagged=false")
|
||||
|
||||
self.assertEqual(events[-1]["skipped"], 1)
|
||||
self.assertEqual(events[-1]["tagged"], 0)
|
||||
|
||||
async def test_generation_errors_are_counted_and_do_not_abort(self):
|
||||
self._seed()
|
||||
self.service.generate_error = RuntimeError("boom")
|
||||
|
||||
_, events = await self._start()
|
||||
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
# a1, b1 and b2 each fail: a failed prompt is retried on its next image
|
||||
self.assertEqual(events[-1]["errors"], 3)
|
||||
self.assertEqual(events[-1]["tagged"], 0)
|
||||
|
||||
async def test_json_body_parameters_are_accepted(self):
|
||||
self._seed()
|
||||
|
||||
_, events = await self._start(
|
||||
"", {"model_type": "wd14-vit", "general_threshold": 0.5, "prompt": "p"}
|
||||
)
|
||||
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
self.assertEqual(self.service.load_calls, [("wd14-vit", True)])
|
||||
self.assertEqual(self.service.generate_calls[0][2], 0.5)
|
||||
self.assertEqual(self.service.custom_prompt, "p")
|
||||
|
||||
async def test_keep_in_memory_false_unloads_after_batch(self):
|
||||
self._seed()
|
||||
|
||||
_, events = await self._start("model_type=gguf&keep_in_memory=false")
|
||||
|
||||
self.assertEqual(events[-1]["model_loaded"], False)
|
||||
self.assertEqual(self.service.unload_calls, 1)
|
||||
|
||||
async def test_model_not_downloaded_is_an_error_event(self):
|
||||
self.service.downloaded = False
|
||||
|
||||
_, events = await self._start()
|
||||
|
||||
self.assertEqual(events[-1]["type"], "error")
|
||||
self.assertEqual(self.service.load_calls, [])
|
||||
|
||||
async def test_load_failure_is_an_error_event_without_details(self):
|
||||
self.service.load_error = RuntimeError("/srv/models/secret.gguf is corrupt")
|
||||
|
||||
_, events = await self._start()
|
||||
|
||||
self.assertEqual(events[-1]["type"], "error")
|
||||
self.assertNotIn("/srv/models", events[-1]["message"])
|
||||
|
||||
async def test_non_numeric_threshold_is_400(self):
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?general_threshold=abc"
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
async def test_invalid_json_body_is_400(self):
|
||||
resp = await self.client.request(
|
||||
"POST",
|
||||
"/prompt_manager/autotag/start",
|
||||
data="{not json",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
async def test_client_disconnect_stops_the_batch_and_unloads(self):
|
||||
self._seed()
|
||||
real_write = web.StreamResponse.write
|
||||
state = {"writes": 0}
|
||||
|
||||
async def flaky_write(response, data):
|
||||
state["writes"] += 1
|
||||
if b"Processing " in data:
|
||||
raise ConnectionResetError("Cannot write to closing transport")
|
||||
return await real_write(response, data)
|
||||
|
||||
with patch.object(web.StreamResponse, "write", flaky_write):
|
||||
try:
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?keep_in_memory=false"
|
||||
)
|
||||
await resp.read()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.assertEqual(len(self.service.generate_calls), 1)
|
||||
self.assertEqual(self.service.unload_calls, 1)
|
||||
|
||||
|
||||
class TestAutotagModelsAndDownloadErrors(AutotagAPITestCase):
|
||||
|
||||
async def test_models_error_is_500(self):
|
||||
@@ -632,15 +834,7 @@ class TestAutotagSingleEdgeCases(AutotagAPITestCase):
|
||||
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(self.service.load_calls, [("wd14-vit", True)])
|
||||
# #379: the request's prompt is for this call only.
|
||||
self.assertEqual(self.service.custom_prompt, "")
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/single",
|
||||
{"path": str(image), "model_type": "gguf", "prompt": "terse"},
|
||||
)
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(self.service.generate_calls[-1][1], "terse")
|
||||
self.assertEqual(self.service.custom_prompt, "")
|
||||
self.assertEqual(self.service.custom_prompt, "terse")
|
||||
|
||||
async def test_image_id_without_stored_path_is_404(self):
|
||||
prompt_id = self._save_prompt()
|
||||
@@ -656,6 +850,64 @@ class TestAutotagSingleEdgeCases(AutotagAPITestCase):
|
||||
self.assertEqual(resp.status, 404)
|
||||
|
||||
|
||||
class TestApplyAutotag(AutotagAPITestCase):
|
||||
|
||||
async def test_missing_prompt_id_is_400(self):
|
||||
resp = await self._post_json("/prompt_manager/autotag/apply", {"tags": ["a"]})
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
async def test_no_tags_is_a_noop(self):
|
||||
resp = await self._post_json("/prompt_manager/autotag/apply", {"prompt_id": 1})
|
||||
data = await resp.json()
|
||||
self.assertTrue(data["success"])
|
||||
self.assertEqual(data["message"], "No tags to apply")
|
||||
|
||||
async def test_unknown_prompt_is_404(self):
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply", {"prompt_id": 999, "tags": ["a"]}
|
||||
)
|
||||
self.assertEqual(resp.status, 404)
|
||||
|
||||
async def test_tags_are_merged_without_duplicates(self):
|
||||
prompt_id = self._save_prompt(tags=["cat"])
|
||||
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply",
|
||||
{"prompt_id": prompt_id, "tags": ["cat", "dog"]},
|
||||
)
|
||||
|
||||
data = await resp.json()
|
||||
self.assertEqual(data["added_tags"], ["dog"])
|
||||
self.assertEqual(data["total_tags"], 2)
|
||||
self.assertEqual(
|
||||
self.api.db.get_prompt_by_id(prompt_id)["tags"], ["cat", "dog"]
|
||||
)
|
||||
|
||||
async def test_string_tags_from_db_are_split(self):
|
||||
prompt_id = self._save_prompt()
|
||||
with patch.object(
|
||||
self.api.db,
|
||||
"get_prompt_by_id",
|
||||
return_value={"id": prompt_id, "tags": "cat, dog"},
|
||||
):
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply",
|
||||
{"prompt_id": prompt_id, "tags": ["dog", "owl"]},
|
||||
)
|
||||
data = await resp.json()
|
||||
self.assertEqual(data["added_tags"], ["owl"])
|
||||
self.assertEqual(data["total_tags"], 3)
|
||||
|
||||
async def test_db_error_is_500(self):
|
||||
with patch.object(
|
||||
self.api.db, "get_prompt_by_id", side_effect=RuntimeError("db")
|
||||
):
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply", {"prompt_id": 1, "tags": ["a"]}
|
||||
)
|
||||
self.assertEqual(resp.status, 500)
|
||||
|
||||
|
||||
class TestAutotagErrorBodiesHideServerPaths(AutotagAPITestCase):
|
||||
"""Every autotag handler reports exceptions through the shared public helper."""
|
||||
|
||||
@@ -680,12 +932,9 @@ class TestAutotagErrorBodiesHideServerPaths(AutotagAPITestCase):
|
||||
await self._assert_hidden(resp)
|
||||
|
||||
async def test_apply_error(self):
|
||||
image = self._make_image(self.output_dir)
|
||||
digest = hashlib.sha256(image.read_bytes()).hexdigest()
|
||||
with patch.object(self.api.db, "add_image_tags", side_effect=self._locked()):
|
||||
with patch.object(self.api.db, "get_prompt_by_id", side_effect=self._locked()):
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply",
|
||||
{"path": str(image), "sha256": digest, "tags": ["a"]},
|
||||
"/prompt_manager/autotag/apply", {"prompt_id": 1, "tags": ["a"]}
|
||||
)
|
||||
await self._assert_hidden(resp)
|
||||
|
||||
@@ -759,13 +1008,10 @@ class TestLegacyRoutesShareTheModelGate(AutotagAPITestCase):
|
||||
"POST", "/prompt_manager/autotag/download/gguf"
|
||||
)
|
||||
)
|
||||
# The legacy batch is read as a stream: one error frame (#379).
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?model_type=gguf"
|
||||
)
|
||||
self.assertEqual(
|
||||
self._sse_events(await resp.text()),
|
||||
[{"type": "error", "status": "error", "message": MODEL_BUSY_ERROR}],
|
||||
await self._busy(
|
||||
await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?model_type=gguf"
|
||||
)
|
||||
)
|
||||
await self._busy(
|
||||
await self.client.request("POST", "/prompt_manager/autotag/unload")
|
||||
@@ -780,7 +1026,7 @@ class TestLegacyRoutesShareTheModelGate(AutotagAPITestCase):
|
||||
self.assertEqual(self.service.unload_calls, 0)
|
||||
self.assertEqual(self.service.get_loaded_model_type(), "wd14-vit")
|
||||
|
||||
# A refused start left no job behind: the next start runs.
|
||||
# A refused start released the batch lock: the next start runs.
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?model_type=gguf"
|
||||
)
|
||||
@@ -992,7 +1238,7 @@ if __name__ == "__main__":
|
||||
|
||||
|
||||
class TestGalleryReviewContract(AutotagAPITestCase):
|
||||
async def test_root_relative_gallery_targets_generate_and_apply_to_their_image(
|
||||
async def test_root_relative_gallery_targets_generate_and_apply_to_their_prompt(
|
||||
self,
|
||||
):
|
||||
second = self.tmp_root / "second-output"
|
||||
@@ -1009,22 +1255,16 @@ class TestGalleryReviewContract(AutotagAPITestCase):
|
||||
self.assertEqual(response.status, 200)
|
||||
data = await response.json()
|
||||
self.assertEqual(data["prompt_id"], pid)
|
||||
self.assertEqual(
|
||||
self.service.generate_calls[-1][0], str((root / "same.png").resolve())
|
||||
)
|
||||
response = await self._post_json(
|
||||
"/prompt_manager/autotag/apply",
|
||||
{
|
||||
"path": "same.png",
|
||||
"root_index": index,
|
||||
"sha256": data["sha256"],
|
||||
"tags": data["tags"],
|
||||
},
|
||||
{"prompt_id": data["prompt_id"], "tags": data["tags"]},
|
||||
)
|
||||
self.assertEqual(response.status, 200)
|
||||
# The image's own tags; the prompt is never written (#379).
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(pid)["tags"], [])
|
||||
locator = os.path.normcase(os.path.realpath(str(root / "same.png")))
|
||||
self.assertEqual(
|
||||
self.api.db.read_image_tags(locator, data["sha256"])["tags"],
|
||||
data["tags"],
|
||||
set(self.api.db.get_prompt_by_id(pid)["tags"]), set(data["tags"])
|
||||
)
|
||||
|
||||
async def test_gallery_unlinked_image_returns_no_prompt_id(self):
|
||||
@@ -1653,15 +1893,13 @@ class TestTagOutputImagesPreview(BatchJobTestCase):
|
||||
self.assertEqual(self.service.generate_calls, [])
|
||||
self.assertEqual(self.service.load_calls, [])
|
||||
|
||||
async def test_preview_counts_listed_still_images_not_prompts(self):
|
||||
async def test_preview_counts_skip_and_merge(self):
|
||||
self._seed()
|
||||
skip = await (await self._preview(existing="skip")).json()
|
||||
# t, f1, f2 and m are listed; gone.png is missing, o.png is outside.
|
||||
self.assertEqual(
|
||||
{k: skip[k] for k in ("images", "already_tagged", "to_tag")},
|
||||
{"images": 4, "already_tagged": 0, "to_tag": 4},
|
||||
{k: skip[k] for k in ("images", "prompts", "already_tagged", "to_tag")},
|
||||
{"images": 6, "prompts": 5, "already_tagged": 1, "to_tag": 4},
|
||||
)
|
||||
self.assertNotIn("prompts", skip)
|
||||
self.assertEqual(
|
||||
skip["model"],
|
||||
{
|
||||
@@ -1678,7 +1916,7 @@ class TestTagOutputImagesPreview(BatchJobTestCase):
|
||||
set(skip["disk"]), {"free_bytes", "required_bytes", "sufficient"}
|
||||
)
|
||||
merge = await (await self._preview(existing="merge")).json()
|
||||
self.assertEqual(merge["to_tag"], 4)
|
||||
self.assertEqual(merge["to_tag"], 5)
|
||||
leaked = [s for s in _strings(skip) if _absolute(s)]
|
||||
self.assertEqual(leaked, [])
|
||||
|
||||
@@ -1686,7 +1924,7 @@ class TestTagOutputImagesPreview(BatchJobTestCase):
|
||||
self._seed()
|
||||
data = await (await self._preview(existing="replace_after_review")).json()
|
||||
self.assertTrue(data["review_required"])
|
||||
self.assertEqual(data["images"], 4)
|
||||
self.assertEqual(data["prompts"], 5)
|
||||
|
||||
async def test_preview_refuses_bad_bodies(self):
|
||||
for body in (
|
||||
@@ -1741,6 +1979,119 @@ class TestTagOutputImagesJob(BatchJobTestCase):
|
||||
"WD14 ViT is not downloaded. Download it in System › Auto-tagging.",
|
||||
)
|
||||
|
||||
async def test_job_writes_what_legacy_writes(self):
|
||||
ids = self._seed()
|
||||
await self._run_job(existing="skip")
|
||||
native = {name: sorted(self._tags(pid)) for name, pid in ids.items()}
|
||||
# The same library through the legacy batch, in a fresh database.
|
||||
for name, pid in ids.items():
|
||||
self.api.db.update_prompt_metadata(
|
||||
pid,
|
||||
tags=(
|
||||
["existing"]
|
||||
if name == "tagged"
|
||||
else (["auto-scanned"] if name == "marked" else [])
|
||||
),
|
||||
)
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/autotag/start?model_type=wd14-vit&skip_tagged=true"
|
||||
)
|
||||
self.assertEqual(self._sse_events(await resp.text())[-1]["type"], "complete")
|
||||
legacy = {name: sorted(self._tags(pid)) for name, pid in ids.items()}
|
||||
self.assertEqual(native, legacy)
|
||||
self.assertEqual(native["tagged"], ["existing"])
|
||||
self.assertEqual(native["fresh"], ["1girl", "smile"])
|
||||
|
||||
async def test_job_outcomes_follow_the_mapping(self):
|
||||
ids = self._seed()
|
||||
self.api.db.update_prompt_metadata(
|
||||
ids["marked"], tags=["auto-scanned", "1girl", "smile"]
|
||||
)
|
||||
|
||||
def details(job):
|
||||
return {
|
||||
group: sorted(row["detail"] for row in rows)
|
||||
for group, rows in job.outcomes.items()
|
||||
}
|
||||
|
||||
merged = await self._run_job(existing="merge")
|
||||
self.assertEqual(merged.state, "finished")
|
||||
self.assertEqual(
|
||||
details(merged),
|
||||
{
|
||||
"tagged": ["Added 2 tags", "Added 2 tags"],
|
||||
"unchanged": ["No new tags"],
|
||||
"skipped": [
|
||||
"File not found",
|
||||
"Outside the output folders",
|
||||
"Prompt handled by another image",
|
||||
],
|
||||
"errors": [],
|
||||
},
|
||||
)
|
||||
self.assertEqual(merged.status_line, "Tagged 2 prompts")
|
||||
|
||||
skipped = await self._run_job(existing="skip")
|
||||
self.assertEqual(details(skipped)["tagged"], [])
|
||||
self.assertEqual(details(skipped)["skipped"].count("Prompt already tagged"), 3)
|
||||
self.assertEqual(skipped.status_line, "Tagged 0 prompts")
|
||||
|
||||
async def test_tagging_errors_are_fixed_strings(self):
|
||||
self._seed()
|
||||
self.service.generate_error = RuntimeError("/srv/secret/model exploded")
|
||||
job = await self._run_job(existing="merge")
|
||||
self.assertTrue(job.outcomes["errors"])
|
||||
self.assertEqual(
|
||||
{r["detail"] for r in job.outcomes["errors"]}, {"Tagging failed"}
|
||||
)
|
||||
self.assertNotIn("/srv/secret", json.dumps(job.report()))
|
||||
|
||||
async def test_job_progress_is_determinate(self):
|
||||
self._seed()
|
||||
seen = []
|
||||
original = self.service.generate_tags
|
||||
|
||||
def generate(*args, **kwargs):
|
||||
job = self.api._jobs.active("autotag")
|
||||
seen.append((job.processed, job.total))
|
||||
return original(*args, **kwargs)
|
||||
|
||||
self.service.generate_tags = generate
|
||||
job = await self._run_job(existing="merge")
|
||||
self.assertEqual((job.processed, job.total, job.progress), (6, 6, 100))
|
||||
self.assertTrue(all(total == 6 for _p, total in seen[1:]), seen)
|
||||
self.assertTrue(job.cancellable)
|
||||
|
||||
async def test_job_cancel_stops_before_the_next_image_and_keeps_writes(self):
|
||||
ids = self._seed()
|
||||
started = threading.Event()
|
||||
proceed = threading.Event()
|
||||
calls = []
|
||||
|
||||
def slow(image_path, **kwargs):
|
||||
calls.append(image_path)
|
||||
started.set()
|
||||
proceed.wait(30)
|
||||
return ["slow"]
|
||||
|
||||
self.service.generate_tags = slow
|
||||
resp = await self._start_job(existing="merge")
|
||||
job = self.api._jobs.get((await resp.json())["job"]["id"])
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, started.wait, 30)
|
||||
cancel = await self.client.post(f"/prompt_manager/jobs/{job.id}/cancel")
|
||||
self.assertTrue((await cancel.json())["cancellation_requested"])
|
||||
finally:
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), 30)
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.status_line, "Cancelled. Tags already added were kept.")
|
||||
self.assertEqual(len(calls), 1)
|
||||
tagged = [n for n, pid in ids.items() if "slow" in self._tags(pid)]
|
||||
self.assertEqual(len(tagged), 1)
|
||||
self.assertIsNone(model_gate.holder())
|
||||
self.assertEqual(self.service.unload_calls, 0)
|
||||
|
||||
async def test_job_refused_while_legacy_batch_holds(self):
|
||||
async with self.api._jobs.hold(
|
||||
"autotag", "Tag output images (legacy page)", admission="records"
|
||||
@@ -1752,6 +2103,41 @@ class TestTagOutputImagesJob(BatchJobTestCase):
|
||||
self.assertEqual(data["running"]["title"], "Tag output images (legacy page)")
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_legacy_refused_while_job_runs(self):
|
||||
self._seed()
|
||||
started, proceed = threading.Event(), threading.Event()
|
||||
|
||||
def slow(image_path, **kwargs):
|
||||
started.set()
|
||||
proceed.wait(30)
|
||||
return ["slow"]
|
||||
|
||||
self.service.generate_tags = slow
|
||||
resp = await self._start_job(existing="merge")
|
||||
job = self.api._jobs.get((await resp.json())["job"]["id"])
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, started.wait, 30)
|
||||
self.assertTrue(model_gate.holder().startswith("autotag-job:"))
|
||||
legacy = await self.client.post("/prompt_manager/autotag/start")
|
||||
# Both legacy pages read a stream: the refusal is one error frame
|
||||
# that names the running job.
|
||||
self.assertEqual(legacy.status, 200)
|
||||
self.assertEqual(
|
||||
self._sse_events(await legacy.text()),
|
||||
[
|
||||
{
|
||||
"type": "error",
|
||||
"status": "error",
|
||||
"message": "Tag output images already running",
|
||||
}
|
||||
],
|
||||
)
|
||||
unload = await self.client.post("/prompt_manager/autotag/unload")
|
||||
self.assertEqual((await unload.json())["operation"], "Tag output images")
|
||||
finally:
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), 30)
|
||||
|
||||
async def test_job_releases_gate_when_start_is_refused(self):
|
||||
class Running:
|
||||
id = "other"
|
||||
@@ -1799,7 +2185,7 @@ class TestTagOutputImagesJob(BatchJobTestCase):
|
||||
("database folder", "the safety backup", 10, 1),
|
||||
10,
|
||||
)
|
||||
with patch("py.api.image_tag_jobs.disk_plan", return_value=short):
|
||||
with patch("py.api.autotag_routes.disk_plan", return_value=short):
|
||||
resp = await self._start_job()
|
||||
data = await resp.json()
|
||||
self.assertEqual(resp.status, 507)
|
||||
@@ -1817,6 +2203,116 @@ class TestTagOutputImagesJob(BatchJobTestCase):
|
||||
inside = [row for row in rows if row["root_index"] == 0]
|
||||
self.assertTrue(inside)
|
||||
|
||||
async def test_native_batch_leaves_custom_prompt_unchanged(self):
|
||||
self._seed()
|
||||
self.service.custom_prompt = "legacy page prompt"
|
||||
await self._run_job(existing="merge", model_type="gguf")
|
||||
self.assertEqual(self.service.custom_prompt, "legacy page prompt")
|
||||
prompts = {call[1] for call in self.service.generate_calls}
|
||||
self.assertEqual(prompts, {"Tags please."})
|
||||
|
||||
|
||||
# The legacy batch's per-image frames over BatchJobTestCase's library, newest
|
||||
# first, with every throttle check passing (pinned from main's generator).
|
||||
GOLDEN_LEGACY_ROWS = [
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 24,
|
||||
"status": "Processing 1/6...",
|
||||
"processed": 1,
|
||||
"tagged": 1,
|
||||
"skipped": 0,
|
||||
"skipped_outside_roots": 0,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 38,
|
||||
"status": "Skipping 2/6 (outside allowed directories)...",
|
||||
"processed": 1,
|
||||
"tagged": 1,
|
||||
"skipped": 1,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 52,
|
||||
"status": "Skipping 3/6 (file missing)...",
|
||||
"processed": 1,
|
||||
"tagged": 1,
|
||||
"skipped": 2,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 66,
|
||||
"status": "Processing 4/6...",
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 2,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 80,
|
||||
"status": "Skipping 5/6 (prompt already processed)...",
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 3,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 95,
|
||||
"status": "Skipping 6/6 (already tagged)...",
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 4,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
class TestLegacyBatchFrames(BatchJobTestCase):
|
||||
|
||||
async def test_legacy_batch_events_unchanged(self):
|
||||
"""Golden frames of the legacy batch over the seeded library."""
|
||||
self._seed()
|
||||
ticks = iter(range(1000)) # every throttle check sees a second pass
|
||||
with patch("py.api.autotag_routes._monotonic", side_effect=lambda: next(ticks)):
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/autotag/start?model_type=wd14-vit&skip_tagged=true"
|
||||
)
|
||||
frames = self._sse_events(await resp.text()) # the stream runs here
|
||||
self.assertEqual(
|
||||
frames,
|
||||
[
|
||||
{"type": "progress", "progress": 0, "status": "Loading model..."},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 5,
|
||||
"status": "Model loaded. Fetching all images from database...",
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 10,
|
||||
"status": "Found 6 images. Processing...",
|
||||
},
|
||||
*GOLDEN_LEGACY_ROWS,
|
||||
{
|
||||
"type": "complete",
|
||||
"progress": 100,
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 4,
|
||||
"errors": 0,
|
||||
"skipped_outside_roots": 1,
|
||||
"status": "Complete",
|
||||
"model_status": "Model kept in memory",
|
||||
"model_loaded": True,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class TestTagOutputImagesEdges(BatchJobTestCase):
|
||||
|
||||
@@ -1846,7 +2342,7 @@ class TestTagOutputImagesEdges(BatchJobTestCase):
|
||||
async def test_unexpected_error_fails_the_job_with_a_reason(self):
|
||||
self._seed()
|
||||
with patch.object(
|
||||
self.api, "_autotag_prepass", side_effect=RuntimeError("/srv/x")
|
||||
self.api.db, "autotag_batch_candidates", side_effect=[[], RuntimeError("x")]
|
||||
):
|
||||
job = await self._run_job(existing="merge")
|
||||
self.assertEqual(job.state, "failed")
|
||||
|
||||
+92
-24
@@ -1,6 +1,5 @@
|
||||
"""Tests for the core API module (py/api/__init__.py): helpers and base routes."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
@@ -166,19 +165,8 @@ class TestGzipMiddleware(AioHTTPTestCase):
|
||||
await resp.write_eof()
|
||||
return resp
|
||||
|
||||
async def declared_length(request):
|
||||
# The log download (#329) declares its length before the middleware
|
||||
# compresses it.
|
||||
body = b"x" * 3000
|
||||
return web.Response(
|
||||
body=body,
|
||||
content_type="text/plain",
|
||||
headers={"Content-Length": str(len(body))},
|
||||
)
|
||||
|
||||
app = web.Application(middlewares=[api_module._gzip_middleware])
|
||||
app.router.add_get("/prompt_manager/big", big_json)
|
||||
app.router.add_get("/prompt_manager/declared", declared_length)
|
||||
app.router.add_get("/prompt_manager/small", small_json)
|
||||
app.router.add_get("/prompt_manager/encoded", already_encoded)
|
||||
app.router.add_get("/prompt_manager/binary", binary)
|
||||
@@ -205,18 +193,6 @@ class TestGzipMiddleware(AioHTTPTestCase):
|
||||
|
||||
self.assertIn(b'"data"', gzip.decompress(await resp.read()))
|
||||
|
||||
async def test_compressed_body_declares_its_own_length(self):
|
||||
# A stale Content-Length (the uncompressed size) makes a browser wait
|
||||
# for bytes that never come (#329: Download log hung on any log > 1 KB).
|
||||
resp = await self._get("/prompt_manager/declared")
|
||||
self.assertEqual(resp.headers.get("Content-Encoding"), "gzip")
|
||||
# Bounded: a stale length would otherwise wait for the missing bytes.
|
||||
body = await asyncio.wait_for(resp.read(), timeout=10)
|
||||
self.assertEqual(int(resp.headers["Content-Length"]), len(body))
|
||||
import gzip
|
||||
|
||||
self.assertEqual(gzip.decompress(body), b"x" * 3000)
|
||||
|
||||
async def test_small_body_not_compressed(self):
|
||||
resp = await self._get("/prompt_manager/small")
|
||||
self.assertIsNone(resp.headers.get("Content-Encoding"))
|
||||
@@ -835,6 +811,7 @@ class TestMetadataParsing(unittest.TestCase):
|
||||
self.assertEqual(parsed["positive_prompt"], "a cat")
|
||||
self.assertEqual(parsed["negative_prompt"], "dog")
|
||||
self.assertEqual(parsed["parameters"]["parameters"], meta["parameters"])
|
||||
self.assertEqual(self.api._extract_readable_prompt(parsed), "a cat")
|
||||
|
||||
def test_parse_comfyui_fields(self):
|
||||
meta = {
|
||||
@@ -855,6 +832,97 @@ class TestMetadataParsing(unittest.TestCase):
|
||||
self.assertEqual(parsed["prompt"], "plain text")
|
||||
self.assertEqual(parsed["workflow"], {"nodes": []})
|
||||
|
||||
def test_extract_readable_prompt_variants(self):
|
||||
extract = self.api._extract_readable_prompt
|
||||
self.assertEqual(extract({"prompt": "plain"}), "plain")
|
||||
self.assertEqual(extract({"prompt": ["a", "", "b"]}), "a b")
|
||||
self.assertEqual(extract({"prompt": 42}), "42")
|
||||
self.assertEqual(
|
||||
extract({"prompt": None, "parameters": {"positive": ["x", "y"]}}), "x y"
|
||||
)
|
||||
self.assertIsNone(extract({"prompt": {}, "parameters": {}}))
|
||||
|
||||
graph = {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "pos"}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": "neg"}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["2", 0]},
|
||||
},
|
||||
}
|
||||
self.assertEqual(extract({"prompt": graph}), "pos")
|
||||
self.assertEqual(extract({"prompt": {"x": 1}, "workflow": graph}), "pos")
|
||||
|
||||
def test_node_inputs_and_text_lookup(self):
|
||||
self.assertEqual(self.api._get_node_inputs("nope"), {})
|
||||
self.assertEqual(self.api._get_node_inputs({"inputs": 5}), {})
|
||||
self.assertEqual(self.api._get_node_inputs({"inputs": {"a": 1}}), {"a": 1})
|
||||
listed = self.api._get_node_inputs(
|
||||
{"inputs": [{"name": "text", "link": None}, {"nope": 1}, "junk"]}
|
||||
)
|
||||
self.assertEqual(list(listed), ["text"])
|
||||
|
||||
self.assertIsNone(self.api._find_text_in_node(None))
|
||||
self.assertEqual(self.api._find_text_in_node({"inputs": {"text": "hi"}}), "hi")
|
||||
self.assertEqual(
|
||||
self.api._find_text_in_node(
|
||||
{"type": "CLIPTextEncode", "widgets_values": ["w"]}
|
||||
),
|
||||
"w",
|
||||
)
|
||||
self.assertIsNone(
|
||||
self.api._find_text_in_node(
|
||||
{"type": "CLIPTextEncode", "widgets_values": [" "]}
|
||||
)
|
||||
)
|
||||
self.assertIsNone(self.api._find_text_in_node({"type": "KSampler"}))
|
||||
|
||||
def test_positive_prompt_from_nodes_array_and_fallbacks(self):
|
||||
extract = self.api._extract_positive_prompt_from_comfyui_data
|
||||
self.assertIsNone(extract("x"))
|
||||
self.assertIsNone(extract({}))
|
||||
self.assertIsNone(extract({"nodes": []}))
|
||||
|
||||
workflow = {
|
||||
"nodes": [
|
||||
{"id": 1, "type": "CLIPTextEncode", "widgets_values": ["pos"]},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "CLIPTextEncode",
|
||||
"widgets_values": ["neg"],
|
||||
"title": "Negative",
|
||||
},
|
||||
{"id": 3, "type": "KSampler", "inputs": [{"name": "seed", "link": 4}]},
|
||||
"junk",
|
||||
]
|
||||
}
|
||||
self.assertEqual(extract(workflow), "pos")
|
||||
|
||||
only_negative = {
|
||||
"nodes": [
|
||||
{
|
||||
"id": 2,
|
||||
"type": "CLIPTextEncode",
|
||||
"widgets_values": ["neg"],
|
||||
"title": "neg",
|
||||
},
|
||||
]
|
||||
}
|
||||
self.assertEqual(extract(only_negative), "neg")
|
||||
|
||||
bad_link = {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "pos"}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": "oops", "negative": []},
|
||||
},
|
||||
"4": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["zz", 0], "negative": []},
|
||||
},
|
||||
}
|
||||
self.assertEqual(extract(bad_link), "pos")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1831,39 +1831,6 @@ class TestWorkspaceCursorEncoding(unittest.TestCase):
|
||||
self.assertEqual(_workspace_cursor_key(token, "scope", order), key)
|
||||
|
||||
|
||||
# The pre-#379 candidate read, kept as the differential reference for the
|
||||
# indexed lookup (`image_tag_ids` + `image_tag_locators`): every byte record
|
||||
# streamed with its tag names, a locator kept when any record's names are
|
||||
# relevant (`_image_tags_relevant`).
|
||||
_REFERENCE_RECORDS = (
|
||||
"SELECT f.locator,b.id,t.name FROM media_annotation_files f "
|
||||
"JOIN media_annotation_bytes b ON b.file_id=f.id "
|
||||
"LEFT JOIN media_image_tags mit ON mit.byte_id=b.id "
|
||||
"LEFT JOIN tags t ON t.id=mit.tag_id "
|
||||
"ORDER BY f.locator,b.first_seen_at,b.id,mit.position"
|
||||
)
|
||||
|
||||
|
||||
def _image_tags_relevant(names, wanted, query):
|
||||
if wanted and not wanted.isdisjoint(names):
|
||||
return True
|
||||
return bool(query) and any(query in name.casefold() for name in names)
|
||||
|
||||
|
||||
def reference_candidates(db, wanted, query):
|
||||
"""Locators the old full stream admitted for (``wanted``, ``query``)."""
|
||||
records = {}
|
||||
for locator, byte_id, name in db.model.get_connection().execute(_REFERENCE_RECORDS):
|
||||
names = records.setdefault((locator, byte_id), [])
|
||||
if name is not None:
|
||||
names.append(name)
|
||||
return {
|
||||
locator
|
||||
for (locator, _byte), names in records.items()
|
||||
if _image_tags_relevant(names, set(wanted), query)
|
||||
}
|
||||
|
||||
|
||||
class TestWorkspaceImageTags(ImageAPITestCase):
|
||||
"""B3: image tags in the Tags filter and search (spec amendment S1)."""
|
||||
|
||||
@@ -2071,7 +2038,7 @@ class TestWorkspaceImageTags(ImageAPITestCase):
|
||||
leak = "no such table: media_annotation_files"
|
||||
with patch.object(
|
||||
self.api.db,
|
||||
"image_tag_locators",
|
||||
"iter_image_tag_records",
|
||||
side_effect=sqlite3.OperationalError(leak),
|
||||
):
|
||||
# As a failed link read does under a Tags filter: a fixed coded
|
||||
@@ -2089,140 +2056,6 @@ class TestWorkspaceImageTags(ImageAPITestCase):
|
||||
self.assertEqual(self.names(body), ["dunes_by_name.png"])
|
||||
self.assertIn("image tags", "\n".join(logged.output))
|
||||
|
||||
def seed_differential(self):
|
||||
"""Mixed-case and underscore names, a stale record, an unlisted file."""
|
||||
a = self.media("a.png", mtime=101)
|
||||
b = self.media("b.png", mtime=102)
|
||||
c = self.media("c.png", mtime=103)
|
||||
self.tag(a, ["Beach", "long_hair", "1girl"])
|
||||
self.tag(b, ["beach", "waves"])
|
||||
self.tag(c, ["Sunset"])
|
||||
# c's bytes change: its old record (Sunset) is stale, the new one current.
|
||||
make_png(c, size=(5, 5))
|
||||
os.utime(c, (104, 104))
|
||||
self.tag(c, ["night_sky", "BEACH"])
|
||||
outside = Path(self._output_tmp.name) / "elsewhere" / "x.png"
|
||||
outside.parent.mkdir(parents=True, exist_ok=True)
|
||||
make_png(outside)
|
||||
self.tag(outside, ["beach", "outside_only"])
|
||||
|
||||
async def test_indexed_candidates_equal_the_full_stream(self):
|
||||
self.seed_differential()
|
||||
cases = [
|
||||
(["beach"], ""),
|
||||
(["Beach"], ""),
|
||||
(["long_hair", "waves"], ""),
|
||||
([], "beach"),
|
||||
([], "hair"),
|
||||
([], "_"),
|
||||
([], "sun"),
|
||||
(["nope"], "zzz"),
|
||||
(["outside_only"], ""),
|
||||
(["beach"], "night"),
|
||||
]
|
||||
db = self.api.db
|
||||
for wanted, query in cases:
|
||||
with self.subTest(wanted=wanted, query=query):
|
||||
indexed = set(db.image_tag_locators(db.image_tag_ids(wanted, query)))
|
||||
self.assertEqual(indexed, reference_candidates(db, wanted, query))
|
||||
# The API answers are the indexed lookup's: exact names under Tags, a
|
||||
# case-folded substring under q, current records only.
|
||||
_, body = await self.get(tags="beach")
|
||||
self.assertEqual(sorted(self.names(body)), ["b.png"])
|
||||
_, body = await self.get(q="beach")
|
||||
self.assertEqual(sorted(self.names(body)), ["a.png", "b.png", "c.png"])
|
||||
_, body = await self.get(q="sunset")
|
||||
self.assertEqual(self.names(body), [])
|
||||
|
||||
async def test_tag_search_reads_image_tags_only_by_tag_id(self):
|
||||
self.seed_differential()
|
||||
statements = []
|
||||
original = self.api.db.model.get_connection
|
||||
|
||||
def trace_connection():
|
||||
connection = original()
|
||||
connection.set_trace_callback(statements.append)
|
||||
return connection
|
||||
|
||||
with patch.object(
|
||||
self.api.db.model, "get_connection", side_effect=trace_connection
|
||||
):
|
||||
for params in ({"tags": "beach"}, {"q": "hair"}):
|
||||
response, _ = await self.get(**params)
|
||||
self.assertEqual(response.status, 200)
|
||||
reads = [
|
||||
sql
|
||||
for sql in statements
|
||||
if "media_image_tags" in sql and sql.lstrip().upper().startswith("SELECT")
|
||||
]
|
||||
self.assertTrue(reads)
|
||||
for sql in reads:
|
||||
# Only a tag-id lookup, an EXISTS probe on the tag index, or the
|
||||
# current records' own tags (byte_id IN) read the junction.
|
||||
self.assertTrue(
|
||||
"tag_id IN" in sql or "EXISTS" in sql or "byte_id IN" in sql, sql
|
||||
)
|
||||
|
||||
async def test_the_image_tag_lookup_never_reads_prompt_tags(self):
|
||||
self.seed_differential()
|
||||
linked = self.media("linked.png", mtime=105)
|
||||
prompt = self._save_prompt(
|
||||
"Prompt with its own tag", tags=["prompt_only_beach"]
|
||||
)
|
||||
self.api.db.link_image_to_prompt(prompt, str(linked))
|
||||
db = self.api.db
|
||||
# The tag is only on a prompt: the image-tag lookup never yields it,
|
||||
# under the Tags filter or the text search.
|
||||
self.assertEqual(db.image_tag_ids(["prompt_only_beach"], ""), [])
|
||||
self.assertEqual(db.image_tag_ids([], "prompt_only"), [])
|
||||
self.assertEqual(
|
||||
db.image_tag_locators(db.image_tag_ids([], "beach")),
|
||||
reference_candidates(db, [], "beach"),
|
||||
)
|
||||
# The search still matches through the linked positive's prompt tags
|
||||
# (the design keeps that, #379 "Behavior that does not change").
|
||||
_, body = await self.get(q="prompt_only")
|
||||
self.assertEqual(self.names(body), ["linked.png"])
|
||||
|
||||
@unittest.skipUnless(os.environ.get("PM_PERF") == "1", "timing run (PM_PERF=1)")
|
||||
async def test_tag_search_timing(self):
|
||||
"""Linux timing for the PR body: 23,600 records, 10 tags each."""
|
||||
import hashlib
|
||||
import statistics
|
||||
import time
|
||||
|
||||
db = self.api.db
|
||||
epoch = db.get_execution_epoch()
|
||||
names = [f"name_{n:04d}" for n in range(2000)]
|
||||
root = os.path.realpath(self.output_dir)
|
||||
for n in range(23_600):
|
||||
path = self.output_dir / "perf" / f"{n:05d}.png"
|
||||
path.parent.mkdir(exist_ok=True)
|
||||
path.write_bytes(b"x")
|
||||
os.utime(path, (n + 1, n + 1))
|
||||
stat = path.stat()
|
||||
db.replace_image_tags(
|
||||
locator=os.path.normcase(os.path.join(root, "perf", path.name)),
|
||||
sha256=hashlib.sha256(str(n).encode()).hexdigest(),
|
||||
size=stat.st_size,
|
||||
mtime_ns=stat.st_mtime_ns,
|
||||
expected_epoch=epoch,
|
||||
expected_tag_revision=0,
|
||||
tags=[names[(n * 7 + k) % 2000] for k in range(10)],
|
||||
)
|
||||
await self.get(limit=1) # the listing, warm
|
||||
for label, params in (
|
||||
("one tag", {"tags": "name_0042"}),
|
||||
("text", {"q": "e_1"}),
|
||||
):
|
||||
times = []
|
||||
for _ in range(5):
|
||||
started = time.perf_counter()
|
||||
response, _ = await self.get(limit=100, **params)
|
||||
times.append(time.perf_counter() - started)
|
||||
self.assertEqual(response.status, 200)
|
||||
print(f"PM_PERF {label}: median {statistics.median(times) * 1000:.0f} ms")
|
||||
|
||||
async def test_legacy_output_failure_logs_its_traceback(self):
|
||||
self.media("legacy.png", mtime=100)
|
||||
with (
|
||||
@@ -2335,162 +2168,3 @@ class TestWorkspaceImageTags(ImageAPITestCase):
|
||||
)
|
||||
]
|
||||
self.assertFalse([step for step in plan if step.startswith("SCAN")], plan)
|
||||
|
||||
|
||||
class TestListingFilterSelection(ImageAPITestCase):
|
||||
"""#392: "Select all N matching" resolves the listing filter on the server."""
|
||||
|
||||
async def tearDownAsync(self):
|
||||
self.api.db.close_all()
|
||||
await super().tearDownAsync()
|
||||
|
||||
def media(self, name, mtime=100):
|
||||
target = self.output_dir / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
make_png(target)
|
||||
os.utime(target, (mtime, mtime))
|
||||
return target
|
||||
|
||||
async def listing(self, **params):
|
||||
response = await self.client.get(
|
||||
"/prompt_manager/images/output",
|
||||
params={"mode": "workspace", "limit": 500, **params},
|
||||
)
|
||||
return (await response.json())["images"]
|
||||
|
||||
async def test_resolve_lists_every_match_in_listing_order_beyond_a_page(self):
|
||||
for number in range(7):
|
||||
self.media(f"bulk/{number:02}.png", 100 + number)
|
||||
self.media("other/x.png", 500)
|
||||
from py.api.images import parse_listing_filter
|
||||
|
||||
params = parse_listing_filter({"subfolder": "bulk", "sort": "newest"})
|
||||
resolved = await self.api.resolve_listing_filter(params, cap=100)
|
||||
|
||||
listed = await self.listing(subfolder="bulk", sort="newest")
|
||||
self.assertEqual(
|
||||
[list(target) for target in resolved.targets],
|
||||
[[i["root_index"], i["root_scope"], i["path"]] for i in listed],
|
||||
)
|
||||
self.assertEqual(len(resolved.targets), 7)
|
||||
self.assertFalse(resolved.truncated)
|
||||
|
||||
async def test_resolve_reports_linked_positives_and_unlinked_count(self):
|
||||
first = self.media("a.png", 300)
|
||||
second = self.media("b.png", 200)
|
||||
self.media("c.png", 100)
|
||||
prompt = self._save_prompt("shared positive")
|
||||
self.api.db.link_image_to_prompt(prompt, str(first))
|
||||
self.api.db.link_image_to_prompt(prompt, str(second))
|
||||
from py.api.images import parse_listing_filter
|
||||
|
||||
resolved = await self.api.resolve_listing_filter(parse_listing_filter({}), 100)
|
||||
|
||||
self.assertEqual(resolved.positive_ids, [prompt])
|
||||
self.assertEqual(resolved.unlinked, 1)
|
||||
|
||||
async def test_resolve_marks_more_than_the_cap_as_truncated(self):
|
||||
for number in range(4):
|
||||
self.media(f"{number}.png", 100 + number)
|
||||
from py.api.images import parse_listing_filter
|
||||
|
||||
resolved = await self.api.resolve_listing_filter(parse_listing_filter({}), 3)
|
||||
|
||||
self.assertTrue(resolved.truncated)
|
||||
|
||||
def test_parse_listing_filter_rejects_anything_but_listing_strings(self):
|
||||
from py.api.images import parse_listing_filter
|
||||
|
||||
self.assertEqual(
|
||||
parse_listing_filter({"q": "cat", "tags": "a,b", "root": "0"}),
|
||||
{"q": "cat", "tags": "a,b", "root": "0"},
|
||||
)
|
||||
for bad in (
|
||||
None,
|
||||
[],
|
||||
"q=cat",
|
||||
{"mode": "workspace"},
|
||||
{"cursor": "x"},
|
||||
{"path": "a.png"},
|
||||
{"q": 1},
|
||||
{"q": "x" * 2049},
|
||||
{"sort": "sideways"},
|
||||
{"media": "audio"},
|
||||
{"favorites": "yes"},
|
||||
{"subfolder": "../up"},
|
||||
{"root": "-1"},
|
||||
):
|
||||
with self.subTest(bad=bad):
|
||||
with self.assertRaises(ValueError):
|
||||
params = parse_listing_filter(bad)
|
||||
# Root bounds need the roots: the resolver checks them.
|
||||
from py.api.images import _workspace_root
|
||||
|
||||
_workspace_root(params, [self.output_dir])
|
||||
|
||||
async def test_bulk_category_takes_the_listing_filter(self):
|
||||
first = self.media("keep/a.png", 300)
|
||||
second = self.media("keep/b.png", 200)
|
||||
self.media("keep/c.png", 100)
|
||||
other = self.media("skip/d.png", 400)
|
||||
prompt = self._save_prompt("in the filter")
|
||||
outside = self._save_prompt("outside the filter")
|
||||
self.api.db.link_image_to_prompt(prompt, str(first))
|
||||
self.api.db.link_image_to_prompt(prompt, str(second))
|
||||
self.api.db.link_image_to_prompt(outside, str(other))
|
||||
|
||||
response = await self.client.post(
|
||||
"/prompt_manager/bulk/category",
|
||||
json={"images_filter": {"subfolder": "keep"}, "category": "Kept"},
|
||||
)
|
||||
data = await response.json()
|
||||
|
||||
self.assertEqual(response.status, 200, data)
|
||||
self.assertEqual(
|
||||
(data["updated_count"], data["prompt_count"], data["skipped"]), (1, 1, 1)
|
||||
)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(prompt)["category"], "Kept")
|
||||
self.assertNotEqual(self.api.db.get_prompt_by_id(outside)["category"], "Kept")
|
||||
|
||||
async def test_bulk_category_filter_with_no_linked_image_changes_nothing(self):
|
||||
self.media("a.png")
|
||||
|
||||
response = await self.client.post(
|
||||
"/prompt_manager/bulk/category",
|
||||
json={"images_filter": {}, "category": "Kept"},
|
||||
)
|
||||
data = await response.json()
|
||||
|
||||
self.assertEqual(response.status, 200, data)
|
||||
self.assertEqual(
|
||||
(data["updated_count"], data["prompt_count"], data["skipped"]), (0, 0, 1)
|
||||
)
|
||||
|
||||
async def test_bulk_category_refuses_a_bad_or_doubled_filter(self):
|
||||
self.media("a.png")
|
||||
prompt = self._save_prompt("p")
|
||||
for body in (
|
||||
{"images_filter": {"sort": "sideways"}, "category": "x"},
|
||||
{"images_filter": "all", "category": "x"},
|
||||
{"images_filter": {}, "prompt_ids": [prompt], "category": "x"},
|
||||
{"images_filter": {"root": "9"}, "category": "x"},
|
||||
):
|
||||
with self.subTest(body=body):
|
||||
response = await self.client.post(
|
||||
"/prompt_manager/bulk/category", json=body
|
||||
)
|
||||
self.assertEqual(response.status, 400, await response.text())
|
||||
|
||||
async def test_bulk_category_refuses_a_renumbered_source(self):
|
||||
self.media("a.png")
|
||||
|
||||
response = await self.client.post(
|
||||
"/prompt_manager/bulk/category",
|
||||
json={
|
||||
"images_filter": {"root": "0", "root_scope": "f" * 64},
|
||||
"category": "x",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status, 409)
|
||||
self.assertEqual((await response.json())["code"], "source-changed")
|
||||
|
||||
@@ -1,696 +0,0 @@
|
||||
"""Tag output images tags each image, never its prompt (#379, PR 1).
|
||||
|
||||
``POST /prompt_manager/jobs/autotag`` runs the ``image-tags`` loop over every
|
||||
still image the output folders list, linked to a prompt or not: each image
|
||||
gets its own model tags in its image tag set and is marked auto-tagged.
|
||||
"Skip already tagged" skips images whose current bytes are marked, counted
|
||||
before the loop without opening them. Prompt tags and prompt text never
|
||||
change. No real model is loaded: the engine is a fake that answers per image.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import ntpath
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import sqlite3
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from test_api_images import make_png
|
||||
from test_image_tag_jobs import HookedTagger, ImageTagJobTestCase
|
||||
except ImportError:
|
||||
from tests.test_api_images import make_png
|
||||
from tests.test_image_tag_jobs import HookedTagger, ImageTagJobTestCase
|
||||
|
||||
from database.execution_history_schema import rotate_history_epoch
|
||||
from py import model_gate
|
||||
from py.api import admin as admin_module
|
||||
from py.api import image_tag_jobs as jobs_module
|
||||
from py.config import PromptManagerConfig
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
JOB = "/prompt_manager/jobs/autotag"
|
||||
PREVIEW = "/prompt_manager/autotag/batch/preview"
|
||||
WAIT = 30.0
|
||||
WD14 = {"general_threshold": 0.35, "character_threshold": 0.85}
|
||||
CANCELLED = "Cancelled. Tags already added were kept."
|
||||
_DRIVE = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
|
||||
|
||||
class PerImageTagger(HookedTagger):
|
||||
"""Answers each image from its own bytes: ``by_bytes`` maps exact file
|
||||
contents to the names the "model" sees; any other image gets SCORED."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.by_bytes = {}
|
||||
|
||||
def generate_tag_suggestions(self, image_path, **kwargs):
|
||||
result = super().generate_tag_suggestions(image_path, **kwargs)
|
||||
names = self.by_bytes.get(self.read_bytes[-1])
|
||||
if names is None:
|
||||
return result
|
||||
return [(name, 0.9) for name in names][: kwargs["limit"]]
|
||||
|
||||
|
||||
def _strings(value):
|
||||
if isinstance(value, str):
|
||||
yield value
|
||||
elif isinstance(value, dict):
|
||||
for item in value.values():
|
||||
yield from _strings(item)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
for item in value:
|
||||
yield from _strings(item)
|
||||
|
||||
|
||||
def _absolute(text):
|
||||
return posixpath.isabs(text) or ntpath.isabs(text) or bool(_DRIVE.match(text))
|
||||
|
||||
|
||||
class BatchTestCase(ImageTagJobTestCase):
|
||||
async def get_application(self):
|
||||
app = await super().get_application()
|
||||
self.tagger = PerImageTagger()
|
||||
patcher = patch("py.autotag.get_autotag_service", return_value=self.tagger)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
backup = PromptManagerConfig.SAFETY_BACKUP
|
||||
self.addCleanup(setattr, PromptManagerConfig, "SAFETY_BACKUP", backup)
|
||||
PromptManagerConfig.SAFETY_BACKUP = False
|
||||
return app
|
||||
|
||||
# -- helpers ---------------------------------------------------------
|
||||
|
||||
def image(self, name, color, sees=None):
|
||||
path = make_png(self.output_dir / name, size=(8, 6), color=color)
|
||||
if sees is not None:
|
||||
self.tagger.by_bytes[path.read_bytes()] = list(sees)
|
||||
return path
|
||||
|
||||
async def relist(self):
|
||||
"""Walk the output folder again (a cached listing is used as it is)."""
|
||||
await self.api._listing(self.output_dir, fresh=True)
|
||||
|
||||
def prompt(self, text, tags=()):
|
||||
return self.api.db.save_prompt(
|
||||
text=text,
|
||||
category=None,
|
||||
tags=list(tags),
|
||||
rating=None,
|
||||
prompt_hash=generate_prompt_hash(text),
|
||||
)
|
||||
|
||||
def link(self, prompt_id, path):
|
||||
return self.api.db.link_image_to_prompt(prompt_id, str(path))
|
||||
|
||||
def prompt_rows(self):
|
||||
conn = sqlite3.connect(self._temp_db.name)
|
||||
try:
|
||||
return {
|
||||
table: conn.execute(f"SELECT * FROM {table} ORDER BY 1,2").fetchall()
|
||||
for table in ("prompts", "prompt_tags")
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def marked(self):
|
||||
conn = sqlite3.connect(self._temp_db.name)
|
||||
try:
|
||||
return {
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT f.locator FROM media_image_autotags m JOIN "
|
||||
"media_annotation_bytes b ON b.id=m.byte_id JOIN "
|
||||
"media_annotation_files f ON f.id=b.file_id"
|
||||
)
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
async def preview(self, existing="skip", model_type="wd14-vit"):
|
||||
response = await self.client.post(
|
||||
PREVIEW, json={"model_type": model_type, "existing": existing}
|
||||
)
|
||||
return response.status, await self.json_of(response)
|
||||
|
||||
async def start_batch(self, existing="skip", model_type="wd14-vit", options=None):
|
||||
if options is None:
|
||||
options = (
|
||||
dict(WD14)
|
||||
if model_type.startswith("wd14")
|
||||
else {"instructions": "Tags please."}
|
||||
)
|
||||
response = await self.client.post(
|
||||
JOB,
|
||||
json={"model_type": model_type, "existing": existing, "options": options},
|
||||
)
|
||||
return response.status, await self.json_of(response)
|
||||
|
||||
async def run_batch(self, **kwargs):
|
||||
status, data = await self.start_batch(**kwargs)
|
||||
self.assertEqual(status, 202, data)
|
||||
job = self.api._jobs.get(data["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
return job
|
||||
|
||||
|
||||
class TestPerImageWrites(BatchTestCase):
|
||||
async def test_two_images_of_one_prompt_get_their_own_tags_and_no_prompt_changes(
|
||||
self,
|
||||
):
|
||||
prompt_id = self.prompt("a girl at the beach", tags=["beach"])
|
||||
a = self.image("a.png", (10, 20, 30), sees=["1girl", "bird"])
|
||||
b = self.image("b.png", (40, 50, 60), sees=["1girl", "waves"])
|
||||
self.link(prompt_id, a)
|
||||
self.link(prompt_id, b)
|
||||
before = self.prompt_rows()
|
||||
|
||||
job = await self.run_batch(existing="skip")
|
||||
|
||||
self.assertEqual(job.state, "finished", job.status_line)
|
||||
self.assertEqual(self.stored("a.png")["tags"], ["1girl", "bird"])
|
||||
self.assertEqual(self.stored("b.png")["tags"], ["1girl", "waves"])
|
||||
self.assertEqual(self.prompt_rows(), before)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(prompt_id)["tags"], ["beach"])
|
||||
self.assertEqual(self.marked(), {self.locator("a.png"), self.locator("b.png")})
|
||||
self.assertEqual(
|
||||
job.counters, {"tagged": 2, "unchanged": 0, "skipped": 0, "errors": 0}
|
||||
)
|
||||
self.assertEqual(job.status_line, "Tagged 2 images")
|
||||
self.assertEqual((job.processed, job.total, job.progress), (2, 2, 100))
|
||||
self.assertEqual(job.scope["images"], 2)
|
||||
# Inference read the private captures, one per image.
|
||||
self.assertEqual(len(self.tagger.calls), 2)
|
||||
|
||||
async def test_unlinked_images_are_tagged_and_videos_are_not_targets(self):
|
||||
self.image("free.png", (1, 2, 3), sees=["tree"])
|
||||
(self.output_dir / "clip.mp4").write_bytes(b"\x00\x00\x00\x18ftypmp42" * 8)
|
||||
|
||||
job = await self.run_batch()
|
||||
|
||||
self.assertEqual(self.stored("free.png")["tags"], ["tree"])
|
||||
self.assertEqual(job.total, 1)
|
||||
self.assertEqual(job.status_line, "Tagged 1 image")
|
||||
rows = [row["path"] for rows in job.outcomes.values() for row in rows]
|
||||
self.assertNotIn("clip.mp4", rows)
|
||||
|
||||
async def test_hand_tags_do_not_block_a_skip_run(self):
|
||||
path = self.image("c.png", (7, 7, 7), sees=["bird"])
|
||||
self.saved_set("c.png", path.read_bytes(), ["keeper"])
|
||||
|
||||
job = await self.run_batch(existing="skip")
|
||||
|
||||
self.assertEqual(self.stored("c.png")["tags"], ["keeper", "bird"])
|
||||
self.assertEqual(job.counters["tagged"], 1)
|
||||
|
||||
async def test_skip_counts_marked_images_without_opening_them_and_merge_retags(
|
||||
self,
|
||||
):
|
||||
self.image("a.png", (1, 1, 1), sees=["bird"])
|
||||
self.image("b.png", (2, 2, 2), sees=["cat"])
|
||||
first = await self.run_batch()
|
||||
self.assertEqual(first.counters["tagged"], 2)
|
||||
self.tagger.calls.clear()
|
||||
|
||||
captures = []
|
||||
store = self.api._capture_store()
|
||||
real_capture = store.capture
|
||||
|
||||
def spy(*args, **kwargs):
|
||||
captures.append(args)
|
||||
return real_capture(*args, **kwargs)
|
||||
|
||||
with patch.object(store, "capture", side_effect=spy):
|
||||
skipped = await self.run_batch(existing="skip")
|
||||
self.assertEqual(captures, [])
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
self.assertEqual(
|
||||
skipped.counters, {"tagged": 0, "unchanged": 0, "skipped": 2, "errors": 0}
|
||||
)
|
||||
self.assertEqual(
|
||||
sorted(self.rows(skipped, "skipped")),
|
||||
[("a.png", "Already auto-tagged"), ("b.png", "Already auto-tagged")],
|
||||
)
|
||||
self.assertFalse(any("Already auto-tagged" in line for line in skipped.log))
|
||||
self.assertEqual((skipped.processed, skipped.total), (2, 2))
|
||||
self.assertEqual(skipped.status_line, "Tagged 0 images")
|
||||
|
||||
self.tagger.by_bytes[(self.output_dir / "a.png").read_bytes()] = ["bird", "sky"]
|
||||
merged = await self.run_batch(existing="merge")
|
||||
self.assertEqual(len(self.tagger.calls), 2)
|
||||
self.assertEqual(self.stored("a.png")["tags"], ["bird", "sky"])
|
||||
self.assertEqual(self.rows(merged, "tagged"), [("a.png", "Added 1 tag")])
|
||||
self.assertEqual(self.rows(merged, "unchanged"), [("b.png", "No new tags")])
|
||||
|
||||
async def test_changed_bytes_are_tagged_again(self):
|
||||
self.image("a.png", (1, 1, 1), sees=["bird"])
|
||||
await self.run_batch()
|
||||
self.image("a.png", (9, 9, 9), sees=["cat"])
|
||||
job = await self.run_batch(existing="skip")
|
||||
self.assertEqual(self.rows(job, "tagged"), [("a.png", "Added 1 tag")])
|
||||
self.assertEqual(self.stored("a.png")["tags"], ["cat"])
|
||||
|
||||
async def test_an_image_with_no_model_tags_is_marked_and_skipped_next_time(self):
|
||||
self.image("plain.png", (3, 3, 3), sees=[])
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(self.rows(job, "unchanged"), [("plain.png", "No new tags")])
|
||||
self.assertEqual(self.marked(), {self.locator("plain.png")})
|
||||
again = await self.run_batch()
|
||||
self.assertEqual(
|
||||
self.rows(again, "skipped"), [("plain.png", "Already auto-tagged")]
|
||||
)
|
||||
|
||||
async def test_unsupported_still_is_marked_and_skipped_by_the_next_run(self):
|
||||
(self.output_dir / "broken.png").write_bytes(b"not an image")
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(
|
||||
self.rows(job, "skipped"), [("broken.png", "Not a supported still image")]
|
||||
)
|
||||
self.assertEqual(self.marked(), {self.locator("broken.png")})
|
||||
again = await self.run_batch()
|
||||
self.assertEqual(
|
||||
self.rows(again, "skipped"), [("broken.png", "Already auto-tagged")]
|
||||
)
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
|
||||
async def test_too_large_is_skipped_before_any_copy(self):
|
||||
self.image("big.png", (5, 5, 5))
|
||||
self.api._capture_store().max_file = 16
|
||||
store = self.api._capture_store()
|
||||
with patch.object(store, "capture", side_effect=AssertionError("copied")):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(
|
||||
self.rows(job, "skipped"), [("big.png", "Too large to verify")]
|
||||
)
|
||||
self.assertEqual(self.marked(), set())
|
||||
|
||||
async def test_overlapping_roots_list_a_file_once(self):
|
||||
self.image("sub/a.png", (4, 4, 4), sees=["bird"])
|
||||
sub = self.output_dir / "sub"
|
||||
self.api._get_all_output_dirs = lambda: [self.output_dir, sub]
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.total, 1)
|
||||
self.assertEqual(len(self.tagger.calls), 1)
|
||||
self.assertEqual(self.rows(job, "tagged"), [("sub/a.png", "Added 1 tag")])
|
||||
status, data = await self.preview()
|
||||
self.assertEqual((status, data["images"]), (200, 1))
|
||||
|
||||
async def test_rows_log_and_status_carry_no_absolute_paths(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
(self.output_dir / "broken.png").write_bytes(b"not an image")
|
||||
job = await self.run_batch()
|
||||
leaked = [s for s in _strings(job.report()) if _absolute(s)]
|
||||
self.assertEqual(leaked, [])
|
||||
self.assertNotIn(str(self.output_dir), json.dumps(job.report()))
|
||||
|
||||
|
||||
class TestBatchLifecycle(BatchTestCase):
|
||||
async def test_cancel_after_one_image_keeps_it_and_the_rerun_skips_it(self):
|
||||
for n in range(3):
|
||||
self.image(f"{n}.png", (n, n, n), sees=[f"t{n}"])
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
status, data = await self.start_batch()
|
||||
self.assertEqual(status, 202, data)
|
||||
job = self.api._jobs.get(data["job"]["id"])
|
||||
try:
|
||||
await self.wait_for(entered)
|
||||
cancel = await self.client.post(f"/prompt_manager/jobs/{job.id}/cancel")
|
||||
self.assertEqual(cancel.status, 200)
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.status_line, CANCELLED)
|
||||
self.assertEqual(job.counters["tagged"], 1)
|
||||
self.assertEqual(len(self.marked()), 1)
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
lines = []
|
||||
real_update = type(job).update
|
||||
|
||||
def spy(self_job, **kwargs):
|
||||
if kwargs.get("status_line"):
|
||||
lines.append(kwargs["status_line"])
|
||||
return real_update(self_job, **kwargs)
|
||||
|
||||
with patch.object(type(job), "update", spy):
|
||||
rerun = await self.run_batch()
|
||||
self.assertEqual(rerun.counters["skipped"], 1)
|
||||
self.assertEqual(rerun.counters["tagged"], 2)
|
||||
self.assertIn("Processing file 2 of 3", lines)
|
||||
self.assertIn("Processing file 3 of 3", lines)
|
||||
self.assertNotIn("Processing file 1 of 3", lines)
|
||||
self.assertEqual((rerun.processed, rerun.total), (3, 3))
|
||||
|
||||
async def test_cancel_in_each_phase_writes_nothing(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
for phase in (
|
||||
"_safety_backup_step",
|
||||
"_prepare_tagging_model",
|
||||
"_autotag_targets",
|
||||
):
|
||||
with self.subTest(phase):
|
||||
real = getattr(self.api, phase)
|
||||
|
||||
async def cancelling(*args, _real=real, **kwargs):
|
||||
result = await _real(*args, **kwargs)
|
||||
self.api._jobs.active("autotag").cancelled = True
|
||||
return result
|
||||
|
||||
with patch.object(self.api, phase, side_effect=cancelling):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.status_line, CANCELLED)
|
||||
self.assertEqual(self.marked(), set())
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_cancel_between_prepass_chunks_keeps_counts(self):
|
||||
for n in range(3):
|
||||
self.image(f"{n}.png", (n, n, n))
|
||||
await self.run_batch()
|
||||
seen = []
|
||||
real = jobs_module._prepass_chunk
|
||||
|
||||
def chunk(db, items, skip_marked, max_file):
|
||||
result = real(db, items, skip_marked, max_file)
|
||||
job = self.api._jobs.active("autotag")
|
||||
seen.append(job.processed)
|
||||
job.cancelled = True # cancel after the first chunk
|
||||
return result
|
||||
|
||||
with (
|
||||
patch.object(jobs_module, "PREPASS_CHUNK", 1),
|
||||
patch.object(jobs_module, "_prepass_chunk", side_effect=chunk),
|
||||
):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.counters["skipped"], 1)
|
||||
self.assertEqual((job.processed, job.total), (1, 3))
|
||||
self.assertEqual(seen, [0])
|
||||
|
||||
async def test_the_job_ends_only_after_its_cleanup(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
PromptManagerConfig.SAFETY_BACKUP = True
|
||||
states = []
|
||||
|
||||
async def prune(job, _db):
|
||||
states.append((job.state, model_gate.holder() is not None))
|
||||
|
||||
with (
|
||||
patch("py.api.maintenance.take_safety_backup", return_value="db.backup_x"),
|
||||
patch.object(self.api, "_prune_backup_pool", side_effect=prune),
|
||||
):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "finished")
|
||||
self.assertEqual(states, [("running", True)])
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_prepass_publishes_its_progress(self):
|
||||
for n in range(2):
|
||||
self.image(f"{n}.png", (n, n, n))
|
||||
await self.run_batch()
|
||||
published = []
|
||||
real = type(self.api._jobs).start
|
||||
|
||||
def start(registry, *args, **kwargs):
|
||||
job = real(registry, *args, **kwargs)
|
||||
original = job.publish
|
||||
job.publish = lambda payload: (
|
||||
published.append(dict(payload)),
|
||||
original(payload),
|
||||
)
|
||||
return job
|
||||
|
||||
with patch.object(type(self.api._jobs), "start", start):
|
||||
await self.run_batch()
|
||||
self.assertIn({"type": "progress", "progress": 100}, published)
|
||||
self.assertEqual(published[-1], {"type": "finished", "progress": 100})
|
||||
|
||||
async def test_restore_mid_run_fails_the_job(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
self.image("b.png", (2, 2, 2))
|
||||
|
||||
def restore(_path):
|
||||
conn = sqlite3.connect(self._temp_db.name)
|
||||
try:
|
||||
rotate_history_epoch(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
self.tagger.on_generate = restore
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertEqual(
|
||||
job.status_line, "Prompt storage was restored; start the job again."
|
||||
)
|
||||
|
||||
async def test_listing_error_fails_the_job(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
with patch.object(
|
||||
self.api, "_get_gallery_files", side_effect=OSError("/secret/out")
|
||||
):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertEqual(
|
||||
job.status_line, "The output folders could not be listed. Try again."
|
||||
)
|
||||
self.assertNotIn("/secret", json.dumps(job.report()))
|
||||
|
||||
async def test_model_that_cannot_load_fails_the_job(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
self.tagger.load_error = RuntimeError("no CUDA")
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertEqual(
|
||||
job.status_line, "Auto-tagging failed. Check server logs for details."
|
||||
)
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_zero_targets_finish_with_tagged_zero(self):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "finished")
|
||||
self.assertEqual(job.status_line, "Tagged 0 images")
|
||||
self.assertEqual((job.processed, job.total), (0, 0))
|
||||
|
||||
async def test_runner_unloads_after_a_cancel_when_not_kept_in_memory(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
owner = model_gate.new_owner("legacy-batch")
|
||||
self.assertTrue(model_gate.try_acquire(owner))
|
||||
run = jobs_module.AutotagRun(
|
||||
model_type="wd14-vit",
|
||||
existing="skip",
|
||||
options={**WD14, "use_gpu": True, "suggestion_limit": 10},
|
||||
keep_in_memory=False,
|
||||
backup_on=False,
|
||||
owner=owner,
|
||||
)
|
||||
|
||||
from py.api.jobs import AUTOTAG_GROUP_NAMES, Job as RealJob
|
||||
|
||||
job = RealJob(
|
||||
"autotag",
|
||||
"Tag output images",
|
||||
admission="records",
|
||||
groups=AUTOTAG_GROUP_NAMES,
|
||||
)
|
||||
real_targets = self.api._autotag_targets
|
||||
|
||||
async def cancel_after_listing():
|
||||
targets = await real_targets()
|
||||
job.cancelled = True
|
||||
return targets
|
||||
|
||||
with patch.object(
|
||||
self.api, "_autotag_targets", side_effect=cancel_after_listing
|
||||
):
|
||||
await self.api._run_autotag_batch(run, job)
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(self.tagger.unload_calls, 1)
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_time_left_uses_the_models_rate_only(self):
|
||||
for n in range(4):
|
||||
self.image(f"{n}.png", (n, n, n))
|
||||
await self.run_batch()
|
||||
for n in range(4, 8):
|
||||
self.image(f"{n}.png", (n, n, n))
|
||||
await self.relist()
|
||||
calls = []
|
||||
real = jobs_module._time_left
|
||||
|
||||
def spy(*args):
|
||||
calls.append(args)
|
||||
return real(*args)
|
||||
|
||||
with patch.object(jobs_module, "_time_left", side_effect=spy):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.counters["skipped"], 4)
|
||||
# (done, total, batches, elapsed): over the 4 images the model runs on.
|
||||
self.assertEqual(
|
||||
[(c[0], c[1], c[2]) for c in calls], [(n, 4, n) for n in range(4)]
|
||||
)
|
||||
# The shared D56 text is unchanged.
|
||||
self.assertEqual(admin_module._time_left(5, 10, 5, 300), " · about 5 min left")
|
||||
|
||||
|
||||
class TestStartRefusals(BatchTestCase):
|
||||
async def test_unavailable_model_and_bad_bodies(self):
|
||||
self.tagger.downloaded["wd14-vit"] = False
|
||||
status, data = await self.start_batch()
|
||||
self.assertEqual((status, data["code"]), (409, "model-unavailable"))
|
||||
status, data = await self.start_batch(existing="replace_after_review")
|
||||
self.assertEqual((status, data["code"]), (400, "review-required"))
|
||||
status, _data = await self.start_batch(options={**WD14, "suggestion_limit": 3})
|
||||
self.assertEqual(status, 400)
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_model_busy_when_the_gate_is_held(self):
|
||||
owner = model_gate.new_owner("image-annotations")
|
||||
self.assertTrue(model_gate.try_acquire(owner))
|
||||
try:
|
||||
status, data = await self.start_batch()
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
self.assertEqual((status, data["code"]), (409, "model-busy"))
|
||||
|
||||
|
||||
class TestBatchBeside(BatchTestCase):
|
||||
async def test_legacy_batch_and_unload_are_refused_while_the_job_runs(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
status, data = await self.start_batch()
|
||||
self.assertEqual(status, 202, data)
|
||||
job = self.api._jobs.get(data["job"]["id"])
|
||||
try:
|
||||
await self.wait_for(entered)
|
||||
self.assertTrue(model_gate.holder().startswith("autotag-job:"))
|
||||
legacy = await self.client.post("/prompt_manager/autotag/start")
|
||||
self.assertEqual(legacy.status, 200)
|
||||
frames = [
|
||||
json.loads(line[len("data: ") :])
|
||||
for line in (await legacy.text()).splitlines()
|
||||
if line.startswith("data: ")
|
||||
]
|
||||
self.assertEqual(
|
||||
frames,
|
||||
[
|
||||
{
|
||||
"type": "error",
|
||||
"status": "error",
|
||||
"message": "Tag output images already running",
|
||||
}
|
||||
],
|
||||
)
|
||||
unload = await self.client.post("/prompt_manager/autotag/unload")
|
||||
self.assertEqual((await unload.json())["operation"], "Tag output images")
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
self.assertEqual(job.state, "finished")
|
||||
|
||||
async def test_language_model_batch_passes_instructions_per_call(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
job = await self.run_batch(model_type="gguf")
|
||||
self.assertEqual(job.state, "finished", job.status_line)
|
||||
self.assertEqual(self.tagger.calls[0]["prompt"], "Tags please.")
|
||||
self.assertEqual(self.tagger.custom_prompt, "legacy prompt")
|
||||
|
||||
async def test_inference_errors_are_fixed_strings_and_not_marked(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
|
||||
def explode(_path):
|
||||
raise RuntimeError("/srv/secret/model exploded")
|
||||
|
||||
self.tagger.on_generate = explode
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(
|
||||
self.rows(job, "errors"),
|
||||
[("a.png", "Auto-tagging failed. Check server logs for details.")],
|
||||
)
|
||||
self.assertNotIn("/srv/secret", json.dumps(job.report()))
|
||||
self.assertEqual(self.marked(), set())
|
||||
|
||||
|
||||
class TestPreview(BatchTestCase):
|
||||
async def test_preview_counts_images_and_estimates_time(self):
|
||||
prompt_id = self.prompt("p")
|
||||
a = self.image("a.png", (1, 1, 1))
|
||||
self.image("b.png", (2, 2, 2))
|
||||
self.image("c.png", (3, 3, 3))
|
||||
self.link(prompt_id, a)
|
||||
(self.output_dir / "clip.mp4").write_bytes(b"\x00\x00\x00\x18ftypmp42" * 8)
|
||||
status, fresh = await self.preview()
|
||||
self.assertEqual(status, 200, fresh)
|
||||
self.assertEqual(
|
||||
{k: fresh[k] for k in ("images", "already_tagged", "to_tag")},
|
||||
{"images": 3, "already_tagged": 0, "to_tag": 3},
|
||||
)
|
||||
self.assertNotIn("prompts", fresh)
|
||||
self.assertEqual(fresh["suggestion_limit"], 10)
|
||||
self.assertEqual(fresh["estimate_device"], "cpu")
|
||||
self.assertAlmostEqual(fresh["estimate_seconds"], 3 * 1.6)
|
||||
|
||||
self.tagger.block = None
|
||||
await self.run_batch()
|
||||
self.image("d.png", (4, 4, 4))
|
||||
await self.relist()
|
||||
skip = (await self.preview("skip"))[1]
|
||||
self.assertEqual(
|
||||
{k: skip[k] for k in ("images", "already_tagged", "to_tag")},
|
||||
{"images": 4, "already_tagged": 3, "to_tag": 1},
|
||||
)
|
||||
merge = (await self.preview("merge"))[1]
|
||||
self.assertEqual(merge["to_tag"], 4)
|
||||
for model_type in ("gguf", "hf"):
|
||||
llm = (await self.preview(model_type=model_type))[1]
|
||||
self.assertIsNone(llm["estimate_seconds"])
|
||||
self.assertIsNone(llm["estimate_device"])
|
||||
self.assertEqual([s for s in _strings(skip) if _absolute(s)], [])
|
||||
|
||||
async def test_preview_uses_the_graphics_card_rate_when_wd14_runs_on_cuda(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
self.tagger.wd14_device = lambda: "gpu"
|
||||
data = (await self.preview())[1]
|
||||
self.assertEqual(data["estimate_device"], "gpu")
|
||||
from py.tagging_defaults import WD14_GPU_SECONDS_PER_IMAGE
|
||||
|
||||
self.assertAlmostEqual(data["estimate_seconds"], WD14_GPU_SECONDS_PER_IMAGE)
|
||||
|
||||
async def test_preview_writes_nothing_and_opens_no_media_file(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
await self.run_batch()
|
||||
self.image("b.png", (2, 2, 2))
|
||||
await self.relist() # the listing, primed
|
||||
files = [self._temp_db.name + suffix for suffix in ("", "-wal")]
|
||||
before = [Path(f).read_bytes() if os.path.exists(f) else None for f in files]
|
||||
media = str(self.output_dir)
|
||||
real_open, real_stat = open, os.stat
|
||||
|
||||
def guarded_open(file, *args, **kwargs):
|
||||
if str(file).startswith(media):
|
||||
raise AssertionError(f"opened {file}")
|
||||
return real_open(file, *args, **kwargs)
|
||||
|
||||
def guarded_stat(path, *args, **kwargs):
|
||||
if str(path).startswith(media) and str(path) != media:
|
||||
raise AssertionError(f"stat {path}")
|
||||
return real_stat(path, *args, **kwargs)
|
||||
|
||||
with patch("builtins.open", guarded_open), patch("os.stat", guarded_stat):
|
||||
status, data = await self.preview()
|
||||
self.assertEqual(status, 200, data)
|
||||
after = [Path(f).read_bytes() if os.path.exists(f) else None for f in files]
|
||||
self.assertEqual(after, before)
|
||||
self.assertEqual(data["to_tag"], 1)
|
||||
self.assertEqual(self.tagger.calls[1:], [])
|
||||
@@ -884,29 +884,3 @@ class TestImageConversion(EngineTestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestWd14Device(EngineTestCase):
|
||||
"""Where WD14 runs, for the Tag output images estimate (#379)."""
|
||||
|
||||
def test_loaded_session_answers_from_its_first_provider(self):
|
||||
cpu = FakeOnnxSession([0.5] * len(WD14_TAGS))
|
||||
service = self._service(("wd14", (cpu, WD14_TAGS)))
|
||||
_touch_model_files(self.models_dir, "wd14-vit")
|
||||
service.load_model("wd14-vit")
|
||||
self.assertEqual(service.wd14_device(), "cpu")
|
||||
cpu.get_providers = lambda: ["CUDAExecutionProvider", "CPUExecutionProvider"]
|
||||
self.assertEqual(service.wd14_device(), "gpu")
|
||||
|
||||
def test_without_a_session_the_available_providers_answer(self):
|
||||
service = self._service(None)
|
||||
fake = types.SimpleNamespace(
|
||||
get_available_providers=lambda: ["CUDAExecutionProvider"]
|
||||
)
|
||||
with patch.dict(sys.modules, {"onnxruntime": fake}):
|
||||
self.assertEqual(service.wd14_device(), "gpu")
|
||||
fake.get_available_providers = lambda: ["CPUExecutionProvider"]
|
||||
with patch.dict(sys.modules, {"onnxruntime": fake}):
|
||||
self.assertEqual(service.wd14_device(), "cpu")
|
||||
with patch.dict(sys.modules, {"onnxruntime": None}):
|
||||
self.assertEqual(service.wd14_device(), "cpu")
|
||||
|
||||
@@ -1,617 +0,0 @@
|
||||
"""The legacy pages and the Tag image dialog tag images, never prompts (#379, PR 2).
|
||||
|
||||
- ``POST /autotag/start`` (legacy admin and gallery batch) starts the
|
||||
``autotag`` job titled "Tag output images (legacy page)" and relays it as
|
||||
the legacy frames; closing the stream cancels it.
|
||||
- ``POST /autotag/single`` suggests on a capture with the saved cap and
|
||||
answers the captured ``sha256``; ``existing: "skip"`` answers
|
||||
``already_tagged`` for an auto-tagged image without the model gate.
|
||||
- ``POST /autotag/apply`` unions into that image's tags and marks it; a
|
||||
``prompt_id`` body is refused.
|
||||
- A Tag image dialog save after Generate suggestions marks the image.
|
||||
|
||||
No real model runs: the per-image fake from test_autotag_batch answers.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
from unittest.mock import patch
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
try:
|
||||
from test_autotag_batch import BatchTestCase, _absolute, _strings
|
||||
except ImportError:
|
||||
from tests.test_autotag_batch import BatchTestCase, _absolute, _strings
|
||||
|
||||
from database.execution_history_schema import rotate_history_epoch
|
||||
from py import model_gate
|
||||
from py.api import autotag_routes, image_tag_jobs
|
||||
from py.config import AutotagConfig
|
||||
|
||||
START = "/prompt_manager/autotag/start"
|
||||
SINGLE = "/prompt_manager/autotag/single"
|
||||
APPLY = "/prompt_manager/autotag/apply"
|
||||
SUGGEST = "/prompt_manager/image-annotations/suggest"
|
||||
OBSERVE = "/prompt_manager/image-annotations/observe"
|
||||
TAGS = "/prompt_manager/image-annotations/tags"
|
||||
WAIT = 30.0
|
||||
LEGACY_TITLE = "Tag output images (legacy page)"
|
||||
|
||||
|
||||
def frames_of(text):
|
||||
return [
|
||||
json.loads(line[len("data: ") :])
|
||||
for line in text.splitlines()
|
||||
if line.startswith("data: ")
|
||||
]
|
||||
|
||||
|
||||
class LegacyTestCase(BatchTestCase):
|
||||
async def legacy(self, query="model_type=wd14-vit&skip_tagged=true"):
|
||||
response = await self.client.post(f"{START}?{query}")
|
||||
return response, frames_of(await response.text())
|
||||
|
||||
def digest(self, path):
|
||||
return hashlib.sha256((self.output_dir / path).read_bytes()).hexdigest()
|
||||
|
||||
async def single(self, body):
|
||||
response = await self.client.post(SINGLE, json=body)
|
||||
return response.status, await response.json()
|
||||
|
||||
async def apply(self, body):
|
||||
response = await self.client.post(APPLY, json=body)
|
||||
return response.status, await response.json()
|
||||
|
||||
def snapshot_prompts(self):
|
||||
conn = sqlite3.connect(self._temp_db.name)
|
||||
try:
|
||||
return [
|
||||
conn.execute(f"SELECT * FROM {t} ORDER BY 1,2").fetchall()
|
||||
for t in ("prompts", "prompt_tags")
|
||||
]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class TestLegacyBatch(LegacyTestCase):
|
||||
async def test_the_legacy_batch_tags_each_image_and_relays_frames(self):
|
||||
prompt = self.prompt("one prompt", tags=["beach"])
|
||||
a = self.image("a.png", (1, 2, 3), sees=["bird"])
|
||||
b = self.image("b.png", (4, 5, 6), sees=["waves"])
|
||||
self.link(prompt, a)
|
||||
self.link(prompt, b)
|
||||
before = self.snapshot_prompts()
|
||||
ticks = iter(range(1000))
|
||||
with patch.object(
|
||||
autotag_routes, "_monotonic", side_effect=lambda: next(ticks)
|
||||
):
|
||||
response, frames = await self.legacy()
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(self.stored("a.png")["tags"], ["bird"])
|
||||
self.assertEqual(self.stored("b.png")["tags"], ["waves"])
|
||||
self.assertEqual(self.snapshot_prompts(), before)
|
||||
phases = [f["status"] for f in frames if f.get("progress") in (0, 5, 8)]
|
||||
self.assertEqual(
|
||||
phases,
|
||||
sorted(
|
||||
phases,
|
||||
key=[
|
||||
"Taking a safety backup…",
|
||||
"Loading model…",
|
||||
"Listing output images…",
|
||||
].index,
|
||||
),
|
||||
)
|
||||
self.assertIn("Listing output images…", phases)
|
||||
rows = [f for f in frames if f["type"] == "progress" and "processed" in f]
|
||||
self.assertTrue(rows)
|
||||
for row in rows:
|
||||
self.assertEqual(
|
||||
set(row),
|
||||
{"type", "progress", "status", "processed", "tagged", "skipped"},
|
||||
)
|
||||
self.assertTrue(10 <= row["progress"] <= 95, row)
|
||||
self.assertEqual(
|
||||
frames[-1],
|
||||
{
|
||||
"type": "complete",
|
||||
"progress": 100,
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 0,
|
||||
"errors": 0,
|
||||
"skipped_outside_roots": 0,
|
||||
"status": "Complete",
|
||||
"model_status": "Model kept in memory",
|
||||
"model_loaded": True,
|
||||
},
|
||||
)
|
||||
job = self.api._jobs.list()[0]
|
||||
self.assertEqual(
|
||||
(job.title, job.kind, job.state), (LEGACY_TITLE, "autotag", "finished")
|
||||
)
|
||||
self.assertEqual([s for s in _strings(frames) if _absolute(s)], [])
|
||||
|
||||
# Skip: the marked images are skipped before the model runs.
|
||||
self.tagger.calls.clear()
|
||||
_, again = await self.legacy()
|
||||
self.assertEqual(again[-1]["skipped"], 2)
|
||||
self.assertEqual(again[-1]["processed"], 0)
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
|
||||
async def test_no_still_images_and_keep_in_memory_false(self):
|
||||
_, frames = await self.legacy("model_type=wd14-vit&keep_in_memory=false")
|
||||
self.assertEqual(frames[-1]["status"], "No still images found")
|
||||
self.assertEqual(frames[-1]["model_loaded"], False)
|
||||
self.assertEqual(frames[-1]["model_status"], "Model unloaded")
|
||||
self.assertEqual(self.tagger.unload_calls, 1)
|
||||
|
||||
async def test_language_model_options_come_from_the_page_or_the_template(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
await self.legacy("model_type=gguf&prompt=Tags%20only")
|
||||
self.assertEqual(self.tagger.calls[-1]["prompt"], "Tags only")
|
||||
self.assertEqual(self.tagger.calls[-1]["limit"], 10)
|
||||
self.image("b.png", (2, 2, 2))
|
||||
await self.relist()
|
||||
await self.legacy("model_type=gguf&prompt=%20")
|
||||
self.assertEqual(self.tagger.calls[-1]["prompt"], AutotagConfig.TEMPLATE)
|
||||
self.assertEqual(self.tagger.custom_prompt, "legacy prompt")
|
||||
|
||||
async def test_each_refusal_is_one_error_frame(self):
|
||||
self.tagger.downloaded["wd14-vit"] = False
|
||||
_, frames = await self.legacy()
|
||||
self.assertEqual(len(frames), 1)
|
||||
self.assertEqual(frames[0]["type"], "error")
|
||||
self.assertEqual(frames[0]["status"], "error")
|
||||
self.assertIn("WD14 ViT is not downloaded", frames[0]["message"])
|
||||
self.tagger.downloaded["wd14-vit"] = True
|
||||
|
||||
_, frames = await self.legacy("model_type=wd14-vit&general_threshold=2")
|
||||
self.assertEqual(
|
||||
frames,
|
||||
[
|
||||
{
|
||||
"type": "error",
|
||||
"status": "error",
|
||||
"message": "general_threshold must be between 0 and 1",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
owner = model_gate.new_owner("image-annotations")
|
||||
self.assertTrue(model_gate.try_acquire(owner))
|
||||
try:
|
||||
_, frames = await self.legacy()
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
self.assertEqual(frames[0]["message"], autotag_routes.MODEL_BUSY_ERROR)
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
response = await self.client.post(f"{START}?general_threshold=abc")
|
||||
self.assertEqual(response.status, 400)
|
||||
|
||||
async def test_busy_jobs_refuse_the_legacy_batch(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
status, data = await self.start_batch()
|
||||
self.assertEqual(status, 202, data)
|
||||
job = self.api._jobs.get(data["job"]["id"])
|
||||
try:
|
||||
await self.wait_for(entered)
|
||||
_, frames = await self.legacy()
|
||||
self.assertEqual(
|
||||
frames,
|
||||
[
|
||||
{
|
||||
"type": "error",
|
||||
"status": "error",
|
||||
"message": "Tag output images already running",
|
||||
}
|
||||
],
|
||||
)
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
|
||||
# A second legacy batch keeps its JSON 409.
|
||||
self.image("b.png", (2, 2, 2))
|
||||
await self.relist()
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
first = asyncio.ensure_future(
|
||||
self.legacy("model_type=wd14-vit&skip_tagged=false")
|
||||
)
|
||||
try:
|
||||
await self.wait_for(entered)
|
||||
second = await self.client.post(f"{START}?model_type=wd14-vit")
|
||||
self.assertEqual(second.status, 409)
|
||||
self.assertEqual(
|
||||
await second.json(),
|
||||
{"success": False, "error": "Auto-tagging is already running"},
|
||||
)
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
_, frames = await first
|
||||
self.assertEqual(frames[-1]["type"], "complete")
|
||||
|
||||
async def test_a_legacy_start_that_loses_the_race_gets_the_json_409(self):
|
||||
# Two starts can both pass the title check before either registers its
|
||||
# job; the loser's busy refusal names the legacy batch and answers the
|
||||
# same JSON 409 as a second legacy batch.
|
||||
refusal = image_tag_jobs.Refusal(
|
||||
409,
|
||||
"busy",
|
||||
f"{autotag_routes.LEGACY_BATCH_TITLE} already running",
|
||||
web.json_response({}, status=409),
|
||||
)
|
||||
|
||||
async def refused(**_kwargs):
|
||||
return None, refusal
|
||||
|
||||
with patch.object(self.api, "_start_autotag", side_effect=refused):
|
||||
response = await self.client.post(f"{START}?model_type=wd14-vit")
|
||||
self.assertEqual(response.status, 409)
|
||||
self.assertEqual(
|
||||
await response.json(),
|
||||
{"success": False, "error": "Auto-tagging is already running"},
|
||||
)
|
||||
|
||||
async def test_closing_the_stream_cancels_the_job(self):
|
||||
for n in range(3):
|
||||
self.image(f"{n}.png", (n, n, n))
|
||||
real_write = web.StreamResponse.write
|
||||
|
||||
async def flaky_write(response, data):
|
||||
if b'"processed"' in data:
|
||||
raise ConnectionResetError("Cannot write to closing transport")
|
||||
return await real_write(response, data)
|
||||
|
||||
with patch.object(web.StreamResponse, "write", flaky_write):
|
||||
try:
|
||||
response = await self.client.post(f"{START}?model_type=wd14-vit")
|
||||
await response.read()
|
||||
except Exception: # nosec B110 - the client sees the dropped stream
|
||||
pass
|
||||
job = self.api._jobs.list()[0]
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertLess(job.counters["tagged"], 3)
|
||||
|
||||
async def test_a_quiet_job_repeats_its_last_frame(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
with patch.object(autotag_routes, "LEGACY_REPEAT_AFTER", 0.05):
|
||||
run = asyncio.ensure_future(self.legacy())
|
||||
await self.wait_for(entered)
|
||||
await asyncio.sleep(0.3)
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
_, frames = await run
|
||||
statuses = [f.get("status") for f in frames]
|
||||
repeated = [s for s in statuses if s and statuses.count(s) > 1]
|
||||
self.assertTrue(repeated, statuses)
|
||||
self.assertEqual(frames[-1]["type"], "complete")
|
||||
|
||||
async def test_a_cancel_from_activity_ends_with_a_cancelled_frame(self):
|
||||
for n in range(2):
|
||||
self.image(f"{n}.png", (n, n, n))
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
run = asyncio.ensure_future(self.legacy())
|
||||
try:
|
||||
await self.wait_for(entered)
|
||||
job = self.api._jobs.active("autotag")
|
||||
await self.client.post(f"/prompt_manager/jobs/{job.id}/cancel")
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
_, frames = await run
|
||||
self.assertEqual(
|
||||
frames[-1],
|
||||
{
|
||||
"type": "cancelled",
|
||||
"message": "Cancelled. Tags already added were kept.",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class TestLegacySingleAndApply(LegacyTestCase):
|
||||
async def test_single_suggests_on_a_capture_and_apply_tags_only_that_image(self):
|
||||
prompt = self.prompt("shared prompt", tags=["keep"])
|
||||
a = self.image("a.png", (1, 2, 3), sees=["bird", "sky"])
|
||||
b = self.image("b.png", (4, 5, 6), sees=["cat"])
|
||||
image_a = self.link(prompt, a)
|
||||
self.link(prompt, b)
|
||||
self.saved_set("b.png", b.read_bytes(), ["hand"])
|
||||
before = self.snapshot_prompts()
|
||||
status, data = await self.single(
|
||||
{"image_id": image_a, "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertEqual(status, 200, data)
|
||||
self.assertEqual(data["tags"], ["bird", "sky"])
|
||||
self.assertEqual(data["sha256"], self.digest("a.png"))
|
||||
self.assertEqual(data["image_tags"], [])
|
||||
self.assertEqual(data["prompt_id"], prompt)
|
||||
self.assertEqual(self.tagger.custom_prompt, "legacy prompt")
|
||||
# Inference read the private capture, not the file.
|
||||
self.assertNotIn(str(self.output_dir), self.tagger.read_paths[-1])
|
||||
status, applied = await self.apply(
|
||||
{"image_id": image_a, "sha256": data["sha256"], "tags": ["bird"]}
|
||||
)
|
||||
self.assertEqual(
|
||||
(status, applied),
|
||||
(200, {"success": True, "added_tags": ["bird"], "total_tags": 1}),
|
||||
)
|
||||
self.assertEqual(self.stored("a.png")["tags"], ["bird"])
|
||||
self.assertEqual(self.stored("b.png")["tags"], ["hand"])
|
||||
self.assertEqual(self.snapshot_prompts(), before)
|
||||
self.assertEqual(self.marked(), {self.locator("a.png")})
|
||||
|
||||
async def test_skip_answers_already_tagged_without_the_gate(self):
|
||||
a = self.image("a.png", (1, 2, 3), sees=["bird"])
|
||||
await self.run_batch()
|
||||
self.tagger.calls.clear()
|
||||
owner = model_gate.new_owner("image-tag-job")
|
||||
self.assertTrue(model_gate.try_acquire(owner))
|
||||
try:
|
||||
status, data = await self.single(
|
||||
{
|
||||
"path": "a.png",
|
||||
"root_index": 0,
|
||||
"existing": "skip",
|
||||
"model_type": "wd14-vit",
|
||||
}
|
||||
)
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
self.assertEqual(status, 200, data)
|
||||
self.assertEqual(data["already_tagged"], True)
|
||||
self.assertEqual(data["image_tags"], ["bird"])
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
# Hand tags alone do not count; merge always suggests.
|
||||
b = self.image("b.png", (9, 9, 9))
|
||||
self.saved_set("b.png", b.read_bytes(), ["hand"])
|
||||
status, data = await self.single(
|
||||
{"path": str(b), "existing": "skip", "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertNotIn("already_tagged", data)
|
||||
self.assertEqual(data["image_tags"], ["hand"])
|
||||
status, data = await self.single(
|
||||
{"path": str(a), "existing": "merge", "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertNotIn("already_tagged", data)
|
||||
|
||||
async def test_single_uses_the_saved_cap_and_the_refusal_table(self):
|
||||
self.image("a.png", (1, 2, 3))
|
||||
AutotagConfig.SUGGESTION_LIMIT = 3
|
||||
status, data = await self.single(
|
||||
{"path": "a.png", "root_index": 0, "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertEqual((status, len(data["tags"])), (200, 3))
|
||||
(self.output_dir / "broken.png").write_bytes(b"not an image")
|
||||
await self.relist()
|
||||
status, data = await self.single(
|
||||
{"path": "broken.png", "root_index": 0, "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (415, "not-a-still"))
|
||||
self.assertIn(self.locator("broken.png"), self.marked())
|
||||
self.tagger.downloaded["gguf"] = False
|
||||
status, data = await self.single(
|
||||
{"path": "a.png", "root_index": 0, "model_type": "gguf"}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (409, "model-unavailable"))
|
||||
self.api._capture_store().max_file = 16
|
||||
status, data = await self.single(
|
||||
{"path": "a.png", "root_index": 0, "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (413, "too-large"))
|
||||
self.api._capture_store().max_file = 2**20
|
||||
owner = model_gate.new_owner("image-annotations")
|
||||
self.assertTrue(model_gate.try_acquire(owner))
|
||||
try:
|
||||
status, data = await self.single(
|
||||
{"path": "a.png", "root_index": 0, "model_type": "wd14-vit"}
|
||||
)
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
self.assertEqual((status, data["code"]), (409, "model-busy"))
|
||||
|
||||
async def test_apply_body_and_answer_table(self):
|
||||
a = self.image("a.png", (1, 2, 3))
|
||||
digest = self.digest("a.png")
|
||||
status, data = await self.apply({"prompt_id": 1, "tags": ["a"]})
|
||||
self.assertEqual(
|
||||
(status, data["error"]), (400, "This page is out of date. Reload it.")
|
||||
)
|
||||
status, _ = await self.apply(
|
||||
{"path": "a.png", "root_index": 0, "sha256": "x", "tags": ["a"]}
|
||||
)
|
||||
self.assertEqual(status, 400)
|
||||
status, _ = await self.apply(
|
||||
{"path": "a.png", "root_index": 0, "sha256": digest, "tags": ["bad,name"]}
|
||||
)
|
||||
self.assertEqual(status, 400)
|
||||
status, _ = await self.apply({"path": "a.png", "sha256": digest, "tags": []})
|
||||
self.assertEqual(status, 403) # a bare relative path is not inside a root
|
||||
# An empty reviewed list only marks the image.
|
||||
status, data = await self.apply({"path": str(a), "sha256": digest, "tags": []})
|
||||
self.assertEqual((status, data["added_tags"]), (200, []))
|
||||
self.assertEqual(self.marked(), {self.locator("a.png")})
|
||||
# The file changed since its tags were generated.
|
||||
status, data = await self.apply(
|
||||
{"path": str(a), "sha256": "0" * 64, "tags": ["x"]}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (409, "file-changed"))
|
||||
# Too many tags.
|
||||
full = self.image("full.png", (7, 7, 7))
|
||||
self.saved_set("full.png", full.read_bytes(), [f"t{i}" for i in range(100)])
|
||||
status, data = await self.apply(
|
||||
{"path": str(full), "sha256": self.digest("full.png"), "tags": ["one_more"]}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (409, "tag-limit"))
|
||||
# Gone.
|
||||
status, data = await self.apply(
|
||||
{"path": "gone.png", "root_index": 0, "sha256": digest, "tags": ["x"]}
|
||||
)
|
||||
self.assertIn(status, (403, 404))
|
||||
|
||||
async def test_apply_after_a_restore_writes_nothing(self):
|
||||
a = self.image("a.png", (1, 2, 3))
|
||||
digest = self.digest("a.png")
|
||||
real = self.api.db.get_execution_epoch
|
||||
|
||||
def stale_epoch():
|
||||
epoch = real()
|
||||
conn = sqlite3.connect(self._temp_db.name)
|
||||
try:
|
||||
rotate_history_epoch(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
return epoch
|
||||
|
||||
with patch.object(self.api.db, "get_execution_epoch", side_effect=stale_epoch):
|
||||
status, data = await self.apply(
|
||||
{"path": str(a), "sha256": digest, "tags": ["x"]}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (409, "storage-changed"))
|
||||
self.assertEqual(self.stored("a.png")["tags"], [])
|
||||
|
||||
async def test_image_id_and_path_bodies_reach_the_same_bytes(self):
|
||||
prompt = self.prompt("p")
|
||||
a = self.image("sub/a.png", (1, 2, 3), sees=["x"])
|
||||
image_id = self.link(prompt, a)
|
||||
_, by_id = await self.single({"image_id": image_id, "model_type": "wd14-vit"})
|
||||
_, by_path = await self.single(
|
||||
{"path": "sub/a.png", "root_index": 0, "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertEqual(by_id["sha256"], by_path["sha256"])
|
||||
status, _ = await self.apply(
|
||||
{"image_id": image_id, "sha256": by_id["sha256"], "tags": ["x"]}
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
status, data = await self.apply(
|
||||
{
|
||||
"path": "sub/a.png",
|
||||
"root_index": 0,
|
||||
"sha256": by_path["sha256"],
|
||||
"tags": ["y"],
|
||||
}
|
||||
)
|
||||
self.assertEqual((status, data["total_tags"]), (200, 2))
|
||||
|
||||
|
||||
class TestDialogMarks(LegacyTestCase):
|
||||
"""DEC4: a Tag image dialog save after Generate suggestions marks the image."""
|
||||
|
||||
def target(self, path):
|
||||
return {
|
||||
"root_index": 0,
|
||||
"root_scope": self.api._gallery_root_scope(self.output_dir, 0),
|
||||
"path": path,
|
||||
}
|
||||
|
||||
async def observe(self, path):
|
||||
response = await self.client.post(OBSERVE, json={"target": self.target(path)})
|
||||
data = await response.json()
|
||||
self.assertEqual(response.status, 200, data)
|
||||
return data
|
||||
|
||||
async def suggest(self, observed, path):
|
||||
response = await self.client.post(
|
||||
SUGGEST,
|
||||
json={
|
||||
"target": self.target(path),
|
||||
"capture_token": observed["capture"]["token"],
|
||||
"expected_epoch": observed["library_epoch"],
|
||||
"model_type": "wd14-vit",
|
||||
"options": {"general_threshold": 0.35, "character_threshold": 0.85},
|
||||
},
|
||||
)
|
||||
self.assertEqual(response.status, 200, await response.text())
|
||||
|
||||
async def save(self, observed, path, tags, revision=None):
|
||||
response = await self.client.put(
|
||||
TAGS,
|
||||
json={
|
||||
"target": self.target(path),
|
||||
"capture_token": observed["capture"]["token"],
|
||||
"sha256": observed["capture"]["sha256"],
|
||||
"expected_epoch": observed["library_epoch"],
|
||||
"expected_tag_revision": (
|
||||
observed["tag_revision"] if revision is None else revision
|
||||
),
|
||||
"tags": tags,
|
||||
},
|
||||
)
|
||||
return response.status, await response.json()
|
||||
|
||||
async def test_save_after_generate_marks_and_a_skip_batch_leaves_it(self):
|
||||
self.image("d.png", (1, 2, 3), sees=["bird", "sky"])
|
||||
observed = await self.observe("d.png")
|
||||
await self.suggest(observed, "d.png")
|
||||
status, _ = await self.save(observed, "d.png", ["sky"]) # bird removed
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(self.marked(), {self.locator("d.png")})
|
||||
job = await self.run_batch(existing="skip")
|
||||
self.assertEqual(self.rows(job, "skipped"), [("d.png", "Already auto-tagged")])
|
||||
self.assertEqual(self.stored("d.png")["tags"], ["sky"])
|
||||
|
||||
async def test_save_without_generate_does_not_mark(self):
|
||||
self.image("e.png", (1, 2, 3))
|
||||
observed = await self.observe("e.png")
|
||||
status, _ = await self.save(observed, "e.png", ["hand"])
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(self.marked(), set())
|
||||
|
||||
async def test_a_conflict_then_a_saved_retry_marks(self):
|
||||
f = self.image("f.png", (1, 2, 3))
|
||||
observed = await self.observe("f.png")
|
||||
await self.suggest(observed, "f.png")
|
||||
self.saved_set("f.png", f.read_bytes(), ["other"]) # another writer
|
||||
status, data = await self.save(observed, "f.png", ["mine"])
|
||||
self.assertEqual((status, data["code"]), (409, "tag-conflict"))
|
||||
self.assertEqual(self.marked(), set())
|
||||
status, _ = await self.save(
|
||||
observed, "f.png", ["mine"], revision=data["tag_revision"]
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(self.marked(), {self.locator("f.png")})
|
||||
|
||||
async def test_another_capture_does_not_inherit_the_mark(self):
|
||||
self.image("g.png", (1, 2, 3))
|
||||
self.image("h.png", (4, 5, 6))
|
||||
first = await self.observe("g.png")
|
||||
await self.suggest(first, "g.png")
|
||||
second = await self.observe("h.png")
|
||||
status, _ = await self.save(second, "h.png", ["x"])
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(self.marked(), set())
|
||||
self.assertIn(first["capture"]["token"], self.api._suggested_tokens)
|
||||
|
||||
async def test_gone_tokens_are_dropped_on_the_next_insert_without_touching_clocks(
|
||||
self,
|
||||
):
|
||||
self.image("i.png", (1, 2, 3))
|
||||
self.image("j.png", (4, 5, 6))
|
||||
first = await self.observe("i.png")
|
||||
await self.suggest(first, "i.png")
|
||||
store = self.api._capture_store()
|
||||
store.release(first["capture"]["token"])
|
||||
second = await self.observe("j.png")
|
||||
seen = []
|
||||
real_get = store.get
|
||||
with patch.object(
|
||||
store, "get", side_effect=lambda t: seen.append(t) or real_get(t)
|
||||
):
|
||||
await self.suggest(second, "j.png")
|
||||
self.assertEqual(seen, [])
|
||||
self.assertEqual(self.api._suggested_tokens, {second["capture"]["token"]})
|
||||
@@ -1,318 +0,0 @@
|
||||
"""Auto-tagged marks schema group (#379): install, validation and restore."""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.models import PromptModel
|
||||
|
||||
try: # guarded so a missing module fails assertions, not test loading
|
||||
from database.autotag_schema import (
|
||||
AUTOTAG_TABLE,
|
||||
autotag_schema_error,
|
||||
autotag_without_annotations_error,
|
||||
migrate_autotag_state,
|
||||
)
|
||||
except ImportError:
|
||||
AUTOTAG_TABLE = None
|
||||
autotag_schema_error = None
|
||||
autotag_without_annotations_error = None
|
||||
migrate_autotag_state = None
|
||||
|
||||
TABLE = "media_image_autotags"
|
||||
DDL = (
|
||||
f"CREATE TABLE {TABLE} (byte_id TEXT PRIMARY KEY "
|
||||
"REFERENCES media_annotation_bytes(id) ON DELETE CASCADE) WITHOUT ROWID"
|
||||
)
|
||||
ANNOTATION_TABLES = (
|
||||
"media_annotation_metadata",
|
||||
"media_annotation_files",
|
||||
"media_annotation_bytes",
|
||||
"media_image_tags",
|
||||
)
|
||||
REVIEW_TABLES = (
|
||||
"media_review_metadata",
|
||||
"media_review_items",
|
||||
"media_review_decisions",
|
||||
)
|
||||
FILE_ID = "0f8e4b2a-1c3d-4e5f-8a9b-0c1d2e3f4a5b"
|
||||
BYTE_ID = "6a1b2c3d-4e5f-4a6b-8c9d-0e1f2a3b4c5d"
|
||||
OTHER_ID = "7b2c3d4e-5f6a-4b7c-9d0e-1f2a3b4c5d6e"
|
||||
DIGEST = hashlib.sha256(b"img").hexdigest()
|
||||
NOW = "2026-10-27T00:00:00.000000+00:00"
|
||||
NEEDS_ANNOTATIONS = "Image auto-tag marks need the image annotation tables"
|
||||
INVALID_VALUES = "Image auto-tag marks contain invalid values"
|
||||
INVALID_SCHEMA = "Image auto-tag marks have an invalid schema"
|
||||
|
||||
|
||||
def _compact(sql):
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def catalog(conn):
|
||||
return {
|
||||
(kind, name, _compact(sql))
|
||||
for kind, name, sql in conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name=? OR tbl_name=?",
|
||||
(TABLE, TABLE),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def marks(conn):
|
||||
return [tuple(row) for row in conn.execute(f"SELECT byte_id FROM {TABLE}")]
|
||||
|
||||
|
||||
def insert_byte(conn, file_id=FILE_ID, byte_id=BYTE_ID, locator="/out/img.png"):
|
||||
conn.execute(
|
||||
"INSERT INTO media_annotation_files VALUES(?,?,?)", (file_id, locator, NOW)
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO media_annotation_bytes VALUES(?,?,?,?,?,?,?,0)",
|
||||
(byte_id, file_id, DIGEST, 3, 1_700_000_000_000_000_000, NOW, NOW),
|
||||
)
|
||||
|
||||
|
||||
class AutotagSchemaTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.addCleanup(assert_closed, self, self.temp.name)
|
||||
self.addCleanup(PromptModel.reset_schema_cache)
|
||||
self.models = []
|
||||
self.addCleanup(self.close_models)
|
||||
self.path = str(Path(self.temp.name) / "live.db")
|
||||
|
||||
def close_models(self):
|
||||
for model in self.models:
|
||||
model.close_all()
|
||||
|
||||
def model(self, path=None):
|
||||
model = PromptModel(path or self.path)
|
||||
self.models.append(model)
|
||||
return model
|
||||
|
||||
def require_module(self):
|
||||
self.assertIsNotNone(
|
||||
autotag_schema_error, "database.autotag_schema is not implemented"
|
||||
)
|
||||
|
||||
def raw(self, path, *statements, foreign_keys=False):
|
||||
with closing(sqlite3.connect(path)) as raw, raw:
|
||||
raw.execute(f"PRAGMA foreign_keys={'ON' if foreign_keys else 'OFF'}")
|
||||
for statement in statements:
|
||||
if isinstance(statement, tuple):
|
||||
raw.execute(*statement)
|
||||
else:
|
||||
raw.execute(statement)
|
||||
|
||||
def closed_current(self, name="current.db", *, with_mark=True):
|
||||
"""A closed, current database with one byte record (and its mark)."""
|
||||
path = str(Path(self.temp.name) / name)
|
||||
model = self.model(path)
|
||||
with model.get_connection() as conn:
|
||||
insert_byte(conn)
|
||||
if with_mark:
|
||||
conn.execute(f"INSERT INTO {TABLE} VALUES(?)", (BYTE_ID,))
|
||||
model.close_all()
|
||||
return path
|
||||
|
||||
def test_fresh_install_has_exactly_the_pinned_table(self):
|
||||
self.require_module()
|
||||
self.assertEqual(AUTOTAG_TABLE, TABLE)
|
||||
model = self.model()
|
||||
with model.get_connection() as conn:
|
||||
# WITHOUT ROWID with a primary key: no sqlite_autoindex row.
|
||||
self.assertEqual(catalog(conn), {("table", TABLE, _compact(DDL))})
|
||||
self.assertEqual(
|
||||
[tuple(row) for row in conn.execute(f"PRAGMA table_info({TABLE})")],
|
||||
[(0, "byte_id", "TEXT", 1, None, 1)],
|
||||
)
|
||||
self.assertIsNone(autotag_schema_error(conn, allow_missing=False))
|
||||
self.assertEqual(marks(conn), [])
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
|
||||
def test_reopen_and_rerun_are_idempotent(self):
|
||||
self.require_module()
|
||||
path = self.closed_current()
|
||||
for _ in range(2):
|
||||
PromptModel.reset_schema_cache()
|
||||
model = self.model(path)
|
||||
with model.get_connection() as conn:
|
||||
migrate_autotag_state(conn)
|
||||
self.assertEqual(marks(conn), [(BYTE_ID,)])
|
||||
self.assertEqual(catalog(conn), {("table", TABLE, _compact(DDL))})
|
||||
model.close_all()
|
||||
|
||||
def test_old_database_without_the_table_gains_it_and_keeps_its_rows(self):
|
||||
self.require_module()
|
||||
path = self.closed_current(with_mark=False)
|
||||
self.raw(path, f"DROP TABLE {TABLE}")
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
before = raw.execute("SELECT * FROM media_annotation_bytes").fetchall()
|
||||
self.assertIsNone(autotag_schema_error(raw))
|
||||
self.assertEqual(
|
||||
autotag_schema_error(raw, allow_missing=False),
|
||||
"Missing image auto-tag marks schema",
|
||||
)
|
||||
self.assertTrue(PromptModel.verify_database_file(path)[0])
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(path, require_library_state=True),
|
||||
(False, "Missing image auto-tag marks schema"),
|
||||
)
|
||||
PromptModel.reset_schema_cache()
|
||||
model = self.model(path)
|
||||
with model.get_connection() as conn:
|
||||
self.assertEqual(catalog(conn), {("table", TABLE, _compact(DDL))})
|
||||
self.assertEqual(
|
||||
[
|
||||
tuple(row)
|
||||
for row in conn.execute("SELECT * FROM media_annotation_bytes")
|
||||
],
|
||||
before,
|
||||
)
|
||||
model.close_all()
|
||||
|
||||
def test_table_without_annotation_group_is_refused(self):
|
||||
self.require_module()
|
||||
path = self.closed_current(with_mark=False)
|
||||
self.raw(
|
||||
path,
|
||||
*(
|
||||
f"DROP TABLE {t}"
|
||||
for t in REVIEW_TABLES + tuple(reversed(ANNOTATION_TABLES))
|
||||
),
|
||||
)
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertEqual(autotag_without_annotations_error(raw), NEEDS_ANNOTATIONS)
|
||||
for allow_missing in (True, False):
|
||||
self.assertEqual(
|
||||
autotag_schema_error(raw, allow_missing=allow_missing),
|
||||
NEEDS_ANNOTATIONS,
|
||||
)
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(path), (False, NEEDS_ANNOTATIONS)
|
||||
)
|
||||
PromptModel.reset_schema_cache()
|
||||
with self.assertRaises(sqlite3.DatabaseError) as raised:
|
||||
self.model(path)
|
||||
self.assertIn(NEEDS_ANNOTATIONS, str(raised.exception))
|
||||
|
||||
def test_invalid_values_are_refused_without_repair(self):
|
||||
self.require_module()
|
||||
cases = {
|
||||
"not-a-uuid": (f"INSERT INTO {TABLE} VALUES('not-a-uuid')",),
|
||||
"upper-uuid": ((f"INSERT INTO {TABLE} VALUES(?)", (OTHER_ID.upper(),)),),
|
||||
"dangling": ((f"INSERT INTO {TABLE} VALUES(?)", (OTHER_ID,)),),
|
||||
}
|
||||
for name, statements in cases.items():
|
||||
with self.subTest(name):
|
||||
path = self.closed_current(f"{name}.db", with_mark=False)
|
||||
self.raw(path, *statements)
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertEqual(autotag_schema_error(raw), INVALID_VALUES)
|
||||
before = marks(raw)
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(path), (False, INVALID_VALUES)
|
||||
)
|
||||
PromptModel.reset_schema_cache()
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
self.model(path)
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertEqual(marks(raw), before)
|
||||
|
||||
def test_altered_definitions_are_refused(self):
|
||||
self.require_module()
|
||||
cases = {
|
||||
"rowid": f"CREATE TABLE {TABLE} (byte_id TEXT PRIMARY KEY "
|
||||
"REFERENCES media_annotation_bytes(id) ON DELETE CASCADE)",
|
||||
"restrict": f"CREATE TABLE {TABLE} (byte_id TEXT PRIMARY KEY "
|
||||
"REFERENCES media_annotation_bytes(id) ON DELETE RESTRICT) WITHOUT ROWID",
|
||||
"extra column": f"CREATE TABLE {TABLE} (byte_id TEXT PRIMARY KEY "
|
||||
"REFERENCES media_annotation_bytes(id) ON DELETE CASCADE, engine TEXT) "
|
||||
"WITHOUT ROWID",
|
||||
}
|
||||
for name, ddl in cases.items():
|
||||
with self.subTest(name):
|
||||
path = self.closed_current(f"{name}.db", with_mark=False)
|
||||
self.raw(path, f"DROP TABLE {TABLE}", ddl)
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertEqual(autotag_schema_error(raw), INVALID_SCHEMA)
|
||||
self.assertFalse(PromptModel.verify_database_file(path)[0])
|
||||
path = self.closed_current("index.db", with_mark=False)
|
||||
self.raw(path, f"CREATE INDEX idx_extra ON {TABLE}(byte_id)")
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertEqual(autotag_schema_error(raw), INVALID_SCHEMA)
|
||||
|
||||
def test_deleting_the_byte_record_cascades_to_its_mark(self):
|
||||
self.require_module()
|
||||
model = self.model()
|
||||
with model.get_connection() as conn:
|
||||
insert_byte(conn)
|
||||
conn.execute(f"INSERT INTO {TABLE} VALUES(?)", (BYTE_ID,))
|
||||
conn.execute(f"INSERT OR IGNORE INTO {TABLE} VALUES(?)", (BYTE_ID,))
|
||||
self.assertEqual(marks(conn), [(BYTE_ID,)])
|
||||
conn.execute("DELETE FROM media_annotation_files")
|
||||
self.assertEqual(marks(conn), [])
|
||||
|
||||
def test_restore_of_a_pre_change_backup_migrates(self):
|
||||
self.require_module()
|
||||
src = self.closed_current("backup.db", with_mark=False)
|
||||
self.raw(src, f"DROP TABLE {TABLE}")
|
||||
live = self.model()
|
||||
live.restore_from_file(src)
|
||||
with closing(sqlite3.connect(self.path)) as raw:
|
||||
self.assertEqual(catalog(raw), {("table", TABLE, _compact(DDL))})
|
||||
self.assertEqual(
|
||||
raw.execute("SELECT id FROM media_annotation_bytes").fetchall(),
|
||||
[(BYTE_ID,)],
|
||||
)
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
with closing(sqlite3.connect(src)) as raw: # the source is untouched
|
||||
self.assertEqual(catalog(raw), set())
|
||||
|
||||
def test_install_failure_rolls_back(self):
|
||||
self.require_module()
|
||||
src = self.closed_current("copy.db", with_mark=False)
|
||||
self.raw(src, f"DROP TABLE {TABLE}")
|
||||
path = str(Path(self.temp.name) / "rollback.db")
|
||||
shutil.copyfile(src, path)
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
raw.execute("BEGIN")
|
||||
raw.execute("CREATE TABLE marker(x)")
|
||||
# An invalid group already present: nothing created, outer work kept.
|
||||
raw.execute(f"CREATE TABLE {TABLE}(byte_id TEXT)")
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
migrate_autotag_state(raw)
|
||||
self.assertEqual(
|
||||
raw.execute(
|
||||
f"SELECT sql FROM sqlite_master WHERE name='{TABLE}'"
|
||||
).fetchone()[0],
|
||||
f"CREATE TABLE {TABLE}(byte_id TEXT)",
|
||||
)
|
||||
self.assertIsNotNone(
|
||||
raw.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE name='marker'"
|
||||
).fetchone()
|
||||
)
|
||||
raw.rollback()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,335 +0,0 @@
|
||||
"""Browser-test resets seed a scratch copy and publish it in one commit (#384)."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
from tests.browser import fast_reset
|
||||
from tests.browser.fast_reset import ScratchSeeder, table_digests
|
||||
|
||||
|
||||
def digests(db: PromptDatabase) -> dict:
|
||||
return table_digests(db.model.get_connection())
|
||||
|
||||
|
||||
def add_prompt(db: PromptDatabase, text: str) -> int:
|
||||
with db.model.get_connection() as conn:
|
||||
return conn.execute(
|
||||
"INSERT INTO prompts (text, hash, created_at, updated_at) "
|
||||
"VALUES (?, ?, '2026-01-01 00:00:00', '2026-01-01 00:00:00')",
|
||||
(text, hashlib.sha256(text.encode()).hexdigest()),
|
||||
).lastrowid
|
||||
|
||||
|
||||
def add_image(db: PromptDatabase, prompt_id: int, name: str) -> int:
|
||||
with db.model.get_connection() as conn:
|
||||
return conn.execute(
|
||||
"INSERT INTO generated_images (prompt_id, image_path, filename, "
|
||||
"generation_time) VALUES (?, ?, ?, '2026-01-01 00:00:00')",
|
||||
(prompt_id, f"/out/{name}", name),
|
||||
).lastrowid
|
||||
|
||||
|
||||
def reseed(api) -> str:
|
||||
"""A seeder in the old style: clears its tables, then many small commits."""
|
||||
db = api.db
|
||||
with db.model.get_connection() as conn:
|
||||
for table in ("generated_images", "prompt_tags", "tags", "prompts"):
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for index in range(30):
|
||||
prompt_id = add_prompt(db, f"seeded {index}")
|
||||
add_image(db, prompt_id, f"seeded-{index}.png")
|
||||
with db.model.get_connection() as conn:
|
||||
tag_id = conn.execute(
|
||||
"INSERT INTO tags (name) VALUES (?)", (f"tag{index}",)
|
||||
).lastrowid
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_tags (prompt_id, tag_id) VALUES (?, ?)",
|
||||
(prompt_id, tag_id),
|
||||
)
|
||||
return "manifest"
|
||||
|
||||
|
||||
class TestScratchSeeder(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.root = Path(self.tmp.name)
|
||||
self.addCleanup(assert_closed, self, self.tmp.name)
|
||||
self.live = self.database("live.db")
|
||||
self.api = types.SimpleNamespace(db=self.live, marker="live api")
|
||||
self.seeder = ScratchSeeder(self.api, self.root / "scratch", memory_root=None)
|
||||
self.addCleanup(self.seeder.close)
|
||||
# Rows from "earlier tests": AUTOINCREMENT positions have moved on.
|
||||
for index in range(5):
|
||||
add_image(self.live, add_prompt(self.live, f"old {index}"), f"o{index}")
|
||||
with self.live.model.get_connection() as conn:
|
||||
conn.execute("DELETE FROM prompts WHERE text = 'old 4'")
|
||||
|
||||
def database(self, name: str) -> PromptDatabase:
|
||||
db = PromptDatabase(str(self.root / name))
|
||||
self.addCleanup(db.close_all)
|
||||
return db
|
||||
|
||||
def oracle(self, seed):
|
||||
"""Run ``seed`` the old way, on a copy of the live database."""
|
||||
path = self.root / "oracle.db"
|
||||
target = sqlite3.connect(path)
|
||||
try:
|
||||
self.live.model.get_connection().backup(target)
|
||||
finally:
|
||||
target.close()
|
||||
db = self.database("oracle.db")
|
||||
seed(types.SimpleNamespace(db=db))
|
||||
return digests(db)
|
||||
|
||||
def test_publishes_exactly_what_the_seed_writes_on_live(self):
|
||||
expected = self.oracle(reseed)
|
||||
self.assertEqual(self.seeder.run(reseed), "manifest")
|
||||
self.assertEqual(digests(self.live), expected)
|
||||
with self.live.model.get_connection() as conn:
|
||||
first = conn.execute("SELECT MIN(id) FROM prompts").fetchone()[0]
|
||||
self.assertEqual(first, 6, "AUTOINCREMENT continues as on live")
|
||||
|
||||
def test_runs_twice_from_the_state_the_first_run_left(self):
|
||||
self.seeder.run(reseed)
|
||||
expected = self.oracle(reseed)
|
||||
self.seeder.run(reseed)
|
||||
self.assertEqual(digests(self.live), expected)
|
||||
|
||||
def test_tables_the_seed_did_not_change_keep_their_rows(self):
|
||||
before = digests(self.live)
|
||||
|
||||
def rename(api):
|
||||
with api.db.model.get_connection() as conn:
|
||||
conn.execute("UPDATE prompts SET text = 'renamed' WHERE id = 1")
|
||||
|
||||
self.seeder.run(rename)
|
||||
after = digests(self.live)
|
||||
changed = {table for table in after if after[table] != before[table]}
|
||||
self.assertEqual(changed, {"prompts"})
|
||||
with self.live.model.get_connection() as conn:
|
||||
images = conn.execute("SELECT COUNT(*) FROM generated_images").fetchone()
|
||||
# Replacing prompts' rows never cascades into generated_images.
|
||||
self.assertEqual(images[0], 4)
|
||||
|
||||
def test_the_seed_sees_the_scratch_database_and_the_live_api(self):
|
||||
seen = {}
|
||||
|
||||
def seed(api):
|
||||
seen["db"] = api.db
|
||||
seen["marker"] = api.marker
|
||||
api.note = "set by the seed"
|
||||
|
||||
self.seeder.run(seed)
|
||||
self.assertIsNot(seen["db"], self.live)
|
||||
self.assertNotEqual(seen["db"].model.db_path, self.live.model.db_path)
|
||||
self.assertEqual(seen["marker"], "live api")
|
||||
self.assertEqual(self.api.note, "set by the seed")
|
||||
self.assertIs(self.api.db, self.live)
|
||||
|
||||
def test_cache_drops_wait_until_the_rows_are_published(self):
|
||||
seen = []
|
||||
|
||||
def invalidate_gallery_cache(retire=False):
|
||||
with self.live.model.get_connection() as conn:
|
||||
count = conn.execute("SELECT COUNT(*) FROM tags").fetchone()[0]
|
||||
seen.append((count, retire))
|
||||
|
||||
self.api.invalidate_gallery_cache = invalidate_gallery_cache
|
||||
|
||||
def seed(api):
|
||||
reseed(api)
|
||||
api.invalidate_gallery_cache(retire=True)
|
||||
self.assertEqual(seen, [], "not dropped while live has the old rows")
|
||||
|
||||
self.seeder.run(seed)
|
||||
self.assertEqual(seen, [(30, True)])
|
||||
|
||||
def test_a_failing_seed_drops_no_cache(self):
|
||||
seen = []
|
||||
self.api.invalidate_gallery_cache = lambda: seen.append(True)
|
||||
|
||||
def broken(api):
|
||||
api.invalidate_gallery_cache()
|
||||
raise ValueError("seed failed")
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
self.seeder.run(broken)
|
||||
self.assertEqual(seen, [])
|
||||
|
||||
def test_a_failing_seed_leaves_live_untouched(self):
|
||||
before = digests(self.live)
|
||||
|
||||
def broken(api):
|
||||
reseed(api)
|
||||
raise ValueError("seed failed")
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "seed failed"):
|
||||
self.seeder.run(broken)
|
||||
self.assertEqual(digests(self.live), before)
|
||||
self.assertIs(self.api.db, self.live)
|
||||
|
||||
def test_a_concurrent_live_commit_fails_the_reset(self):
|
||||
other = self.database("live.db")
|
||||
|
||||
def racing(api):
|
||||
reseed(api)
|
||||
add_prompt(other, "written to live meanwhile")
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "changed while"):
|
||||
self.seeder.run(racing)
|
||||
with self.live.model.get_connection() as conn:
|
||||
texts = [row[0] for row in conn.execute("SELECT text FROM prompts")]
|
||||
self.assertIn("written to live meanwhile", texts)
|
||||
self.assertNotIn("seeded 0", texts)
|
||||
|
||||
def test_a_write_open_during_the_seed_and_committed_later_fails_it(self):
|
||||
other = self.database("live.db")
|
||||
began, finished = threading.Event(), []
|
||||
|
||||
def write():
|
||||
with other.model.get_connection() as conn:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
conn.execute(
|
||||
"INSERT INTO prompts (text, hash) VALUES ('concurrent', 'c')"
|
||||
)
|
||||
began.set()
|
||||
time.sleep(0.5) # the publish waits on this lock meanwhile
|
||||
other.model.close()
|
||||
finished.append(True)
|
||||
|
||||
writer = threading.Thread(target=write)
|
||||
|
||||
def racing(api):
|
||||
reseed(api)
|
||||
writer.start()
|
||||
self.assertTrue(began.wait(10))
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "changed while"):
|
||||
self.seeder.run(racing)
|
||||
writer.join()
|
||||
self.assertEqual(finished, [True])
|
||||
with self.live.model.get_connection() as conn:
|
||||
texts = [row[0] for row in conn.execute("SELECT text FROM prompts")]
|
||||
self.assertIn("concurrent", texts)
|
||||
self.assertNotIn("seeded 0", texts)
|
||||
self.assert_live_connection_restored()
|
||||
|
||||
def test_a_failed_publish_leaves_live_and_its_connection_as_they_were(self):
|
||||
before = digests(self.live)
|
||||
columns = fast_reset._columns
|
||||
|
||||
def broken_columns(conn, table, *schema):
|
||||
# The publish asks for the live table's columns without a schema.
|
||||
return columns(conn, table, *schema) if schema else ["no_such_column"]
|
||||
|
||||
with mock.patch.object(fast_reset, "_columns", broken_columns):
|
||||
with self.assertRaises(sqlite3.OperationalError):
|
||||
self.seeder.run(reseed)
|
||||
self.assertEqual(digests(self.live), before)
|
||||
self.assert_live_connection_restored()
|
||||
|
||||
def assert_live_connection_restored(self):
|
||||
conn = self.live.model.get_connection()
|
||||
self.assertFalse(conn.in_transaction)
|
||||
self.assertEqual(conn.execute("PRAGMA foreign_keys").fetchone()[0], 1)
|
||||
names = [row[1] for row in conn.execute("PRAGMA database_list")]
|
||||
self.assertEqual(names, ["main"])
|
||||
|
||||
def test_the_live_connection_is_left_as_it_was(self):
|
||||
self.seeder.run(reseed)
|
||||
conn = self.live.model.get_connection()
|
||||
self.assertFalse(conn.in_transaction)
|
||||
self.assertEqual(conn.execute("PRAGMA foreign_keys").fetchone()[0], 1)
|
||||
names = [row[1] for row in conn.execute("PRAGMA database_list")]
|
||||
self.assertEqual(names, ["main"])
|
||||
self.assertEqual(conn.execute("PRAGMA foreign_key_check").fetchall(), [])
|
||||
|
||||
def test_on_live_mode_runs_the_seed_on_the_live_api(self):
|
||||
seeder = ScratchSeeder(
|
||||
self.api, self.root / "unused", on_live=True, memory_root=self.root
|
||||
)
|
||||
self.addCleanup(seeder.close)
|
||||
expected = self.oracle(reseed)
|
||||
seen = []
|
||||
self.assertEqual(
|
||||
seeder.run(lambda api: seen.append(api) or reseed(api)), "manifest"
|
||||
)
|
||||
self.assertIs(seen[0], self.api)
|
||||
self.assertEqual(digests(self.live), expected)
|
||||
self.assertFalse((self.root / "unused").exists())
|
||||
|
||||
def test_the_scratch_lives_under_the_memory_root_when_there_is_one(self):
|
||||
memory = self.root / "memory"
|
||||
memory.mkdir()
|
||||
seeder = ScratchSeeder(self.api, self.root / "fallback", memory_root=memory)
|
||||
self.addCleanup(seeder.close)
|
||||
expected = self.oracle(reseed)
|
||||
seeder.run(reseed)
|
||||
self.assertEqual(digests(self.live), expected)
|
||||
self.assertEqual(len(list(memory.glob("pm-reset-*/seed.db"))), 1)
|
||||
self.assertFalse((self.root / "fallback").exists())
|
||||
seeder.close()
|
||||
self.assertEqual(list(memory.iterdir()), [])
|
||||
|
||||
def test_a_full_memory_root_puts_the_scratch_on_disk(self):
|
||||
memory = self.root / "memory"
|
||||
memory.mkdir()
|
||||
seeder = ScratchSeeder(self.api, self.root / "fallback", memory_root=memory)
|
||||
self.addCleanup(seeder.close)
|
||||
full = mock.patch.object(
|
||||
fast_reset.shutil,
|
||||
"disk_usage",
|
||||
return_value=types.SimpleNamespace(free=fast_reset.MEMORY_FREE_BYTES - 1),
|
||||
)
|
||||
with full:
|
||||
seeder.run(reseed)
|
||||
self.assertEqual(list(memory.iterdir()), [])
|
||||
self.assertTrue((self.root / "fallback" / "seed.db").exists())
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "process probe is POSIX only")
|
||||
def test_scratch_folders_of_dead_servers_are_removed(self):
|
||||
memory = self.root / "memory"
|
||||
memory.mkdir()
|
||||
dead = subprocess.Popen([sys.executable, "-c", "pass"])
|
||||
dead.wait()
|
||||
orphan = memory / f"pm-reset-{dead.pid}-abc"
|
||||
alive = memory / f"pm-reset-{os.getpid()}-def"
|
||||
other = memory / "pm-reset-notapid"
|
||||
for folder in (orphan, alive, other):
|
||||
folder.mkdir()
|
||||
(folder / "seed.db").write_bytes(b"x")
|
||||
seeder = ScratchSeeder(self.api, self.root / "fallback", memory_root=memory)
|
||||
self.addCleanup(seeder.close)
|
||||
seeder.run(reseed)
|
||||
self.assertFalse(orphan.exists())
|
||||
self.assertTrue(alive.exists())
|
||||
self.assertTrue(other.exists())
|
||||
own = [p for p in memory.glob(f"pm-reset-{os.getpid()}-*") if p != alive]
|
||||
self.assertEqual(len(own), 1)
|
||||
|
||||
def test_close_releases_the_scratch_files(self):
|
||||
self.seeder.run(reseed)
|
||||
self.seeder.close()
|
||||
assert_closed(self, self.root / "scratch")
|
||||
self.seeder.close() # idempotent
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,67 +0,0 @@
|
||||
"""Browser-test media folders are emptied by a rename, deleted later (#384)."""
|
||||
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from tests.browser.folder_trash import TRASH_NAME, close_trash, replace_folder
|
||||
|
||||
|
||||
class TestFolderTrash(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.root = Path(self.tmp.name)
|
||||
self.addCleanup(close_trash, self.root)
|
||||
|
||||
def trash_threads(self):
|
||||
return [t for t in threading.enumerate() if t.name == "pm-reset-trash"]
|
||||
|
||||
def test_replaces_the_folder_with_an_empty_one(self):
|
||||
folder = self.root / "output"
|
||||
(folder / "nested").mkdir(parents=True)
|
||||
(folder / "nested" / "image.png").write_bytes(b"png")
|
||||
replace_folder(folder)
|
||||
self.assertTrue(folder.is_dir())
|
||||
self.assertEqual(list(folder.iterdir()), [])
|
||||
close_trash(self.root)
|
||||
self.assertFalse((self.root / TRASH_NAME).exists())
|
||||
self.assertEqual(self.trash_threads(), [])
|
||||
|
||||
def test_a_missing_folder_is_created_without_a_trash(self):
|
||||
replace_folder(self.root / "input")
|
||||
self.assertTrue((self.root / "input").is_dir())
|
||||
self.assertFalse((self.root / TRASH_NAME).exists())
|
||||
self.assertEqual(self.trash_threads(), [])
|
||||
|
||||
def test_repeated_replacements_share_one_thread_and_all_are_removed(self):
|
||||
folder = self.root / "output"
|
||||
for number in range(5):
|
||||
folder.mkdir(exist_ok=True)
|
||||
(folder / f"{number}.png").write_bytes(b"png")
|
||||
replace_folder(folder)
|
||||
self.assertEqual(len(self.trash_threads()), 1)
|
||||
close_trash(self.root)
|
||||
self.assertEqual(list(folder.iterdir()), [])
|
||||
self.assertFalse((self.root / TRASH_NAME).exists())
|
||||
close_trash(self.root) # idempotent
|
||||
|
||||
|
||||
class TestFolderTrashAfterClose(unittest.TestCase):
|
||||
def test_a_replacement_after_close_deletes_in_place_without_a_thread(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
close_trash(root)
|
||||
folder = root / "output"
|
||||
folder.mkdir()
|
||||
(folder / "late.png").write_bytes(b"png")
|
||||
replace_folder(folder)
|
||||
self.assertEqual(list(folder.iterdir()), [])
|
||||
self.assertFalse((root / TRASH_NAME).exists())
|
||||
names = [t.name for t in threading.enumerate()]
|
||||
self.assertNotIn("pm-reset-trash", names)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -221,12 +221,12 @@ class TestSlowResetOnTheServer(unittest.TestCase):
|
||||
self.assertRegex(
|
||||
text, r"kernel: asyncio_\d+ \(0x[0-9a-f]{16}\) state \w, waiting in "
|
||||
)
|
||||
# The plain reset seeds a scratch copy (#384), then waits to publish it.
|
||||
self.assertRegex(text, r"line \d+ in _publish\n")
|
||||
# The plain reset's seed() waits on its first DELETE.
|
||||
self.assertRegex(text, r"line \d+ in seed\n")
|
||||
self.assertIn("POST /__test__/reset answered 200 after", text)
|
||||
self.assertRegex(
|
||||
text,
|
||||
r"executor call ScratchSeeder.run: queued \d+\.\d+s, ran \d+\.\d+s",
|
||||
r"executor call create_app.<locals>.seed: queued \d+\.\d+s, ran \d+\.\d+s",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1398,6 +1398,50 @@ class TestModelHousekeeping(DatabaseTestCase):
|
||||
self.assertIsNot(self.db.model.get_connection(), failing)
|
||||
|
||||
|
||||
class TestAutotagBatchCandidates(DatabaseTestCase):
|
||||
"""#328: the batch's candidate read never loads workflow documents."""
|
||||
|
||||
def test_autotag_candidates_skip_workflow_data(self):
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
first = self.db.save_prompt(
|
||||
text="a", tags=["one", "two"], prompt_hash=generate_prompt_hash("a")
|
||||
)
|
||||
second = self.db.save_prompt(text="b", prompt_hash=generate_prompt_hash("b"))
|
||||
old = self.db.link_image_to_prompt(
|
||||
first, "/out/a.png", {"workflow": {"big": "x" * 1000}}
|
||||
)
|
||||
new = self.db.link_image_to_prompt(second, "/out/b.png")
|
||||
statements = []
|
||||
conn = self.db.model.get_connection() # this thread's own connection
|
||||
conn.set_trace_callback(statements.append)
|
||||
try:
|
||||
rows = self.db.autotag_batch_candidates()
|
||||
finally:
|
||||
conn.set_trace_callback(None)
|
||||
self.assertEqual(
|
||||
rows[0],
|
||||
{
|
||||
"id": new,
|
||||
"image_path": "/out/b.png",
|
||||
"prompt_id": second,
|
||||
"prompt_tags": [],
|
||||
},
|
||||
)
|
||||
self.assertEqual(
|
||||
{**rows[1], "prompt_tags": sorted(rows[1]["prompt_tags"])},
|
||||
{
|
||||
"id": old,
|
||||
"image_path": "/out/a.png",
|
||||
"prompt_id": first,
|
||||
"prompt_tags": ["one", "two"],
|
||||
},
|
||||
)
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertTrue(statements)
|
||||
self.assertFalse(any("workflow_data" in sql for sql in statements))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
@@ -468,7 +468,6 @@ class TestRestoreFromFile(BackupTestCase):
|
||||
|
||||
def test_restore_old_backup_without_annotations(self):
|
||||
tables = (
|
||||
"media_image_autotags",
|
||||
"media_review_decisions",
|
||||
"media_review_items",
|
||||
"media_review_metadata",
|
||||
|
||||
@@ -356,8 +356,6 @@ class HistoryHooksTests(unittest.TestCase):
|
||||
self.assertGreaterEqual(time.monotonic() - began, 1.8)
|
||||
self.assertLess(time.monotonic() - began, CEILING_SECONDS)
|
||||
release.set()
|
||||
# Drain first: the 2 s bound is not under test here (#370).
|
||||
self.drain(registration)
|
||||
self.assertTrue(registration.dispose(2))
|
||||
|
||||
def test_restore_disables_capture_and_clears_handles_before_fresh_epoch(self):
|
||||
|
||||
@@ -399,10 +399,6 @@ class OutputHooksTests(fixtures.unittest.TestCase):
|
||||
self.queue.task_done(1, history("rejected.png"), fixtures.status())
|
||||
registration.writer.invalidate()
|
||||
release.set()
|
||||
# Let the writer work through the 1,024 queued commands first
|
||||
# (#370's pattern): on a busy Windows runner that took longer
|
||||
# than dispose's 2 s bound (merge queue, #382).
|
||||
self.drain(registration)
|
||||
self.assertTrue(registration.dispose(2))
|
||||
self.assertEqual(self.outputs(0)["state"], "not_captured")
|
||||
self.assertEqual(len(self.rows()), 1)
|
||||
|
||||
@@ -28,11 +28,6 @@ except ImportError:
|
||||
# Upper bound a blocked fake write waits for its release. Tests prove ordering
|
||||
# with events, so this only caps a failing run and never decides a pass.
|
||||
BLOCK_SECONDS = 10
|
||||
# A restore is a full verify/migrate/backup/replace cycle and is bound by the
|
||||
# disk: a busy Windows runner took over 10 s (merge queue, #382). Waits that
|
||||
# span a restore use this ceiling, which stays under the runner's 120 s hang
|
||||
# watchdog, so a real hang still fails with the thread dump.
|
||||
RESTORE_SECONDS = 90
|
||||
|
||||
|
||||
class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
@@ -322,8 +317,8 @@ class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
|
||||
Returns the patch and an event set once that rotation has returned on
|
||||
the worker. A restore is a full verify/migrate/backup/replace cycle, so
|
||||
the handoff waits on this event with the ``RESTORE_SECONDS`` ceiling
|
||||
instead of the short worker-behavior deadline (TESTING.md,
|
||||
the handoff waits on this event with the generous ``BLOCK_SECONDS``
|
||||
ceiling instead of the short worker-behavior deadline (TESTING.md,
|
||||
Windows portability preflight).
|
||||
"""
|
||||
backup = os.path.join(self.directory.name, f"restore-{start.run_id}.db")
|
||||
@@ -366,7 +361,7 @@ class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
)
|
||||
with rotation:
|
||||
self.finalization(writer, start)
|
||||
self.assertTrue(rotated.wait(RESTORE_SECONDS))
|
||||
self.assertTrue(rotated.wait(BLOCK_SECONDS))
|
||||
self.wait_for(lambda: writer.status().pending == 0)
|
||||
self.assertEqual(
|
||||
self.db.get_execution_outputs(start.run_id)["state"],
|
||||
@@ -408,7 +403,7 @@ class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
self.assertEqual(writer.status().pending, 1)
|
||||
self.assertEqual(writer.context_budget.charged_bytes, charge)
|
||||
release.set()
|
||||
self.assertTrue(rotated.wait(RESTORE_SECONDS))
|
||||
self.assertTrue(rotated.wait(BLOCK_SECONDS))
|
||||
self.wait_for(lambda: writer.status().pending == 0)
|
||||
self.assertEqual(writer.context_budget.charged_bytes, 0)
|
||||
self.assertFalse(writer.status().accepting)
|
||||
@@ -572,8 +567,6 @@ class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
self.assertIs(replacement.context_budget, writer.context_budget)
|
||||
self.assertIsNone(replacement.context_budget.reserve(8388608))
|
||||
release.set()
|
||||
# Drain first: the 2 s bound is not under test here (#370).
|
||||
drain_writer(writer)
|
||||
self.assertTrue(writer.close(2))
|
||||
finally:
|
||||
release.set()
|
||||
@@ -615,8 +608,6 @@ class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
writer.invalidate()
|
||||
self.assertEqual(writer.context_budget.charged_bytes, 0)
|
||||
release.set()
|
||||
# Drain first: the 2 s bound is not under test here (#370).
|
||||
drain_writer(writer)
|
||||
self.assertTrue(writer.close(2))
|
||||
|
||||
def test_malformed_context_omission_preserves_independent_valid_output(self):
|
||||
@@ -707,8 +698,6 @@ class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
release.set()
|
||||
invalidator.join(BLOCK_SECONDS)
|
||||
self.assertFalse(invalidator.is_alive())
|
||||
# Drain first: the 2 s bound is not under test here (#370).
|
||||
drain_writer(writer)
|
||||
self.assertTrue(writer.close(2))
|
||||
self.assertEqual(
|
||||
self.db.get_execution_outputs(start.run_id)["state"], "complete"
|
||||
|
||||
@@ -11,7 +11,6 @@ engine is B2's fake service.
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import unittest
|
||||
@@ -691,32 +690,6 @@ class TestFixRound(ImageTagJobTestCase):
|
||||
],
|
||||
)
|
||||
|
||||
async def test_an_ended_job_shows_no_time_left(self):
|
||||
"""#379 live check: a finished or cancelled job never says "min left"."""
|
||||
for index in range(3):
|
||||
self.png(f"f{index}.png", color=(index, 7, 7))
|
||||
clock = iter([0.0, 0.0, 120.0, 240.0])
|
||||
with patch(
|
||||
"py.api.image_tag_jobs._monotonic", side_effect=lambda: next(clock, 240.0)
|
||||
):
|
||||
job = await self.run_job(self.auto_body([f"f{i}.png" for i in range(3)]))
|
||||
self.assertEqual(job.state, "finished")
|
||||
self.assertEqual(job.status_line, "Auto-tag finished · 3 files processed")
|
||||
|
||||
self.png("g0.png", color=(1, 1, 1))
|
||||
self.png("g1.png", color=(2, 2, 2))
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
job = await self.start_ok(self.auto_body(["g0.png", "g1.png"]))
|
||||
await self.wait_for(entered)
|
||||
await self.client.post(f"/prompt_manager/jobs/{job.id}/cancel")
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.status_line, "Auto-tag cancelled · 1 file processed")
|
||||
self.assertNotIn("left", job.status_line)
|
||||
|
||||
def maximal_target(self, number):
|
||||
"""A distinct target whose path is 1024 non-ASCII code points long."""
|
||||
head = f"{number:03d}/" + ("界" * 200 + "/") * 5
|
||||
@@ -829,108 +802,3 @@ class TestFixRound(ImageTagJobTestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestFilterStart(ImageTagJobTestCase):
|
||||
"""#392: "Select all N matching" starts the job with the listing filter."""
|
||||
|
||||
async def test_filter_start_tags_every_matching_file_in_listing_order(self):
|
||||
for number in range(6):
|
||||
path = self.png(f"bulk/{number:02}.png", color=(number, 1, 1))
|
||||
os.utime(path, (100 + number, 100 + number))
|
||||
self.png("other/x.png")
|
||||
|
||||
job = await self.run_job(
|
||||
{
|
||||
"mode": "add",
|
||||
"filter": {"subfolder": "bulk", "sort": "oldest"},
|
||||
"tags": ["picked"],
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(job.scope, {"mode": "add", "images": 6})
|
||||
self.assertEqual(job.counters["tagged"], 6)
|
||||
self.assertEqual(
|
||||
[path for path, _detail in self.rows(job, "tagged")],
|
||||
[f"bulk/{number:02}.png" for number in range(6)],
|
||||
)
|
||||
self.assertEqual(self.stored("bulk/03.png")["tags"], ["picked"])
|
||||
self.assertEqual(self.stored("other/x.png")["tags"], [])
|
||||
|
||||
async def test_auto_filter_start_runs_the_model_on_the_matching_files(self):
|
||||
self.png("bulk/a.png")
|
||||
self.png("other/b.png", color=(1, 2, 3))
|
||||
|
||||
job = await self.run_job(
|
||||
{
|
||||
"mode": "auto",
|
||||
"filter": {"subfolder": "bulk"},
|
||||
"model_type": "wd14-vit",
|
||||
"options": dict(WD14),
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(job.scope["images"], 1)
|
||||
self.assertEqual(job.counters["tagged"], 1)
|
||||
self.assertEqual(self.stored("other/b.png")["tags"], [])
|
||||
|
||||
async def test_filter_with_no_match_is_refused_without_a_job(self):
|
||||
self.png("a.png")
|
||||
|
||||
status, data = await self.start(
|
||||
{"mode": "add", "filter": {"subfolder": "missing"}, "tags": ["x"]}
|
||||
)
|
||||
|
||||
self.assertEqual((status, data["code"]), (400, "no-images"))
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
|
||||
async def test_filter_over_the_cap_is_refused_without_a_job(self):
|
||||
from py.api import image_tag_jobs
|
||||
|
||||
for number in range(3):
|
||||
self.png(f"{number}.png", color=(number, 0, 0))
|
||||
with patch.object(image_tag_jobs, "MAX_FILTER_TARGETS", 2):
|
||||
status, data = await self.start(
|
||||
{"mode": "add", "filter": {}, "tags": ["x"]}
|
||||
)
|
||||
|
||||
self.assertEqual((status, data["code"]), (400, "too-many-images"))
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
self.assertEqual(image_tag_jobs.MAX_FILTER_TARGETS, 10_000)
|
||||
|
||||
async def test_filter_bodies_are_validated(self):
|
||||
self.png("a.png")
|
||||
target = self.target("a.png")
|
||||
for body in (
|
||||
{"mode": "add", "filter": {}, "targets": [target], "tags": ["x"]},
|
||||
{"mode": "add", "filter": "all", "tags": ["x"]},
|
||||
{"mode": "add", "filter": {"sort": "sideways"}, "tags": ["x"]},
|
||||
{"mode": "add", "filter": {"cursor": "x"}, "tags": ["x"]},
|
||||
{"mode": "add", "filter": {"root": "7"}, "tags": ["x"]},
|
||||
{"mode": "add", "filter": {}, "tags": []},
|
||||
):
|
||||
with self.subTest(body=body):
|
||||
status, data = await self.start(body)
|
||||
self.assertEqual(status, 400, data)
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
|
||||
async def test_a_filter_whose_source_was_renumbered_is_refused(self):
|
||||
self.png("a.png")
|
||||
scope = self.target("a.png")["root_scope"]
|
||||
|
||||
job = await self.run_job(
|
||||
{"mode": "add", "filter": {"root": "0", "root_scope": scope}, "tags": ["x"]}
|
||||
)
|
||||
self.assertEqual(job.counters["tagged"], 1)
|
||||
status, data = await self.start(
|
||||
{
|
||||
"mode": "add",
|
||||
"filter": {"root": "0", "root_scope": "f" * 64},
|
||||
"tags": ["x"],
|
||||
}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (409, "source-changed"))
|
||||
status, data = await self.start(
|
||||
{"mode": "add", "filter": {"root_scope": scope}, "tags": ["x"]}
|
||||
)
|
||||
self.assertEqual(status, 400, data)
|
||||
|
||||
@@ -535,9 +535,7 @@ class TestHeldJobStates(_LegacyHoldCase):
|
||||
self.assertIn("event: error", text)
|
||||
self.assertEqual(await self._listed(), [("thumbnails", "failed")])
|
||||
|
||||
async def test_legacy_autotag_refused_at_start_leaves_no_job(self):
|
||||
# #379: the legacy batch is the autotag job; a model that is not
|
||||
# downloaded refuses the start (one error frame) and registers nothing.
|
||||
async def test_legacy_autotag_that_ends_in_error_is_failed(self):
|
||||
service = MagicMock()
|
||||
service.get_models_status.return_value = {} # model not downloaded
|
||||
with patch("py.autotag.get_autotag_service", return_value=service):
|
||||
@@ -547,7 +545,7 @@ class TestHeldJobStates(_LegacyHoldCase):
|
||||
text = await resp.text()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertIn('"type": "error"', text)
|
||||
self.assertEqual(await self._listed(), [])
|
||||
self.assertEqual(await self._listed(), [("autotag", "failed")])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
"""The legacy pages' backend contract does not move under System S13 (#330).
|
||||
|
||||
S13 points the legacy admin and gallery buttons at native System and removes
|
||||
their duplicate dialogs, changing no backend file. This snapshot pins what the
|
||||
legacy pages talk to: every registered route (method and path) and the key
|
||||
paths of ``GET /prompt_manager/settings``, the one route whose legacy request
|
||||
body S13 shrinks. Record again only with a reason in the PR body:
|
||||
|
||||
PM_RECORD_LEGACY_SNAPSHOT=1 python3 -m unittest tests.test_legacy_contract_snapshot
|
||||
|
||||
Cutover (#334) removes this test and its fixture.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from tests.test_api_admin import AdminAPITestCase
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_admin import AdminAPITestCase
|
||||
|
||||
from py.config import GalleryConfig # noqa: E402
|
||||
|
||||
SNAPSHOT = (
|
||||
Path(__file__).resolve().parent / "fixtures" / "legacy_contract_snapshot.json"
|
||||
)
|
||||
RECORD_ENV = "PM_RECORD_LEGACY_SNAPSHOT"
|
||||
SETTINGS = "/prompt_manager/settings"
|
||||
DEPTH = 2
|
||||
CHANGED = (
|
||||
"the legacy contract changed; if the change is intended (a new route, a new"
|
||||
" settings key), record it with PM_RECORD_LEGACY_SNAPSHOT=1 and say why in"
|
||||
" the PR body"
|
||||
)
|
||||
|
||||
|
||||
def key_paths(value, prefix="", depth=1):
|
||||
"""The key paths of a JSON object to ``DEPTH`` levels.
|
||||
|
||||
``{"a": {"b": 1}}`` gives ``a`` and ``a.b``; the items of a list of
|
||||
objects give the union of their keys as ``a[].b``.
|
||||
"""
|
||||
paths = set()
|
||||
for key, child in value.items():
|
||||
path = f"{prefix}{key}"
|
||||
paths.add(path)
|
||||
if depth >= DEPTH:
|
||||
continue
|
||||
if isinstance(child, dict):
|
||||
paths |= key_paths(child, f"{path}.", depth + 1)
|
||||
elif isinstance(child, list):
|
||||
for item in child:
|
||||
if isinstance(item, dict):
|
||||
paths |= key_paths(item, f"{path}[].", depth + 1)
|
||||
return paths
|
||||
|
||||
|
||||
class TestLegacyContractSnapshot(AdminAPITestCase):
|
||||
"""Routes and settings keys against tests/fixtures/legacy_contract_snapshot.json.
|
||||
|
||||
The base case isolates config.json (``PROMPT_MANAGER_CONFIG_PATH`` in the
|
||||
temp tree), stubs ComfyUI's ``folder_paths`` and resets the gallery roots;
|
||||
nothing here reads the maintainer's install.
|
||||
"""
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.api._get_all_output_dirs = lambda: [str(self.output_dir)]
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)]
|
||||
|
||||
def _routes(self):
|
||||
return sorted(
|
||||
{
|
||||
f"{route.method} {route.resource.canonical}"
|
||||
for route in self.app.router.routes()
|
||||
if route.method != "HEAD"
|
||||
}
|
||||
)
|
||||
|
||||
async def _settings_keys(self):
|
||||
resp = await self.client.request("GET", SETTINGS)
|
||||
body = await resp.json()
|
||||
self.assertEqual(resp.status, 200, body)
|
||||
return sorted(key_paths(body))
|
||||
|
||||
def _snapshot(self):
|
||||
with open(SNAPSHOT, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
def _record(self, key, value):
|
||||
recorded = self._snapshot() if SNAPSHOT.exists() else {}
|
||||
recorded[key] = value
|
||||
with open(SNAPSHOT, "w", encoding="utf-8", newline="\n") as f:
|
||||
json.dump(recorded, f, indent=2)
|
||||
f.write("\n")
|
||||
|
||||
async def test_routes_match_the_snapshot(self):
|
||||
routes = self._routes()
|
||||
if os.environ.get(RECORD_ENV) == "1":
|
||||
self._record("routes", routes)
|
||||
self.assertGreater(len(routes), 50)
|
||||
self.assertEqual(routes, self._snapshot()["routes"], CHANGED)
|
||||
|
||||
async def test_settings_response_keeps_its_keys(self):
|
||||
keys = await self._settings_keys()
|
||||
if os.environ.get(RECORD_ENV) == "1":
|
||||
self._record("settings_keys", keys)
|
||||
self.assertIn("settings.infinite_scroll", keys)
|
||||
self.assertEqual(keys, self._snapshot()["settings_keys"], CHANGED)
|
||||
|
||||
|
||||
class TestKeyPaths(unittest.TestCase):
|
||||
def test_depth_and_list_items(self):
|
||||
value = {
|
||||
"success": True,
|
||||
"settings": {"a": 1, "deep": {"hidden": 1}},
|
||||
"rows": [{"path": "x"}, {"label": "y"}],
|
||||
}
|
||||
self.assertEqual(
|
||||
sorted(key_paths(value)),
|
||||
[
|
||||
"rows",
|
||||
"rows[].label",
|
||||
"rows[].path",
|
||||
"settings",
|
||||
"settings.a",
|
||||
"settings.deep",
|
||||
"success",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+1
-158
@@ -115,12 +115,7 @@ class TestPairScan(AdminAPITestCase):
|
||||
self.assertEqual(second["pairs_added"], 0)
|
||||
|
||||
def test_legacy_single_prompt_second_pass_not_reported_as_new_link(self):
|
||||
# A sampler-less graph: one encoder wired into a guider (#252).
|
||||
single = {
|
||||
"1": graph()["1"],
|
||||
"2": {"class_type": "BasicGuider", "inputs": {"conditioning": ["1", 0]}},
|
||||
}
|
||||
meta = {"prompt": json.dumps(single)}
|
||||
meta = {"prompt": json.dumps({"1": graph()["1"]})}
|
||||
path = self.output_dir / "old.png"
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, meta), "added")
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, meta), "unchanged")
|
||||
@@ -297,155 +292,3 @@ class TestPairScan(AdminAPITestCase):
|
||||
self.api.db.get_image_prompt_info(str(path))["workflow_data"], workflow
|
||||
)
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, metadata), "unchanged")
|
||||
|
||||
|
||||
def flux_graph(text="a red fox", encoder="CLIPTextEncode"):
|
||||
"""A sampler-less Flux graph: the prompt reaches a guider's conditioning (#252)."""
|
||||
return {
|
||||
"1": {"class_type": "UNETLoader", "inputs": {"unet_name": "flux.safetensors"}},
|
||||
"2": {"class_type": "DualCLIPLoader", "inputs": {"clip_name1": "t5"}},
|
||||
"6": {"class_type": encoder, "inputs": {"text": text, "clip": ["2", 0]}},
|
||||
"22": {
|
||||
"class_type": "BasicGuider",
|
||||
"inputs": {"model": ["1", 0], "conditioning": ["6", 0]},
|
||||
},
|
||||
"13": {
|
||||
"class_type": "SamplerCustomAdvanced",
|
||||
"inputs": {"guider": ["22", 0], "sigmas": ["17", 0]},
|
||||
},
|
||||
"17": {"class_type": "BasicScheduler", "inputs": {"steps": 20}},
|
||||
"9": {"class_type": "SaveImage", "inputs": {"images": ["13", 0]}},
|
||||
}
|
||||
|
||||
|
||||
class TestSamplerLessScan(AdminAPITestCase):
|
||||
"""Graphs where no node takes a positive or negative input (#252)."""
|
||||
|
||||
def _scanned(self, name, prompt_graph, workflow=None):
|
||||
meta = {"prompt": json.dumps(prompt_graph)}
|
||||
if workflow is not None:
|
||||
meta["workflow"] = json.dumps(workflow)
|
||||
return self.api._ingest_scanned_file(self.output_dir / name, meta)
|
||||
|
||||
def _texts(self):
|
||||
return [p["text"] for p in self.api.db.get_recent_prompts()["prompts"]]
|
||||
|
||||
def test_the_one_wired_prompt_is_linked(self):
|
||||
self.assertEqual(self._scanned("flux.png", flux_graph()), "added")
|
||||
self.assertEqual(self._texts(), ["a red fox"])
|
||||
|
||||
def test_a_text_node_feeding_the_encoder_is_one_prompt(self):
|
||||
g = flux_graph(text=["30", 0])
|
||||
g["30"] = {"class_type": "TextInput", "inputs": {"text": "a red fox"}}
|
||||
self.assertEqual(self._scanned("textinput.png", g), "added")
|
||||
self.assertEqual(self._texts(), ["a red fox"])
|
||||
|
||||
def test_a_display_node_that_feeds_nothing_is_never_the_prompt(self):
|
||||
# A real scan stored a "=== GIT CLONE REPORT ===" display text this way.
|
||||
g = flux_graph()
|
||||
g["40"] = {
|
||||
"class_type": "easy showAnything",
|
||||
"inputs": {"text": "=== GIT CLONE REPORT ===\nTotal repositories: 1"},
|
||||
}
|
||||
self.assertEqual(self._scanned("report.png", g), "added")
|
||||
self.assertEqual(self._texts(), ["a red fox"])
|
||||
|
||||
def test_only_unconsumed_texts_store_nothing(self):
|
||||
g = {
|
||||
"40": {"class_type": "easy showAnything", "inputs": {"text": "report"}},
|
||||
"41": {"class_type": "Textbox", "inputs": {"text": "install note"}},
|
||||
}
|
||||
self.assertEqual(self._scanned("notes.png", g), "found")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_two_wired_texts_are_ambiguous_and_stay_unlinked(self):
|
||||
# A LoRA loader's text input is wired too: the scan stored "<lora:...>".
|
||||
g = flux_graph()
|
||||
g["50"] = {
|
||||
"class_type": "Lora Loader (LoraManager)",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"clip": ["2", 0],
|
||||
"text": "<lora:style:1.00>",
|
||||
},
|
||||
}
|
||||
g["22"]["inputs"]["model"] = ["50", 0]
|
||||
g["6"]["inputs"]["clip"] = ["50", 1]
|
||||
self.assertEqual(self._scanned("lora.png", g), "unresolved")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_an_untraceable_wired_text_makes_the_graph_ambiguous(self):
|
||||
# The encoder's text comes from a node no rule reads: never guess the other.
|
||||
g = flux_graph(text=["60", 0])
|
||||
g["60"] = {"class_type": "SomeLLMNode", "inputs": {"seed": 1}}
|
||||
g["50"] = {
|
||||
"class_type": "Lora Loader (LoraManager)",
|
||||
"inputs": {"model": ["1", 0], "text": "<lora:style:1.00>"},
|
||||
}
|
||||
g["22"]["inputs"]["model"] = ["50", 0]
|
||||
self.assertEqual(self._scanned("llm.png", g), "unresolved")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_this_repositorys_prompt_node_is_authoritative(self):
|
||||
g = flux_graph(text="a red fox", encoder="PromptManager")
|
||||
g["50"] = {
|
||||
"class_type": "Lora Loader (LoraManager)",
|
||||
"inputs": {"model": ["1", 0], "text": "<lora:style:1.00>"},
|
||||
}
|
||||
g["22"]["inputs"]["model"] = ["50", 0]
|
||||
self.assertEqual(self._scanned("own.png", g), "added")
|
||||
self.assertEqual(self._texts(), ["a red fox"])
|
||||
|
||||
def test_a_numeric_text_is_never_stored_as_a_prompt(self):
|
||||
self.assertEqual(self._scanned("number.png", flux_graph(text="2048")), "found")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_workflow_widget_values_are_never_read(self):
|
||||
# The API text is linked from a node no rule reads; the UI workflow
|
||||
# still holds a stale widget value, which the old scan stored.
|
||||
g = flux_graph(text=["70", 0])
|
||||
g["70"] = {"class_type": "PromptBuilder", "inputs": {"prompt": "x", "n": 2}}
|
||||
workflow = {
|
||||
"nodes": [
|
||||
{"id": 6, "type": "CLIPTextEncode", "widgets_values": ["stale text"]}
|
||||
]
|
||||
}
|
||||
self.assertEqual(self._scanned("stale.png", g, workflow), "unresolved")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_a_lora_loaders_trigger_word_output_is_not_its_text(self):
|
||||
# The encoder reads the loader's trigger-word output (slot 2), not its
|
||||
# text input: the prompt is unknown, never the "<lora:...>" syntax.
|
||||
g = flux_graph(text=["50", 2])
|
||||
g["50"] = {
|
||||
"class_type": "Lora Loader (LoraManager)",
|
||||
"inputs": {"model": ["1", 0], "text": "<lora:style:1.00>"},
|
||||
}
|
||||
g["22"]["inputs"]["model"] = ["50", 0]
|
||||
self.assertEqual(self._scanned("trigger.png", g), "unresolved")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_a_seeded_text_node_holds_a_template_not_the_prompt(self):
|
||||
g = flux_graph(text=["80", 0])
|
||||
g["80"] = {
|
||||
"class_type": "RandomPrompt",
|
||||
"inputs": {"text": "a {red|blue} fox", "seed": 5},
|
||||
}
|
||||
self.assertEqual(self._scanned("random.png", g), "unresolved")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_a1111_parameters_beside_an_undecided_graph_are_read(self):
|
||||
# Metadata savers write both; the old scan read the A1111 positive.
|
||||
g = flux_graph()
|
||||
g["50"] = {
|
||||
"class_type": "Lora Loader (LoraManager)",
|
||||
"inputs": {"model": ["1", 0], "text": "<lora:style:1.00>"},
|
||||
}
|
||||
g["22"]["inputs"]["model"] = ["50", 0]
|
||||
meta = {
|
||||
"prompt": json.dumps(g),
|
||||
"parameters": "a red fox\nNegative prompt: blur\nSteps: 20",
|
||||
}
|
||||
path = self.output_dir / "saver.png"
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, meta), "added")
|
||||
self.assertEqual(self._texts(), ["a red fox"])
|
||||
|
||||
@@ -371,10 +371,7 @@ class ReviewSchemaTests(unittest.TestCase):
|
||||
self.require_module()
|
||||
for label, dropped in (
|
||||
("annotations only", TABLES),
|
||||
(
|
||||
"predates annotations",
|
||||
("media_image_autotags",) + ANNOTATION_TABLES + TABLES,
|
||||
),
|
||||
("predates annotations", ANNOTATION_TABLES + TABLES),
|
||||
):
|
||||
with self.subTest(label):
|
||||
path = str(Path(self.temp.name) / f"{label.replace(' ', '-')}.db")
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
"""``sampler_less_prompt``: the one wired text of a graph with no sampler (#252)."""
|
||||
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from utils.prompt_pair_resolver import sampler_less_prompt
|
||||
|
||||
|
||||
def _guided(**nodes):
|
||||
"""``nodes`` plus a guider consuming node ``"1"``."""
|
||||
graph = {"9": {"class_type": "BasicGuider", "inputs": {"conditioning": ["1", 0]}}}
|
||||
graph.update(nodes)
|
||||
return graph
|
||||
|
||||
|
||||
class TestSamplerLessPrompt(unittest.TestCase):
|
||||
def test_not_a_graph_or_too_large_is_absent(self):
|
||||
for graph in (None, [], {}, "text"):
|
||||
self.assertEqual(sampler_less_prompt(graph)["state"], "absent")
|
||||
big = {str(i): {"class_type": "X", "inputs": {}} for i in range(3)}
|
||||
self.assertEqual(sampler_less_prompt(big, max_visits=2)["state"], "absent")
|
||||
|
||||
def test_one_literal_text(self):
|
||||
graph = _guided(**{"1": {"class_type": "Enc", "inputs": {"text": " a fox "}}})
|
||||
self.assertEqual(
|
||||
sampler_less_prompt(graph),
|
||||
{"state": "resolved", "text": "a fox", "texts": ["a fox"]},
|
||||
)
|
||||
|
||||
def test_a_number_literal_is_never_a_prompt(self):
|
||||
for value in (2048, "2048", "1.5", " 7 "):
|
||||
graph = _guided(**{"1": {"class_type": "Enc", "inputs": {"text": value}}})
|
||||
self.assertEqual(sampler_less_prompt(graph)["state"], "absent", value)
|
||||
|
||||
def test_a_long_digit_run_with_a_letter_is_text_and_fast(self):
|
||||
text = "9" * 50_000 + "x"
|
||||
graph = _guided(**{"1": {"class_type": "Enc", "inputs": {"text": text}}})
|
||||
started = time.monotonic()
|
||||
self.assertEqual(sampler_less_prompt(graph)["text"], text)
|
||||
self.assertLess(time.monotonic() - started, 1.0)
|
||||
|
||||
def test_number_like_words_are_prompts(self):
|
||||
for value in ("nan", "inf", "Infinity", "1_000"):
|
||||
graph = _guided(**{"1": {"class_type": "Enc", "inputs": {"text": value}}})
|
||||
self.assertEqual(sampler_less_prompt(graph)["text"], value)
|
||||
|
||||
def test_an_own_node_with_unreadable_text_beside_another_text_is_ambiguous(self):
|
||||
graph = _guided(
|
||||
**{
|
||||
"1": {"class_type": "PromptManager", "inputs": {"text": ["5", 0]}},
|
||||
"5": {"class_type": "LLM", "inputs": {"seed": 3}},
|
||||
"2": {"class_type": "Enc", "inputs": {"text": "<lora:x:1>"}},
|
||||
"3": {"class_type": "Sink", "inputs": {"c": ["2", 0]}},
|
||||
}
|
||||
)
|
||||
self.assertEqual(sampler_less_prompt(graph)["state"], "ambiguous")
|
||||
|
||||
def test_unhashable_class_type_is_an_unknown_node_not_a_crash(self):
|
||||
graph = _guided(**{"1": {"class_type": ["x"], "inputs": {"text": "a fox"}}})
|
||||
self.assertEqual(sampler_less_prompt(graph)["text"], "a fox")
|
||||
|
||||
def test_own_negative_node_is_not_authoritative(self):
|
||||
graph = _guided(
|
||||
**{
|
||||
"1": {"class_type": "Enc", "inputs": {"text": "a fox"}},
|
||||
"2": {
|
||||
"class_type": "PromptManagerNegative",
|
||||
"inputs": {"text": "blur"},
|
||||
},
|
||||
"3": {"class_type": "Sink", "inputs": {"c": ["2", 0]}},
|
||||
}
|
||||
)
|
||||
found = sampler_less_prompt(graph)
|
||||
self.assertEqual(found["state"], "ambiguous")
|
||||
self.assertEqual(found["texts"], ["a fox", "blur"])
|
||||
|
||||
def test_own_node_text_wins_over_an_untraceable_text(self):
|
||||
graph = _guided(
|
||||
**{
|
||||
"1": {"class_type": "PromptManagerText", "inputs": {"text": "a fox"}},
|
||||
"2": {"class_type": "Enc", "inputs": {"text": ["5", 0]}},
|
||||
"5": {"class_type": "LLM", "inputs": {"seed": 3}},
|
||||
"6": {"class_type": "Sink", "inputs": {"c": ["2", 0]}},
|
||||
}
|
||||
)
|
||||
self.assertEqual(sampler_less_prompt(graph)["text"], "a fox")
|
||||
|
||||
def test_two_own_texts_are_ambiguous(self):
|
||||
graph = _guided(
|
||||
**{
|
||||
"1": {"class_type": "PromptManager", "inputs": {"text": "a fox"}},
|
||||
"2": {"class_type": "PromptManager", "inputs": {"text": "a cat"}},
|
||||
"3": {"class_type": "Sink", "inputs": {"c": ["2", 0]}},
|
||||
}
|
||||
)
|
||||
self.assertEqual(sampler_less_prompt(graph)["state"], "ambiguous")
|
||||
|
||||
def test_only_an_untraceable_text_is_ambiguous_with_nothing_to_show(self):
|
||||
graph = _guided(
|
||||
**{
|
||||
"1": {"class_type": "Enc", "inputs": {"text": ["5", 0]}},
|
||||
"5": {"class_type": "LLM", "inputs": {"seed": 3}},
|
||||
}
|
||||
)
|
||||
self.assertEqual(
|
||||
sampler_less_prompt(graph),
|
||||
{"state": "ambiguous", "text": None, "texts": []},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Vendored
-21
@@ -28,24 +28,3 @@ for (const name of ['frontend-coverage', 'browser-workflows']) {
|
||||
assert.match(body, /node --version \| grep -q '\^v24\\\.'/, 'the image Node is the Node 24 the project requires');
|
||||
});
|
||||
}
|
||||
|
||||
// The real-host job drives Chromium against a managed ComfyUI. `playwright install --with-deps` pulled the browser's
|
||||
// system packages from Ubuntu's apt mirror on every run, and three runs (#349, #378, #382) crawled at ~26 kB/s until
|
||||
// the job hit its limit after every test had passed. The pinned image ships Chromium and its libraries, so the job runs
|
||||
// there; the image Python must be the reviewed host Python from tools/vendor/comfyui-runtime.json.
|
||||
test('real ComfyUI compatibility runs in the pinned Playwright image on the reviewed host Python', () => {
|
||||
const testWorkflow = readFileSync(new URL('../../.github/workflows/test.yml', import.meta.url), 'utf8');
|
||||
const runtime = JSON.parse(readFileSync(new URL('../../tools/vendor/comfyui-runtime.json', import.meta.url), 'utf8'));
|
||||
const match = testWorkflow.match(/^ {2}comfyui:\n([\s\S]*?)(?=^ {2}[a-z][a-z-]*:\n|(?![\s\S]))/m);
|
||||
assert.ok(match, 'comfyui job exists');
|
||||
const body = match[1];
|
||||
assert.match(body, new RegExp(`^ container:\\n image: mcr\\.microsoft\\.com/playwright:v${version.replaceAll('.', '\\.')}-noble$`, 'm'));
|
||||
assert.doesNotMatch(body, /playwright install/, 'Chromium and its system libraries come from the image, not apt');
|
||||
assert.doesNotMatch(body, /apt-get|actions\/setup-(python|node)@/);
|
||||
assert.match(body, /python3 -m venv --without-pip /, 'a user-space venv on the image Python');
|
||||
assert.match(body, /sha256sum -c/, 'the pip wheel that bootstraps the venv is checksum-pinned');
|
||||
assert.match(body, /node --version \| grep -q '\^v24\\\.'/, 'the image Node is Node 24');
|
||||
// The image Python is checked against the pin at run time, so a pin or image bump that disagrees fails loudly.
|
||||
assert.match(body, /steps\.runtime\.outputs\.pythonVersion/, 'the job compares the image Python with the reviewed pin');
|
||||
assert.match(runtime.pythonVersion, /^3\.12$/, 'the reviewed host Python is the image Python (3.12 on noble)');
|
||||
});
|
||||
|
||||
-213
@@ -1,213 +0,0 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { COLUMNS, CONTROL_EXEMPTIONS, PAGES, REMOVAL_COLUMNS, STATUSES, declaredTitles, duplicateRemovals, idUses, methodUses,
|
||||
parseControl, parseCoverageTable, parseRemovalTable, readCoverageTable, readPage, readRemovalTable, removalProblems,
|
||||
rowProblems } from '../../tools/vendor/legacy-coverage.mjs';
|
||||
|
||||
// TESTING.md "Legacy System capability coverage" (#249 System S12, #329): every legacy System capability names an
|
||||
// owner, a known status, a well-formed native control and a test its file really declares.
|
||||
|
||||
const ROW = Object.freeze({ Capability: 'Clear/truncate logs', 'Legacy anchor': '`admin.html:915`', Owner: 'S12 Logs',
|
||||
Status: 'covered', 'Native control': '`diagnostics` · button · "Clear all logs…"', Test: 'clears the logs (clear.spec.mjs)' });
|
||||
const declaring = source => ({ locate: file => `tests/browser/${file}`, read: () => source });
|
||||
|
||||
test('every coverage row has an owner, a known status and a declared test', () => {
|
||||
const rows = readCoverageTable();
|
||||
assert.ok(rows.length >= 54, `the table lists ${rows.length} capabilities`);
|
||||
for (const row of rows) {
|
||||
assert.deepEqual(rowProblems(row), [], `TESTING.md:${row.line} ${row.Capability}`);
|
||||
assert.ok(row.Owner.trim());
|
||||
assert.ok(STATUSES.includes(row.Status));
|
||||
}
|
||||
assert.ok(rows.some(row => row.Status === 'kept in legacy') && rows.some(row => row.Status === 'approved removal'));
|
||||
// A broken row fails: an empty owner, an unknown status, a test its file does not declare.
|
||||
const [first] = rows;
|
||||
assert.deepEqual(rowProblems({ ...first, Owner: '' }), ['no owner']);
|
||||
assert.ok(rowProblems({ ...first, Status: 'done' }).includes('unknown status "done"'));
|
||||
assert.match(rowProblems({ ...first, Test: first.Test.replace(/^(.+) \(/, '$1 again (') }).join(), /declares no test/);
|
||||
assert.match(rowProblems({ ...first, Test: 'some title (missing.spec.mjs)' }).join(), /no test file missing\.spec\.mjs/);
|
||||
});
|
||||
|
||||
test('a title that appears only in a comment does not count', () => {
|
||||
const commented = [
|
||||
"// test('clears the logs', async () => {});",
|
||||
"/* test('clears the logs', async () => {}); */",
|
||||
'/**',
|
||||
" * it('clears the logs')",
|
||||
' */',
|
||||
"const title = 'clears the logs';",
|
||||
"test.describe('clears the logs', () => {});",
|
||||
].join('\n');
|
||||
assert.equal(declaredTitles(commented, 'clear.spec.mjs').has('clears the logs'), false);
|
||||
assert.deepEqual(rowProblems(ROW, declaring(commented)), ['clear.spec.mjs declares no test "clears the logs"']);
|
||||
const declared = `${commented}\nfor (const theme of THEMES) {\n test('clears the logs', async ({ page }) => {});\n}\n`;
|
||||
assert.deepEqual(rowProblems(ROW, declaring(declared)), []);
|
||||
// Escaped quotes, template literals, modifiers and a title on the next line are declarations too.
|
||||
const titles = declaredTitles("test('a gate\\'s refusal', () => {});\ntest.only(`b`, () => {});\nit(\n 'c', () => {});",
|
||||
'x.test.js');
|
||||
assert.deepEqual([...titles], ["a gate's refusal", 'b', 'c']);
|
||||
// A block comment that starts after code on a line, and a `//` or `/*` inside a string or a regular expression.
|
||||
const trailing = [
|
||||
"const url = 'http://host/*'; /* test('clears the logs', () => {});",
|
||||
"test('clears the logs', () => {}); */",
|
||||
"const pattern = /['\"`]\\/\\*/;",
|
||||
"test('after the pattern', () => {});",
|
||||
].join('\n');
|
||||
assert.deepEqual([...declaredTitles(trailing, 'x.spec.mjs')], ['after the pattern']);
|
||||
// Python: only a real `def test_…` counts, never a commented one.
|
||||
const python = ['# def test_report_lists_config_only_keys(self):', 'class T:', ' \"\"\"Example:', '',
|
||||
' def test_in_a_docstring(self):', ' \"\"\"', ' def test_other(self):', ' def test_nested():',
|
||||
' pass', 'def test_module():', ' pass', ''].join('\n');
|
||||
assert.deepEqual([...declaredTitles(python, 'test_x.py')], ['test_other', 'test_module']);
|
||||
});
|
||||
|
||||
test('a malformed native control cell fails', () => {
|
||||
assert.deepEqual(parseControl('`diagnostics` · button · "Clear all logs…"'),
|
||||
{ section: 'diagnostics', role: 'button', name: 'Clear all logs…', row: null });
|
||||
assert.deepEqual(parseControl('`maintenance` · button · "Review plan" · in "Duplicate prompt text"'),
|
||||
{ section: 'maintenance', role: 'button', name: 'Review plan', row: 'Duplicate prompt text' });
|
||||
assert.equal(parseControl('—'), null);
|
||||
for (const [cell, message] of [
|
||||
['diagnostics · button · "Clear all logs…"', /malformed/],
|
||||
['`diagnostics` · button · Clear all logs…', /malformed/],
|
||||
['`diagnostics` - button - "Clear all logs…"', /malformed/],
|
||||
['', /malformed/],
|
||||
['`logs` · button · "Clear all logs…"', /unknown System section "logs"/],
|
||||
['`diagnostics` · link · "Clear all logs…"', /unknown role "link"/],
|
||||
]) {
|
||||
assert.throws(() => parseControl(cell), message, cell);
|
||||
assert.match(rowProblems({ ...ROW, 'Native control': cell }, declaring("test('clears the logs', () => {});")).join(), message);
|
||||
}
|
||||
// Only a covered row names a control.
|
||||
assert.deepEqual(rowProblems({ ...ROW, Status: 'kept in legacy' }, declaring("test('clears the logs', () => {});")),
|
||||
['a kept in legacy row names a native control']);
|
||||
// The table itself must keep its six columns.
|
||||
assert.throws(() => parseCoverageTable(`### Legacy System capability coverage\n\n| Capability | Owner |\n| --- | --- |\n`), /columns/);
|
||||
assert.deepEqual(COLUMNS, ['Capability', 'Legacy anchor', 'Owner', 'Status', 'Native control', 'Test']);
|
||||
});
|
||||
|
||||
test('a skipped test does not prove a capability', () => {
|
||||
for (const modifier of ['skip', 'fixme', 'fail']) {
|
||||
assert.deepEqual(rowProblems(ROW, declaring(`test.${modifier}('clears the logs', async () => {});`)),
|
||||
['clear.spec.mjs only skips "clears the logs"'], modifier);
|
||||
}
|
||||
for (const modifier of ['only', 'slow']) {
|
||||
assert.deepEqual(rowProblems(ROW, declaring(`test.${modifier}('clears the logs', async () => {});`)), [], modifier);
|
||||
}
|
||||
// A running declaration beside a skipped one counts.
|
||||
assert.deepEqual(rowProblems(ROW, declaring("test.skip('clears the logs', () => {});\ntest('clears the logs', () => {});")), []);
|
||||
});
|
||||
|
||||
test('a covered row names its native control unless it is exempt', () => {
|
||||
const source = declaring("test('clears the logs', () => {});");
|
||||
assert.deepEqual(rowProblems({ ...ROW, 'Native control': '—' }, source), ['a covered row names no native control']);
|
||||
for (const status of ['kept in legacy', 'approved removal']) {
|
||||
assert.deepEqual(rowProblems({ ...ROW, Status: status, 'Native control': '—' }, source), [], status);
|
||||
}
|
||||
const [exempt] = Object.keys(CONTROL_EXEMPTIONS);
|
||||
assert.deepEqual(rowProblems({ ...ROW, Capability: exempt, 'Native control': '—' }, source), []);
|
||||
// Every exemption has a reason and names a covered row of the table that has no control.
|
||||
const rows = readCoverageTable();
|
||||
for (const [capability, reason] of Object.entries(CONTROL_EXEMPTIONS)) {
|
||||
assert.ok(reason.length > 20, capability);
|
||||
const row = rows.find(item => item.Capability === capability);
|
||||
assert.ok(row, `${capability} is in the table`);
|
||||
assert.deepEqual([row.Status, row['Native control']], ['covered', '—']);
|
||||
}
|
||||
});
|
||||
|
||||
// TESTING.md "Legacy System UI removed by S13" (#330): a legacy element is removed only when its capability is
|
||||
// `covered`, the removal copies that row's native test, and the element is really gone from its page.
|
||||
const COVERAGE_HEADER = '| Capability | Legacy anchor | Owner | Status | Native control | Test |\n| --- | --- | --- | --- | --- | --- |';
|
||||
const REMOVAL_HEADER = '| Capability | Page | Removed ids | Removed methods | Native test |\n| --- | --- | --- | --- | --- |';
|
||||
const FIXTURE_COVERAGE = parseCoverageTable(['### Legacy System capability coverage', '', COVERAGE_HEADER,
|
||||
'| View logs | `admin.html:1` | S12 Logs | covered | `diagnostics` · combobox · "Level" | level filter works (logs.spec.mjs) |',
|
||||
'| Infinite scrolling default | `admin.html:2` | Legacy prompt list | kept in legacy | — | scrolls (list-state.test.js) |',
|
||||
''].join('\n'));
|
||||
const removalRow = cells => parseRemovalTable(['### Legacy System UI removed by S13', '', REMOVAL_HEADER, `| ${cells} |`, '']
|
||||
.join('\n'))[0];
|
||||
const EMPTY_PAGE = () => ({ html: '<button id="kept">Kept</button>', script: 'class Admin {\n kept() {}\n}\n' });
|
||||
|
||||
test('every removed legacy element cites a covered capability and its native test', () => {
|
||||
const coverage = readCoverageTable();
|
||||
const rows = readRemovalTable();
|
||||
for (const row of rows) {
|
||||
const cited = coverage.find(item => item.Capability === row.Capability);
|
||||
assert.ok(cited, `TESTING.md:${row.line} cites "${row.Capability}"`);
|
||||
assert.equal(cited.Status, 'covered', `TESTING.md:${row.line} ${row.Capability}`);
|
||||
assert.equal(row['Native test'], cited.Test, `TESTING.md:${row.line} ${row.Capability}`);
|
||||
assert.ok(Object.keys(PAGES).includes(row.Page), `TESTING.md:${row.line} page ${row.Page}`);
|
||||
}
|
||||
assert.deepEqual(duplicateRemovals(rows), []);
|
||||
assert.deepEqual(REMOVAL_COLUMNS, ['Capability', 'Page', 'Removed ids', 'Removed methods', 'Native test']);
|
||||
});
|
||||
|
||||
test('a removed id or method is gone from its page', () => {
|
||||
const coverage = readCoverageTable();
|
||||
for (const row of readRemovalTable()) {
|
||||
assert.deepEqual(removalProblems(row, coverage, page => readPage(page)), [], `TESTING.md:${row.line} ${row.Capability}`);
|
||||
}
|
||||
// Each way a page can still hold an id or a method is found; a call through another object, or a longer name, is not
|
||||
// a definition.
|
||||
const page = {
|
||||
html: '<div id="logsModal"></div><button onclick="window.admin.closeLogs()">x</button>',
|
||||
script: [
|
||||
"document.getElementById('logLevel').value;",
|
||||
"this.showModal('diagnosticsModal');",
|
||||
'<span data-id="logsContainer"></span>',
|
||||
'document.querySelector("#logLimit option");',
|
||||
"const backdrops = ['settingsModal', 'duplicatesModal'];",
|
||||
'class Admin {',
|
||||
' async refreshLogs(level) {',
|
||||
' }',
|
||||
'} displayLogs(logs) {',
|
||||
' }',
|
||||
'this.loadLogStats();',
|
||||
'// downloadLogs() {',
|
||||
'other.formatBytesLater(1);',
|
||||
].join('\n'),
|
||||
};
|
||||
assert.deepEqual(idUses('logsModal', page), ['id="logsModal" in the markup']);
|
||||
assert.deepEqual(idUses('logLevel', page), ["getElementById('logLevel')"]);
|
||||
assert.deepEqual(idUses('logLimit', page), ["querySelector('#logLimit')"]);
|
||||
assert.deepEqual(idUses('duplicatesModal', page), ["'duplicatesModal' in a modal list"]);
|
||||
assert.deepEqual(idUses('logLevelX', page), []);
|
||||
assert.deepEqual(idUses('diagnosticsModal', page), ["a …Modal('diagnosticsModal') call"]);
|
||||
assert.deepEqual(idUses('logsContainer', page), []);
|
||||
assert.deepEqual(methodUses('closeLogs', page), ['.closeLogs( called in the markup']);
|
||||
assert.deepEqual(methodUses('refreshLogs', page), ['refreshLogs() defined']);
|
||||
assert.deepEqual(methodUses('displayLogs', page), ['displayLogs() defined']);
|
||||
assert.deepEqual(methodUses('loadLogStats', page), ['.loadLogStats( called']);
|
||||
assert.deepEqual(methodUses('downloadLogs', page), []);
|
||||
assert.deepEqual(methodUses('formatBytes', page), []);
|
||||
// One id or method is listed on one row per page.
|
||||
const twice = [removalRow('View logs | admin | `#logsModal` | — | level filter works (logs.spec.mjs)'),
|
||||
{ ...removalRow('View logs | admin | `#logsModal` | — | level filter works (logs.spec.mjs)'), line: 9 }];
|
||||
assert.match(duplicateRemovals(twice).join(), /admin #logsModal is listed on rows/);
|
||||
assert.deepEqual(removalProblems(removalRow('View logs | admin | `logsModal`, `#ok` | `close` | level filter works (logs.spec.mjs)'),
|
||||
FIXTURE_COVERAGE, EMPTY_PAGE), ['malformed entry "logsModal"', 'malformed entry "close"']);
|
||||
});
|
||||
|
||||
test('a removal citing a kept-in-legacy capability fails', () => {
|
||||
const row = removalRow('Infinite scrolling default | admin | `#infiniteScrollDefault` | — | scrolls (list-state.test.js)');
|
||||
assert.deepEqual(removalProblems(row, FIXTURE_COVERAGE, EMPTY_PAGE), ['cites a kept in legacy capability']);
|
||||
const unknown = removalRow('Something else | admin | — | — | scrolls (list-state.test.js)');
|
||||
assert.deepEqual(removalProblems(unknown, FIXTURE_COVERAGE, EMPTY_PAGE), ['cites no single coverage row "Something else"']);
|
||||
});
|
||||
|
||||
test('a removal whose native test differs fails', () => {
|
||||
const row = removalRow('View logs | admin | `#logsModal` | `showLogsModal()` | another title (logs.spec.mjs)');
|
||||
assert.deepEqual(removalProblems(row, FIXTURE_COVERAGE, EMPTY_PAGE),
|
||||
['its native test differs from the coverage row\'s "level filter works (logs.spec.mjs)"']);
|
||||
assert.deepEqual(removalProblems({ ...row, 'Native test': 'level filter works (logs.spec.mjs)' }, FIXTURE_COVERAGE, EMPTY_PAGE), []);
|
||||
});
|
||||
|
||||
test('a removed id still on its page fails', () => {
|
||||
const row = removalRow('View logs | admin | `#logsModal` | `showLogsModal()` | level filter works (logs.spec.mjs)');
|
||||
const page = () => ({ html: '<div id="logsModal" class="hidden"></div>', script: 'class Admin {\n showLogsModal() {\n }\n}\n' });
|
||||
assert.deepEqual(removalProblems(row, FIXTURE_COVERAGE, page), [
|
||||
'#logsModal is still on the admin page: id="logsModal" in the markup',
|
||||
'showLogsModal() is still on the admin page: showLogsModal() defined',
|
||||
]);
|
||||
assert.deepEqual(removalProblems({ ...row, Page: 'metadata' }, FIXTURE_COVERAGE, page), ['unknown page "metadata"']);
|
||||
});
|
||||
+23
-35
@@ -1,9 +1,9 @@
|
||||
{
|
||||
"admin.js": {
|
||||
"lines": 37,
|
||||
"lines": 32,
|
||||
"branches": 61,
|
||||
"functions": 32,
|
||||
"statements": 37
|
||||
"functions": 27,
|
||||
"statements": 32
|
||||
},
|
||||
"api-client.js": {
|
||||
"lines": 98,
|
||||
@@ -48,10 +48,10 @@
|
||||
"statements": 98
|
||||
},
|
||||
"gallery.js": {
|
||||
"lines": 49,
|
||||
"branches": 68,
|
||||
"functions": 50,
|
||||
"statements": 49
|
||||
"lines": 45,
|
||||
"branches": 66,
|
||||
"functions": 48,
|
||||
"statements": 45
|
||||
},
|
||||
"html-escape.js": {
|
||||
"lines": 98,
|
||||
@@ -133,7 +133,7 @@
|
||||
},
|
||||
"sse-stream.js": {
|
||||
"lines": 98,
|
||||
"branches": 92,
|
||||
"branches": 90,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
@@ -181,7 +181,7 @@
|
||||
},
|
||||
"workspace/activity-scan.mjs": {
|
||||
"lines": 96,
|
||||
"branches": 90,
|
||||
"branches": 87,
|
||||
"functions": 98,
|
||||
"statements": 96
|
||||
},
|
||||
@@ -275,12 +275,6 @@
|
||||
"functions": 85,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/images-multiselect.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 98,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/images-selection.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 93,
|
||||
@@ -426,10 +420,10 @@
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/maintenance-review.mjs": {
|
||||
"lines": 95,
|
||||
"branches": 87,
|
||||
"functions": 91,
|
||||
"statements": 95
|
||||
"lines": 94,
|
||||
"branches": 86,
|
||||
"functions": 90,
|
||||
"statements": 94
|
||||
},
|
||||
"workspace/media-viewer.mjs": {
|
||||
"lines": 98,
|
||||
@@ -521,21 +515,15 @@
|
||||
"functions": 89,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/system-diagnostics.mjs": {
|
||||
"lines": 95,
|
||||
"branches": 86,
|
||||
"functions": 94,
|
||||
"statements": 95
|
||||
},
|
||||
"workspace/system-integrations.mjs": {
|
||||
"lines": 97,
|
||||
"branches": 90,
|
||||
"functions": 92,
|
||||
"statements": 97
|
||||
"lines": 96,
|
||||
"branches": 88,
|
||||
"functions": 91,
|
||||
"statements": 96
|
||||
},
|
||||
"workspace/system-maintenance.mjs": {
|
||||
"lines": 93,
|
||||
"branches": 81,
|
||||
"branches": 80,
|
||||
"functions": 92,
|
||||
"statements": 93
|
||||
},
|
||||
@@ -558,10 +546,10 @@
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/system-tagging.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 94,
|
||||
"functions": 95,
|
||||
"statements": 98
|
||||
"lines": 80,
|
||||
"branches": 70,
|
||||
"functions": 75,
|
||||
"statements": 80
|
||||
},
|
||||
"workspace/system-thumbnails.mjs": {
|
||||
"lines": 98,
|
||||
@@ -572,7 +560,7 @@
|
||||
"workspace/system.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 92,
|
||||
"functions": 98,
|
||||
"functions": 86,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/theme.mjs": {
|
||||
|
||||
Vendored
-347
@@ -1,347 +0,0 @@
|
||||
// The legacy System capability coverage table (TESTING.md "Legacy System capability coverage", #249 System S12 #329):
|
||||
// every capability of the legacy admin and gallery pages names its owner, a status, the native control that replaces it
|
||||
// and a declared test. tests/vendor/legacy-coverage.test.mjs checks each row against the test files; the browser spec
|
||||
// `every covered legacy System capability has its native control` finds each covered row's control on its System page.
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
export const ROOT = join(dirname(fileURLToPath(import.meta.url)), '../..');
|
||||
export const HEADING = '### Legacy System capability coverage';
|
||||
export const COLUMNS = Object.freeze(['Capability', 'Legacy anchor', 'Owner', 'Status', 'Native control', 'Test']);
|
||||
export const STATUSES = Object.freeze(['covered', 'kept in legacy', 'approved removal']);
|
||||
export const SECTIONS = Object.freeze(['overview', 'appearance', 'sources', 'integrations', 'tagging', 'maintenance', 'backups',
|
||||
'diagnostics']);
|
||||
export const ROLES = Object.freeze(['button', 'combobox', 'switch', 'slider', 'spinbutton', 'textbox', 'group', 'status', 'term']);
|
||||
// Where a Test cell's file is looked up, in order.
|
||||
export const TEST_FOLDERS = Object.freeze(['tests/browser', 'tests/js', 'tests/vendor', 'tests']);
|
||||
const NONE = '—';
|
||||
/**
|
||||
* Covered rows whose replacement is not on a System section, so they name no native control (`—`), each with why. Every
|
||||
* other covered row must name one.
|
||||
*/
|
||||
export const CONTROL_EXEMPTIONS = Object.freeze({
|
||||
'Gallery per-image auto-tag / review': 'owned by the Images selection bar (Auto-tag selected), not a System section',
|
||||
});
|
||||
|
||||
/** Split a Markdown table row into its cells (a `|` inside backticks is not a separator). */
|
||||
function cells(line) {
|
||||
const out = [];
|
||||
let cell = '';
|
||||
let code = false;
|
||||
for (const char of line.trim().replace(/^\|/, '').replace(/\|$/, '')) {
|
||||
if (char === '`') code = !code;
|
||||
if (char === '|' && !code) {
|
||||
out.push(cell.trim());
|
||||
cell = '';
|
||||
} else {
|
||||
cell += char;
|
||||
}
|
||||
}
|
||||
out.push(cell.trim());
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The rows, keyed by `columns`, of the first Markdown table under `heading` (`label` names it in errors). */
|
||||
function parseTable(markdown, heading, columns, label) {
|
||||
const lines = markdown.split('\n');
|
||||
const start = lines.findIndex(line => line.trim() === heading);
|
||||
if (start === -1) throw new Error(`TESTING.md has no "${heading}" section`);
|
||||
const header = lines.findIndex((line, index) => index > start && line.trim().startsWith('|'));
|
||||
if (header === -1 || lines.slice(start + 1, header).some(line => /^#{1,3} /.test(line))) {
|
||||
throw new Error(`The ${label} section has no table`);
|
||||
}
|
||||
const found = cells(lines[header]);
|
||||
if (found.join('|') !== columns.join('|')) throw new Error(`The ${label} table's columns are ${found.join(', ')}`);
|
||||
const rows = [];
|
||||
for (let index = header + 2; index < lines.length && lines[index].trim().startsWith('|'); index += 1) {
|
||||
const values = cells(lines[index]);
|
||||
rows.push({ line: index + 1, ...Object.fromEntries(columns.map((column, at) => [column, values[at] ?? ''])) });
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** The table's rows as objects keyed by COLUMNS, from the section that starts at HEADING. */
|
||||
export function parseCoverageTable(markdown) {
|
||||
return parseTable(markdown, HEADING, COLUMNS, 'coverage');
|
||||
}
|
||||
|
||||
/**
|
||||
* A Native control cell: `` `<section>` · <role> · "<name>" ``, optionally ` · in "<row title>"` when the name is shared
|
||||
* by several rows of the section → {section, role, name, row}; `—` → null; else it throws.
|
||||
*/
|
||||
export function parseControl(cell) {
|
||||
if (cell === NONE) return null;
|
||||
const match = /^`([a-z]+)` · ([a-z]+) · "([^"]+)"(?: · in "([^"]+)")?$/.exec(cell);
|
||||
if (!match) throw new Error(`malformed native control "${cell}"`);
|
||||
const [, section, role, name, row = null] = match;
|
||||
if (!SECTIONS.includes(section)) throw new Error(`unknown System section "${section}"`);
|
||||
if (!ROLES.includes(role)) throw new Error(`unknown role "${role}"`);
|
||||
return { section, role, name, row };
|
||||
}
|
||||
|
||||
/** A Test cell: `<title> (<file>)` → {title, file}. */
|
||||
export function parseTest(cell) {
|
||||
const match = /^(.+) \(([\w.-]+\.(?:mjs|js|py))\)$/.exec(cell);
|
||||
if (!match) throw new Error(`malformed test cell "${cell}"`);
|
||||
return { title: match[1], file: match[2] };
|
||||
}
|
||||
|
||||
// Modifiers that still run the test; `.skip`, `.fixme` and `.fail` do not prove the capability.
|
||||
const RUNS = new Set(['', 'only', 'slow']);
|
||||
// After these characters (or at the start) a `/` opens a regular expression, not a division.
|
||||
const REGEX_BEFORE = /[(,=:[!&|?{};+\-*%<>~^]$/;
|
||||
|
||||
/** Where the string, template literal or regular expression opened by `source[start]` ends (its closing index), or -1. */
|
||||
function literalEnd(source, start) {
|
||||
const quote = source[start];
|
||||
let inClass = false;
|
||||
for (let end = start + 1; end < source.length; end += 1) {
|
||||
const at = source[end];
|
||||
if (at === '\\') end += 1;
|
||||
else if (quote === '/' && at === '[') inClass = true;
|
||||
else if (quote === '/' && at === ']') inClass = false;
|
||||
else if (at === quote && !inClass) return end;
|
||||
// A quoted string or a regular expression never spans lines.
|
||||
else if (at === '\n' && quote !== '`') return -1;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* JavaScript `source` with every comment blanked (newlines kept), wherever it starts on a line. Strings, template
|
||||
* literals and regular expressions are skipped whole, so a `//` or `/*` inside them is not a comment.
|
||||
*/
|
||||
export function stripJsComments(source) {
|
||||
let out = '';
|
||||
let index = 0;
|
||||
while (index < source.length) {
|
||||
const char = source[index];
|
||||
const next = source[index + 1];
|
||||
if (char === '/' && next === '/') {
|
||||
while (index < source.length && source[index] !== '\n') index += 1;
|
||||
continue;
|
||||
}
|
||||
if (char === '/' && next === '*') {
|
||||
const close = source.indexOf('*/', index + 2);
|
||||
const stop = close === -1 ? source.length : close + 2;
|
||||
out += source.slice(index, stop).replace(/[^\n]/g, ' ');
|
||||
index = stop;
|
||||
continue;
|
||||
}
|
||||
const opens = char === "'" || char === '"' || char === '`' || (char === '/' && REGEX_BEFORE.test(out.trimEnd() || '('));
|
||||
const end = opens ? literalEnd(source, index) : -1;
|
||||
if (end === -1) {
|
||||
out += char;
|
||||
index += 1;
|
||||
} else {
|
||||
out += source.slice(index, end + 1);
|
||||
index = end + 1;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Python `source` with its comments and triple-quoted strings (docstrings) blanked, newlines kept. */
|
||||
export function stripPythonComments(source) {
|
||||
return source.replace(/("""|''')[\s\S]*?\1|#[^\n]*/g, text => text.replace(/[^\n]/g, ' '));
|
||||
}
|
||||
|
||||
/**
|
||||
* The tests a source declares, title → whether it runs: a `test('…'` or `it('…'` call at a line's start (`.only` and
|
||||
* `.slow` run; `.skip`, `.fixme` and `.fail` do not) in JavaScript, a `def test_…` at function or method indentation in
|
||||
* Python. Comments, and Python docstrings, never count.
|
||||
*/
|
||||
export function declarations(source, file) {
|
||||
const found = new Map();
|
||||
const add = (title, runs) => found.set(title, found.get(title) || runs);
|
||||
if (file.endsWith('.py')) {
|
||||
for (const match of stripPythonComments(source).matchAll(/^(?: {4})?(?:async[ \t]+)?def[ \t]+(test_\w+)[ \t]*\(/gm)) {
|
||||
add(match[1], true);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
const declaration = /^[ \t]*(?:test|it)(?:\.(only|skip|fixme|fail|slow))?\(\s*(['"`])((?:\\.|(?!\2)[^\\])*)\2/gm;
|
||||
for (const match of stripJsComments(source).matchAll(declaration)) add(match[3].replace(/\\(.)/g, '$1'), RUNS.has(match[1] ?? ''));
|
||||
return found;
|
||||
}
|
||||
|
||||
/** The titles of the tests a source declares and runs. */
|
||||
export function declaredTitles(source, file) {
|
||||
return new Set([...declarations(source, file)].filter(([, runs]) => runs).map(([title]) => title));
|
||||
}
|
||||
|
||||
/** The repository path of a Test cell's file, or null. */
|
||||
export function testFile(file, root = ROOT) {
|
||||
for (const folder of TEST_FOLDERS) {
|
||||
const path = join(root, folder, file);
|
||||
if (existsSync(path)) return path;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Every problem of one row (empty: the row is sound). `locate(file)` finds a test file's path, `read(path)` reads it. */
|
||||
export function rowProblems(row, { root = ROOT, locate = file => testFile(file, root), read = path => readFileSync(path, 'utf8') } = {}) {
|
||||
const problems = [];
|
||||
if (!row.Capability) problems.push('no capability');
|
||||
if (!row['Legacy anchor']) problems.push('no legacy anchor');
|
||||
if (!row.Owner) problems.push('no owner');
|
||||
if (!STATUSES.includes(row.Status)) problems.push(`unknown status "${row.Status}"`);
|
||||
let control = null;
|
||||
try {
|
||||
control = parseControl(row['Native control']);
|
||||
} catch (error) {
|
||||
problems.push(error.message);
|
||||
}
|
||||
if (control && row.Status !== 'covered') problems.push(`a ${row.Status} row names a native control`);
|
||||
if (row.Status === 'covered' && row['Native control'] === NONE && !Object.hasOwn(CONTROL_EXEMPTIONS, row.Capability)) {
|
||||
problems.push('a covered row names no native control');
|
||||
}
|
||||
let test;
|
||||
try {
|
||||
test = parseTest(row.Test);
|
||||
} catch (error) {
|
||||
problems.push(error.message);
|
||||
return problems;
|
||||
}
|
||||
const path = locate(test.file);
|
||||
if (!path) problems.push(`no test file ${test.file}`);
|
||||
else {
|
||||
const runs = declarations(read(path), test.file).get(test.title);
|
||||
if (runs === undefined) problems.push(`${test.file} declares no test "${test.title}"`);
|
||||
else if (!runs) problems.push(`${test.file} only skips "${test.title}"`);
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
/** The table read from TESTING.md. */
|
||||
export function readCoverageTable(root = ROOT) {
|
||||
return parseCoverageTable(readFileSync(join(root, 'TESTING.md'), 'utf8'));
|
||||
}
|
||||
|
||||
// ── Legacy System UI removed by S13 (#330) ──────────────────────────────────────────────────────────────────────────
|
||||
// TESTING.md "Legacy System UI removed by S13": one row per covered capability whose legacy UI S13 removed, with the
|
||||
// element ids and methods that went. A row must cite a `covered` coverage row and copy its Test cell, and none of its
|
||||
// ids or methods may still be on its page.
|
||||
export const REMOVAL_HEADING = '### Legacy System UI removed by S13';
|
||||
export const REMOVAL_COLUMNS = Object.freeze(['Capability', 'Page', 'Removed ids', 'Removed methods', 'Native test']);
|
||||
/** Each legacy page's markup and script. */
|
||||
export const PAGES = Object.freeze({
|
||||
admin: Object.freeze({ html: 'web/admin.html', script: 'web/js/admin.js' }),
|
||||
gallery: Object.freeze({ html: 'web/gallery.html', script: 'web/js/gallery.js' }),
|
||||
});
|
||||
|
||||
/** The removal table's rows, from the section that starts at REMOVAL_HEADING. */
|
||||
export function parseRemovalTable(markdown) {
|
||||
return parseTable(markdown, REMOVAL_HEADING, REMOVAL_COLUMNS, 'removal');
|
||||
}
|
||||
|
||||
/** The removal table read from TESTING.md. */
|
||||
export function readRemovalTable(root = ROOT) {
|
||||
return parseRemovalTable(readFileSync(join(root, 'TESTING.md'), 'utf8'));
|
||||
}
|
||||
|
||||
/** A Removed ids or Removed methods cell → its entries without backticks (`—` → none). */
|
||||
export function listCell(cell) {
|
||||
if (cell.trim() === NONE) return [];
|
||||
return cell.split(',').map(entry => entry.trim().replace(/^`|`$/g, '')).filter(Boolean);
|
||||
}
|
||||
|
||||
/** The removed ids of a row (without `#`) and its malformed entries. */
|
||||
function removedIds(row) {
|
||||
const entries = listCell(row['Removed ids']);
|
||||
return { ids: entries.filter(entry => /^#[\w-]+$/.test(entry)).map(entry => entry.slice(1)),
|
||||
bad: entries.filter(entry => !/^#[\w-]+$/.test(entry)) };
|
||||
}
|
||||
|
||||
/** The removed methods of a row (without `()`) and its malformed entries. */
|
||||
function removedMethods(row) {
|
||||
const entries = listCell(row['Removed methods']);
|
||||
return { names: entries.filter(entry => /^[A-Za-z_$][\w$]*\(\)$/.test(entry)).map(entry => entry.slice(0, -2)),
|
||||
bad: entries.filter(entry => !/^[A-Za-z_$][\w$]*\(\)$/.test(entry)) };
|
||||
}
|
||||
|
||||
const escapeRegExp = text => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
/**
|
||||
* Where `id` is still on a page: `id="<id>"` in its markup or script; in its script a lookup, a `…Modal("<id>")` call
|
||||
* (`showModal`, `hideModal`) or a modal-list entry.
|
||||
*/
|
||||
export function idUses(id, { html, script }) {
|
||||
const name = escapeRegExp(id);
|
||||
const code = stripJsComments(script);
|
||||
const uses = [];
|
||||
// `id=`, not `data-id=`: the attribute starts after a space or a tag's `<name`.
|
||||
const attribute = new RegExp(`(?:^|\\s)id=["']${name}["']`, 'm');
|
||||
if (attribute.test(html)) uses.push(`id="${id}" in the markup`);
|
||||
if (attribute.test(code)) uses.push(`id="${id}" in the script`);
|
||||
if (new RegExp(`getElementById\\(\\s*["'\`]${name}["'\`]\\s*\\)`).test(code)) uses.push(`getElementById('${id}')`);
|
||||
if (new RegExp(`querySelector(?:All)?\\(\\s*["'\`]#${name}\\b`).test(code)) uses.push(`querySelector('#${id}')`);
|
||||
if (new RegExp(`\\w*Modal\\(\\s*["'\`]${name}["'\`]`).test(code)) uses.push(`a …Modal('${id}') call`);
|
||||
// A string list of modal ids (the gallery's backdrop list): the entry next to another `…Modal` entry.
|
||||
for (const list of code.matchAll(/\[[^\[\]]*\]/g)) {
|
||||
const entry = new RegExp(`["']${name}["']`);
|
||||
if (entry.test(list[0]) && /["'][\w-]*Modal["']/.test(list[0].replace(entry, ''))) {
|
||||
uses.push(`'${id}' in a modal list`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return uses;
|
||||
}
|
||||
|
||||
/**
|
||||
* Where method `name` is still on a page: a definition in its script, or a `.<name>(` call (`this.`, `window.admin.`)
|
||||
* in its script or markup.
|
||||
*/
|
||||
export function methodUses(name, { html, script }) {
|
||||
const method = escapeRegExp(name);
|
||||
const code = stripJsComments(script);
|
||||
const uses = [];
|
||||
if (new RegExp(`(?:^|[\\s;}])(?:async\\s+)?${method}\\s*\\([^)]*\\)\\s*\\{`, 'm').test(code)) uses.push(`${name}() defined`);
|
||||
if (new RegExp(`\\.${method}\\s*\\(`).test(code)) uses.push(`.${name}( called`);
|
||||
if (new RegExp(`\\.${method}\\s*\\(`).test(html)) uses.push(`.${name}( called in the markup`);
|
||||
return uses;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every problem of one removal row (empty: sound). `coverage` is the coverage table's rows; `readPage(page)` returns
|
||||
* that page's `{html, script}`.
|
||||
*/
|
||||
export function removalProblems(row, coverage, readPage) {
|
||||
const problems = [];
|
||||
const cited = coverage.filter(item => item.Capability === row.Capability);
|
||||
if (cited.length !== 1) problems.push(`cites no single coverage row "${row.Capability}"`);
|
||||
else if (cited[0].Status !== 'covered') problems.push(`cites a ${cited[0].Status} capability`);
|
||||
else if (cited[0].Test !== row['Native test']) problems.push(`its native test differs from the coverage row's "${cited[0].Test}"`);
|
||||
if (!Object.hasOwn(PAGES, row.Page)) {
|
||||
problems.push(`unknown page "${row.Page}"`);
|
||||
return problems;
|
||||
}
|
||||
const { ids, bad: badIds } = removedIds(row);
|
||||
const { names, bad: badNames } = removedMethods(row);
|
||||
for (const entry of [...badIds, ...badNames]) problems.push(`malformed entry "${entry}"`);
|
||||
const page = readPage(row.Page);
|
||||
for (const id of ids) for (const use of idUses(id, page)) problems.push(`#${id} is still on the ${row.Page} page: ${use}`);
|
||||
for (const name of names) for (const use of methodUses(name, page)) problems.push(`${name}() is still on the ${row.Page} page: ${use}`);
|
||||
return problems;
|
||||
}
|
||||
|
||||
/** Ids and methods listed on more than one row of the same page. */
|
||||
export function duplicateRemovals(rows) {
|
||||
const seen = new Map();
|
||||
const duplicates = [];
|
||||
for (const row of rows) {
|
||||
for (const entry of [...listCell(row['Removed ids']), ...listCell(row['Removed methods'])]) {
|
||||
const key = `${row.Page} ${entry}`;
|
||||
if (seen.has(key)) duplicates.push(`${key} is listed on rows ${seen.get(key)} and ${row.line}`);
|
||||
else seen.set(key, row.line);
|
||||
}
|
||||
}
|
||||
return duplicates;
|
||||
}
|
||||
|
||||
/** A page's markup and script from the repository. */
|
||||
export function readPage(page, root = ROOT) {
|
||||
const files = PAGES[page];
|
||||
return { html: readFileSync(join(root, files.html), 'utf8'), script: readFileSync(join(root, files.script), 'utf8') };
|
||||
}
|
||||
@@ -2578,115 +2578,3 @@ def resolve_text_input(
|
||||
return resolver.text(inputs.get(input_name), "string")[0]
|
||||
except _Unresolved:
|
||||
return None
|
||||
|
||||
|
||||
_NUMBER = re.compile(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?")
|
||||
|
||||
|
||||
def _numeric(text: str) -> bool:
|
||||
"""Whether ``text`` is a plain number (a widget value such as ``2048``)."""
|
||||
return _NUMBER.fullmatch(text) is not None
|
||||
|
||||
|
||||
def sampler_less_prompt(graph: Any, *, max_visits: int = 500) -> Dict[str, Any]:
|
||||
"""The positive prompt of an API graph with no sampler pair, if one (#252).
|
||||
|
||||
For graphs ``_no_sampler`` finds ``absent`` (no node takes a ``positive``
|
||||
or ``negative`` input, such as a Flux guider or a video wrapper). Only the
|
||||
executed API graph is read, never the UI workflow. A candidate is a node
|
||||
with a ``text`` input whose output another node consumes: a text that
|
||||
feeds nothing did not make the image. Its text is traced as a STRING
|
||||
(``resolve_text_input``); a linked text no rule reads is unknown unless
|
||||
it is built from a candidate's text (``_covered``). Blank and
|
||||
numeric texts are never prompts, and a node with a seed input holds a
|
||||
template, not the prompt (unknown). This repository's own positive-text
|
||||
nodes are authoritative: when they give a text, only they count.
|
||||
|
||||
Returns ``{"state", "text", "texts"}``: ``resolved`` with the one distinct
|
||||
text; ``ambiguous`` when several texts or an unknown one could be the
|
||||
prompt (``texts`` lists the readable ones); ``absent`` when none.
|
||||
"""
|
||||
absent = {"state": "absent", "text": None, "texts": []}
|
||||
if not isinstance(graph, dict) or not graph or len(graph) > max_visits:
|
||||
return absent
|
||||
nodes = {str(key): value for key, value in graph.items()}
|
||||
consumed = {
|
||||
str(value[0])
|
||||
for node in nodes.values()
|
||||
if isinstance(node, dict) and isinstance(node.get("inputs"), dict)
|
||||
for value in node["inputs"].values()
|
||||
if _link(value)
|
||||
}
|
||||
own: list = []
|
||||
other: list = []
|
||||
unknown = False
|
||||
for node_id, node in nodes.items():
|
||||
inputs = node.get("inputs") if isinstance(node, dict) else None
|
||||
if node_id not in consumed or not _has_text_input(node):
|
||||
continue
|
||||
if _seeded(inputs):
|
||||
# Generated at run time (a wildcard or a language model): its
|
||||
# text input is a template, not the prompt that ran.
|
||||
unknown = True
|
||||
continue
|
||||
value = inputs["text"]
|
||||
text = resolve_text_input(nodes, node_id, "text", max_visits=max_visits)
|
||||
if text is None:
|
||||
# A literal that is not a string (a number) is never a prompt.
|
||||
if _link(value) and not _covered(nodes, value):
|
||||
unknown = True
|
||||
continue
|
||||
text = text.strip()
|
||||
if not text or _numeric(text):
|
||||
continue
|
||||
kind = node.get("class_type")
|
||||
if isinstance(kind, str) and kind in _OWN_CLASSES and "Negative" not in kind:
|
||||
own.append(text)
|
||||
else:
|
||||
other.append(text)
|
||||
texts = list(dict.fromkeys(own or other))
|
||||
if len(texts) == 1 and (own or not unknown):
|
||||
return {"state": "resolved", "text": texts[0], "texts": texts}
|
||||
if texts or unknown:
|
||||
return {"state": "ambiguous", "text": None, "texts": texts}
|
||||
return absent
|
||||
|
||||
|
||||
def _has_text_input(node: Any) -> bool:
|
||||
inputs = node.get("inputs") if isinstance(node, dict) else None
|
||||
return isinstance(inputs, dict) and "text" in inputs
|
||||
|
||||
|
||||
def _seeded(inputs: dict) -> bool:
|
||||
"""A seed input: the node's string output is a run-time value (the
|
||||
resolver's ``runtime-value`` rule)."""
|
||||
return any(isinstance(name, str) and name.endswith("seed") for name in inputs)
|
||||
|
||||
|
||||
def _covered(nodes: dict, link: list) -> bool:
|
||||
"""Whether an unreadable text link is built from a candidate's text.
|
||||
|
||||
True when a node upstream of ``link`` is read through its first output
|
||||
(slot 0, the legacy encoder rule's text) and has an unseeded ``text``
|
||||
input: that candidate is read on its own, and whatever else is joined to
|
||||
it at run time (a caption, a language model) is no stored prompt to
|
||||
choose. Another output of such a node (a loader's trigger words) is not
|
||||
its text. False when the text comes only from inputs no candidate holds
|
||||
(a builder's ``prompt`` field): the prompt is unknown then.
|
||||
"""
|
||||
seen, stack = set(), [(str(link[0]), link[1])]
|
||||
while stack:
|
||||
node_id, slot = stack.pop()
|
||||
if (node_id, slot) in seen:
|
||||
continue
|
||||
seen.add((node_id, slot))
|
||||
node = nodes.get(node_id)
|
||||
inputs = node.get("inputs") if isinstance(node, dict) else None
|
||||
if not isinstance(inputs, dict):
|
||||
continue
|
||||
if "text" in inputs and slot == 0 and not _seeded(inputs):
|
||||
return True
|
||||
stack.extend(
|
||||
(str(value[0]), value[1]) for value in inputs.values() if _link(value)
|
||||
)
|
||||
return False
|
||||
|
||||
+165
-12
@@ -168,11 +168,10 @@
|
||||
class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors">
|
||||
Settings
|
||||
</button>
|
||||
<a id="diagnosticsBtn" href="workspace#/system/diagnostics" target="_blank" rel="noopener"
|
||||
title="Opens System › Diagnostics & logs in a new tab"
|
||||
class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors">
|
||||
<button id="diagnosticsBtn"
|
||||
class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors">
|
||||
Diagnostics
|
||||
</a>
|
||||
</button>
|
||||
<button id="maintenanceBtn"
|
||||
class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors">
|
||||
Maintenance
|
||||
@@ -193,11 +192,10 @@
|
||||
class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors">
|
||||
Auto Tag
|
||||
</button>
|
||||
<a id="logsBtn" href="workspace#/system/diagnostics" target="_blank" rel="noopener"
|
||||
title="Opens System › Diagnostics & logs in a new tab"
|
||||
class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors">
|
||||
<button id="logsBtn"
|
||||
class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors">
|
||||
View Logs
|
||||
</a>
|
||||
</button>
|
||||
<button id="metadataBtn"
|
||||
class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors">
|
||||
Metadata
|
||||
@@ -643,6 +641,37 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Diagnostics Modal -->
|
||||
<div id="diagnosticsModal" class="fixed inset-0 bg-black/60 backdrop-blur-[2px] hidden items-center justify-center z-50">
|
||||
<div class="bg-pm-surface rounded-pm-lg max-w-4xl w-full max-h-[90vh] overflow-hidden mx-4 border border-pm shadow-pm">
|
||||
<div class="p-4 border-b border-pm">
|
||||
<div class="flex items-center justify-between">
|
||||
<h3 class="text-sm font-semibold text-pm">Gallery System Diagnostics</h3>
|
||||
<button onclick="window.admin.closeDiagnostics()" class="p-1 rounded-pm-sm hover:bg-pm-hover transition-colors">
|
||||
<svg class="w-4 h-4 text-pm-secondary" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="p-4 overflow-y-auto max-h-[calc(90vh-120px)]">
|
||||
<div class="flex items-center justify-between mb-3">
|
||||
<button id="runDiagnosticsBtn" class="px-2.5 py-1.5 bg-pm-accent hover:bg-pm-accent-hover text-pm-accent-fg text-xs font-medium rounded-pm-sm transition-colors">
|
||||
Run Diagnostics
|
||||
</button>
|
||||
<button id="testImageLinkBtn" class="px-2.5 py-1.5 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors">
|
||||
Test Image Link
|
||||
</button>
|
||||
</div>
|
||||
<div id="diagnosticsContent" class="space-y-3">
|
||||
<div class="text-center py-6 text-pm-secondary text-xs">
|
||||
Click "Run Diagnostics" to check the gallery system status
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Maintenance Modal -->
|
||||
<div id="maintenanceModal" class="fixed inset-0 bg-black/60 backdrop-blur-[2px] hidden items-center justify-center z-50">
|
||||
<div class="bg-pm-surface rounded-pm-lg max-w-4xl w-full max-h-[90vh] overflow-hidden mx-4 border border-pm shadow-pm">
|
||||
@@ -835,6 +864,130 @@
|
||||
<span class="inline-block w-1.5 h-1.5 rounded-full bg-pm-accent mr-1.5 align-middle animate-pulse"></span><span id="scanPillText" class="align-middle">Scanning...</span>
|
||||
</button>
|
||||
|
||||
<!-- Logs Modal -->
|
||||
<div id="logsModal" class="fixed inset-0 bg-black/60 backdrop-blur-[2px] hidden items-center justify-center z-50">
|
||||
<div class="bg-pm-surface rounded-pm-lg max-w-6xl w-full max-h-[90vh] overflow-hidden mx-4 border border-pm shadow-pm">
|
||||
<div class="p-4 border-b border-pm">
|
||||
<div class="flex items-center justify-between">
|
||||
<h3 class="text-sm font-semibold text-pm">System Logs</h3>
|
||||
<button onclick="window.admin.closeLogs()" class="p-1 rounded-pm-sm hover:bg-pm-hover transition-colors">
|
||||
<svg class="w-4 h-4 text-pm-secondary" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Log Controls -->
|
||||
<div class="p-4 border-b border-pm">
|
||||
<div class="grid grid-cols-1 md:grid-cols-5 gap-3 mb-3">
|
||||
<div>
|
||||
<label class="block text-xs font-medium text-pm-secondary mb-1">Log Level</label>
|
||||
<select id="logLevel" class="w-full px-2.5 py-1.5 bg-pm-input border border-pm rounded-pm-sm text-pm text-[13px] focus:outline-none">
|
||||
<option value="">All Levels</option>
|
||||
<option value="DEBUG">Debug</option>
|
||||
<option value="INFO">Info</option>
|
||||
<option value="WARNING">Warning</option>
|
||||
<option value="ERROR">Error</option>
|
||||
<option value="CRITICAL">Critical</option>
|
||||
</select>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-xs font-medium text-pm-secondary mb-1">Limit</label>
|
||||
<select id="logLimit" class="w-full px-2.5 py-1.5 bg-pm-input border border-pm rounded-pm-sm text-pm text-[13px] focus:outline-none">
|
||||
<option value="50">Last 50</option>
|
||||
<option value="100" selected>Last 100</option>
|
||||
<option value="200">Last 200</option>
|
||||
<option value="500">Last 500</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="flex items-end">
|
||||
<button id="refreshLogsBtn" class="w-full px-2.5 py-1.5 bg-pm-accent hover:bg-pm-accent-hover text-pm-accent-fg text-xs font-medium rounded-pm-sm transition-colors">
|
||||
Refresh
|
||||
</button>
|
||||
</div>
|
||||
<div class="flex items-end">
|
||||
<button id="downloadLogsBtn" class="w-full px-2.5 py-1.5 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors">
|
||||
Download
|
||||
</button>
|
||||
</div>
|
||||
<div class="flex items-end">
|
||||
<button id="clearLogsBtn" class="w-full px-2.5 py-1.5 text-pm-error bg-transparent hover:bg-pm-error-tint border border-pm text-xs font-medium rounded-pm-sm transition-colors">
|
||||
Clear Logs
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Log Stats -->
|
||||
<div class="grid grid-cols-2 md:grid-cols-5 gap-3 mb-3">
|
||||
<div class="bg-pm-input rounded-pm-md p-2">
|
||||
<div class="text-[11px] text-pm-secondary">Total Logs</div>
|
||||
<div class="text-sm font-semibold text-pm-accent" id="logStatsTotal">-</div>
|
||||
</div>
|
||||
<div class="bg-pm-input rounded-pm-md p-2">
|
||||
<div class="text-[11px] text-pm-secondary">Errors</div>
|
||||
<div class="text-sm font-semibold text-pm-error" id="logStatsErrors">-</div>
|
||||
</div>
|
||||
<div class="bg-pm-input rounded-pm-md p-2">
|
||||
<div class="text-[11px] text-pm-secondary">Warnings</div>
|
||||
<div class="text-sm font-semibold text-pm-warning" id="logStatsWarnings">-</div>
|
||||
</div>
|
||||
<div class="bg-pm-input rounded-pm-md p-2">
|
||||
<div class="text-[11px] text-pm-secondary">Current Level</div>
|
||||
<div class="text-sm font-semibold text-pm-success" id="logStatsLevel">-</div>
|
||||
</div>
|
||||
<div class="bg-pm-input rounded-pm-md p-2">
|
||||
<div class="text-[11px] text-pm-secondary">Log Size</div>
|
||||
<div class="text-sm font-semibold text-pm" id="logStatsSize">-</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Log Settings -->
|
||||
<div class="grid grid-cols-1 md:grid-cols-3 gap-3">
|
||||
<div>
|
||||
<label class="block text-xs font-medium text-pm-secondary mb-1">Set Log Level</label>
|
||||
<select id="setLogLevel" class="w-full px-2.5 py-1.5 bg-pm-input border border-pm rounded-pm-sm text-pm text-[13px] focus:outline-none">
|
||||
<option value="DEBUG">Debug</option>
|
||||
<option value="INFO" selected>Info</option>
|
||||
<option value="WARNING">Warning</option>
|
||||
<option value="ERROR">Error</option>
|
||||
<option value="CRITICAL">Critical</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="flex items-center">
|
||||
<label class="flex items-center gap-1.5 text-pm">
|
||||
<input type="checkbox" id="consoleLogging" checked class="w-3.5 h-3.5 accent-[var(--pm-accent)]">
|
||||
<span class="text-xs">Console Logging</span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="flex items-end">
|
||||
<button id="updateLogConfigBtn" class="w-full px-2.5 py-1.5 bg-pm-accent hover:bg-pm-accent-hover text-pm-accent-fg text-xs font-medium rounded-pm-sm transition-colors">
|
||||
Update Config
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Log Viewer -->
|
||||
<div class="flex-1 overflow-hidden">
|
||||
<div class="h-96 overflow-y-auto bg-pm-primary font-mono text-xs">
|
||||
<div id="logsContainer" class="p-3 space-y-0.5">
|
||||
<div class="text-center py-6 text-pm-muted text-xs">
|
||||
Click "Refresh" to load logs
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Log Files -->
|
||||
<div class="p-4 border-t border-pm">
|
||||
<h4 class="text-xs font-semibold text-pm-secondary uppercase tracking-wide mb-3">Log Files</h4>
|
||||
<div id="logFilesList" class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-2">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Add Prompt Modal -->
|
||||
<div id="addPromptModal" class="fixed inset-0 bg-black/60 backdrop-blur-[2px] hidden items-center justify-center z-50">
|
||||
<div class="bg-pm-surface rounded-pm-lg p-4 max-w-2xl w-full mx-4 border border-pm max-h-[90vh] overflow-y-auto shadow-pm">
|
||||
@@ -1034,13 +1187,14 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr
|
||||
<div class="space-y-1.5">
|
||||
<label class="flex items-center p-2.5 bg-pm-input rounded-pm-md cursor-pointer hover:bg-pm-hover border border-pm">
|
||||
<input type="radio" name="autoTagMode" value="skip" checked class="w-3.5 h-3.5 accent-[var(--pm-accent)]">
|
||||
<span class="ml-2.5 text-pm text-xs">Skip images already auto-tagged</span>
|
||||
<span class="ml-2.5 text-pm text-xs">Skip images with existing tags</span>
|
||||
</label>
|
||||
<label class="flex items-center p-2.5 bg-pm-input rounded-pm-md cursor-pointer hover:bg-pm-hover border border-pm">
|
||||
<input type="radio" name="autoTagMode" value="retag" class="w-3.5 h-3.5 accent-[var(--pm-accent)]">
|
||||
<span class="ml-2.5 text-pm text-xs">Re-tag all images</span>
|
||||
</label>
|
||||
</div>
|
||||
<p class="text-pm-muted text-[11px] mt-1">Note: "auto-scanned" is not counted as a real tag</p>
|
||||
</div>
|
||||
|
||||
<!-- Action Buttons -->
|
||||
@@ -1139,7 +1293,6 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr
|
||||
<div class="w-full bg-pm-input rounded-full h-1.5">
|
||||
<div id="autoTagProgressBar" class="bg-pm-accent h-1.5 rounded-full transition-all duration-300" style="width: 0%"></div>
|
||||
</div>
|
||||
<p id="autoTagCloseNote" class="text-pm-muted text-[11px] mt-1.5">Closing this window stops tagging. Tags already added are kept. Prompt Manager's Activity page shows the time left.</p>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-3 gap-3 mb-4">
|
||||
@@ -1179,7 +1332,7 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr
|
||||
<!-- Re-tag Confirmation Modal -->
|
||||
<div id="retagConfirmModal" class="fixed inset-0 bg-black/60 backdrop-blur-[2px] hidden items-center justify-center z-50">
|
||||
<div class="bg-pm-surface rounded-pm-lg p-4 max-w-lg w-full mx-4 border border-pm shadow-pm">
|
||||
<h3 class="text-sm font-semibold text-pm mb-3">Image Already Auto-tagged</h3>
|
||||
<h3 class="text-sm font-semibold text-pm mb-3">Image Already Tagged</h3>
|
||||
|
||||
<div class="mb-3">
|
||||
<img id="retagPreviewImage" src="" alt="Preview" class="w-full h-40 object-contain bg-pm-primary rounded-pm-md">
|
||||
@@ -1192,7 +1345,7 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p class="text-pm-secondary text-xs mb-4">A tagging model already tagged this image. Generate new tags?</p>
|
||||
<p class="text-pm-secondary text-xs mb-4">This image already has tags. Would you like to generate new tags?</p>
|
||||
|
||||
<div class="flex gap-2 justify-end">
|
||||
<button id="retagSkipBtn"
|
||||
|
||||
@@ -682,7 +682,6 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr
|
||||
<div class="w-full bg-pm-input rounded-full h-2">
|
||||
<div id="autoTagProgressBar" class="bg-pm-accent h-2 rounded-full transition-all duration-300" style="width: 0%"></div>
|
||||
</div>
|
||||
<p id="autoTagCloseNote" class="text-pm-muted text-xs mt-1.5">Closing this window stops tagging. Tags already added are kept. Prompt Manager's Activity page shows the time left.</p>
|
||||
</div>
|
||||
|
||||
<!-- Stats -->
|
||||
|
||||
+508
-52
@@ -47,6 +47,7 @@
|
||||
edit: ({ promptId }) => this.editPrompt(promptId),
|
||||
delete: ({ promptId }) => this.deletePrompt(promptId),
|
||||
"open-film": ({ promptId, index }) => this.openFilmStripViewer(promptId, index),
|
||||
"download-log": ({ filename }) => this.downloadLogFile(filename),
|
||||
"remove-review-tag": ({ index }) => this.removeReviewTag(index),
|
||||
"pick-tag-suggestion": ({ tag }) => this.pickTagSuggestion(tag),
|
||||
};
|
||||
@@ -127,12 +128,14 @@
|
||||
document.getElementById("exportBtn").addEventListener("click", () => this.exportPrompts());
|
||||
document.getElementById("settingsBtn").addEventListener("click", () => this.showSettingsModal());
|
||||
document.getElementById("statsBtn").addEventListener("click", () => this.showStats());
|
||||
document.getElementById("diagnosticsBtn").addEventListener("click", () => this.showDiagnosticsModal());
|
||||
document.getElementById("maintenanceBtn").addEventListener("click", () => this.showMaintenanceModal());
|
||||
document.getElementById("backupBtn").addEventListener("click", () => this.backupDatabase());
|
||||
document.getElementById("restoreBtn").addEventListener("click", () => this.showRestoreModal());
|
||||
document.getElementById("scanBtn").addEventListener("click", () => this.showScanModal());
|
||||
document.getElementById("autoTagBtn").addEventListener("click", () => this.showAutoTagModal());
|
||||
document.getElementById("addPromptBtn").addEventListener("click", () => this.showAddPromptModal());
|
||||
document.getElementById("logsBtn").addEventListener("click", () => this.showLogsModal());
|
||||
document.getElementById("metadataBtn").addEventListener("click", () => this.openMetadataViewer());
|
||||
document.getElementById("galleryBtn").addEventListener("click", () => this.openGallery());
|
||||
|
||||
@@ -212,6 +215,10 @@
|
||||
document.getElementById("confirmRestore").addEventListener("click", () => this.confirmRestore());
|
||||
document.getElementById("cancelRestore").addEventListener("click", () => this.hideModal("restoreModal"));
|
||||
|
||||
// Diagnostics modal
|
||||
document.getElementById("runDiagnosticsBtn").addEventListener("click", () => this.runDiagnostics());
|
||||
document.getElementById("testImageLinkBtn").addEventListener("click", () => this.testImageLink());
|
||||
|
||||
// Maintenance modal
|
||||
document.getElementById("runMaintenanceBtn").addEventListener("click", () => this.runMaintenance());
|
||||
document.getElementById("selectAllMaintenanceBtn").addEventListener("click", () => this.selectAllMaintenance());
|
||||
@@ -223,6 +230,12 @@
|
||||
document.getElementById("scanPill").addEventListener("click", () => this.showScanModal());
|
||||
document.getElementById("quickBackupBtn").addEventListener("click", () => this.quickBackup());
|
||||
|
||||
// Logs modal
|
||||
document.getElementById("refreshLogsBtn").addEventListener("click", () => this.refreshLogs());
|
||||
document.getElementById("downloadLogsBtn").addEventListener("click", () => this.downloadLogs());
|
||||
document.getElementById("clearLogsBtn").addEventListener("click", () => this.clearLogs());
|
||||
document.getElementById("updateLogConfigBtn").addEventListener("click", () => this.updateLogConfig());
|
||||
|
||||
// Auto Tag modals
|
||||
document.getElementById("startAutoTagBtn").addEventListener("click", () => this.startAutoTag());
|
||||
document.getElementById("startReviewBtn").addEventListener("click", () => this.startReview());
|
||||
@@ -1788,6 +1801,15 @@
|
||||
return `${size.toFixed(1)} ${units[unitIndex]}`;
|
||||
}
|
||||
|
||||
// Diagnostics functionality
|
||||
showDiagnosticsModal() {
|
||||
this.showModal("diagnosticsModal");
|
||||
}
|
||||
|
||||
closeDiagnostics() {
|
||||
this.hideModal("diagnosticsModal");
|
||||
}
|
||||
|
||||
// Maintenance functionality
|
||||
showMaintenanceModal() {
|
||||
this.showModal("maintenanceModal");
|
||||
@@ -1959,6 +1981,160 @@
|
||||
resultsContainer.innerHTML = html;
|
||||
}
|
||||
|
||||
async runDiagnostics() {
|
||||
const content = document.getElementById("diagnosticsContent");
|
||||
content.innerHTML = `
|
||||
<div class="text-center py-4">
|
||||
<div class="w-12 h-12 border-4 border-orange-500/30 border-t-orange-500 rounded-full animate-spin mx-auto mb-4"></div>
|
||||
<p class="text-pm-secondary">Running diagnostics...</p>
|
||||
</div>
|
||||
`;
|
||||
|
||||
try {
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/diagnostics');
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
if (data.success) {
|
||||
this.renderDiagnostics(data.diagnostics);
|
||||
} else {
|
||||
throw new Error(data.error || 'Diagnostics failed');
|
||||
}
|
||||
} else {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Diagnostics error:', error);
|
||||
content.innerHTML = `
|
||||
<div class="bg-pm-error-tint border border-pm-error rounded-pm-sm p-4">
|
||||
<h4 class="text-pm-error font-medium mb-2">❌ Diagnostics Failed</h4>
|
||||
<p class="text-pm-secondary">${escapeHtml(error.message)}</p>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
renderDiagnostics(diagnostics) {
|
||||
const content = document.getElementById("diagnosticsContent");
|
||||
|
||||
let html = '<div class="space-y-4">';
|
||||
|
||||
// Summary
|
||||
html += '<div class="bg-pm-surface rounded-pm-sm p-4">';
|
||||
html += '<h4 class="text-pm font-medium mb-3">📋 Diagnostic Summary</h4>';
|
||||
html += '<div class="grid grid-cols-1 md:grid-cols-2 gap-3">';
|
||||
|
||||
for (const [category, result] of Object.entries(diagnostics)) {
|
||||
const status = result.status === 'ok' ? '✅ PASS' : (result.status === 'warning' ? '⚠️ WARNING' : '❌ FAIL');
|
||||
const statusColor = result.status === 'ok' ? 'text-pm-success' : (result.status === 'warning' ? 'text-pm-warning' : 'text-pm-error');
|
||||
|
||||
html += `
|
||||
<div class="flex justify-between items-center p-2 bg-pm-input rounded">
|
||||
<span class="text-pm-secondary capitalize">${escapeHtml(category)}</span>
|
||||
<span class="${statusColor} font-mono text-sm">${status}</span>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
html += '</div></div>';
|
||||
|
||||
// Detailed results
|
||||
for (const [category, result] of Object.entries(diagnostics)) {
|
||||
const bgColor = result.status === 'ok' ? 'bg-pm-success-tint border-pm-success' :
|
||||
(result.status === 'warning' ? 'bg-pm-warning/20 border-pm-warning' : 'bg-pm-error-tint border-pm-error');
|
||||
|
||||
html += `<div class="${bgColor} border rounded-pm-sm p-4">`;
|
||||
html += `<h4 class="text-pm font-medium mb-2 capitalize">${escapeHtml(category)}</h4>`;
|
||||
|
||||
if (result.message) {
|
||||
html += `<p class="text-pm-secondary mb-2">${escapeHtml(result.message)}</p>`;
|
||||
}
|
||||
|
||||
// Show specific details based on category
|
||||
if (category === 'database' && result.status === 'ok') {
|
||||
html += `<div class="text-sm text-pm-secondary">`;
|
||||
html += `<p>Prompts: ${escapeHtml(result.prompt_count || 0)}</p>`;
|
||||
html += `<p>Images table: ${result.has_images_table ? 'Yes' : 'No'}</p>`;
|
||||
html += `</div>`;
|
||||
}
|
||||
|
||||
if (category === 'images_table' && result.status === 'ok') {
|
||||
html += `<div class="text-sm text-pm-secondary">`;
|
||||
html += `<p>Images: ${escapeHtml(result.image_count || 0)}</p>`;
|
||||
if (result.recent_images && result.recent_images.length > 0) {
|
||||
html += `<p>Recent images:</p>`;
|
||||
html += `<ul class="ml-4 list-disc">`;
|
||||
result.recent_images.slice(0, 3).forEach(img => {
|
||||
html += `<li>${escapeHtml(img.filename)} → Prompt ${escapeHtml(img.prompt_id)}</li>`;
|
||||
});
|
||||
html += `</ul>`;
|
||||
}
|
||||
html += `</div>`;
|
||||
}
|
||||
|
||||
if (category === 'comfyui_output' && result.output_dirs) {
|
||||
html += `<div class="text-sm text-pm-secondary">`;
|
||||
html += `<p>Output directories found:</p>`;
|
||||
html += `<ul class="ml-4 list-disc">`;
|
||||
result.output_dirs.forEach(dir => {
|
||||
html += `<li>${escapeHtml(dir)}</li>`;
|
||||
});
|
||||
html += `</ul>`;
|
||||
html += `</div>`;
|
||||
}
|
||||
|
||||
if (category === 'dependencies' && result.dependencies) {
|
||||
html += `<div class="text-sm text-pm-secondary">`;
|
||||
for (const [dep, available] of Object.entries(result.dependencies)) {
|
||||
const status = available ? '✅' : '❌';
|
||||
html += `<p>${status} ${escapeHtml(dep)}</p>`;
|
||||
}
|
||||
html += `</div>`;
|
||||
}
|
||||
|
||||
html += '</div>';
|
||||
}
|
||||
|
||||
html += '</div>';
|
||||
content.innerHTML = html;
|
||||
}
|
||||
|
||||
async testImageLink() {
|
||||
if (this.prompts.length === 0) {
|
||||
this.showNotification('No prompts available for testing', 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
// Use the first prompt for testing
|
||||
const testPrompt = this.prompts[0];
|
||||
|
||||
try {
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/diagnostics/test-link', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
prompt_id: testPrompt.id,
|
||||
image_path: '/test/fake/image.png'
|
||||
})
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
if (data.success) {
|
||||
this.showNotification('✅ Test image link created successfully!', 'success');
|
||||
// Refresh the gallery to show the test image
|
||||
setTimeout(() => this.refreshList(), 1000);
|
||||
} else {
|
||||
throw new Error(data.result?.message || 'Test failed');
|
||||
}
|
||||
} else {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Test link error:', error);
|
||||
this.showNotification(`❌ Test failed: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// Database backup and restore functionality
|
||||
async backupDatabase() {
|
||||
try {
|
||||
@@ -2321,6 +2497,273 @@
|
||||
this.showNotification(`Scan failed: ${state.message}`, "error");
|
||||
}
|
||||
|
||||
// Logs functionality
|
||||
showLogsModal() {
|
||||
this.showModal("logsModal");
|
||||
this.loadLogStats();
|
||||
this.loadLogFiles();
|
||||
this.loadLogConfig();
|
||||
}
|
||||
|
||||
closeLogs() {
|
||||
this.hideModal("logsModal");
|
||||
}
|
||||
|
||||
async loadLogStats() {
|
||||
try {
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/logs/stats');
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
const stats = data.stats;
|
||||
document.getElementById("logStatsTotal").textContent = stats.buffer_count || 0;
|
||||
document.getElementById("logStatsErrors").textContent = stats.level_counts?.ERROR || 0;
|
||||
document.getElementById("logStatsWarnings").textContent = stats.level_counts?.WARNING || 0;
|
||||
document.getElementById("logStatsLevel").textContent = stats.current_level || "INFO";
|
||||
document.getElementById("logStatsSize").textContent = this.formatBytes(stats.total_log_size || 0);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to load log stats:", error);
|
||||
}
|
||||
}
|
||||
|
||||
async loadLogConfig() {
|
||||
try {
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/logs/config');
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
const config = data.config;
|
||||
document.getElementById("setLogLevel").value = config.level || "INFO";
|
||||
document.getElementById("consoleLogging").checked = config.console_logging !== false;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to load log config:", error);
|
||||
}
|
||||
}
|
||||
|
||||
async loadLogFiles() {
|
||||
try {
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/logs/files');
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
const container = document.getElementById("logFilesList");
|
||||
|
||||
if (data.files.length === 0) {
|
||||
container.innerHTML = '<div class="col-span-full text-center text-pm-muted">No log files found</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = data.files.map(file => `
|
||||
<div class="bg-pm-surface rounded-pm-sm p-3">
|
||||
<div class="flex items-center justify-between mb-2">
|
||||
<span class="text-sm font-medium text-pm">${escapeHtml(file.filename)}</span>
|
||||
${file.is_main ? '<span class="bg-pm-accent text-xs px-2 py-1 rounded">Active</span>' : ''}
|
||||
</div>
|
||||
<div class="text-xs text-pm-secondary mb-2">
|
||||
Size: ${this.formatBytes(file.size)} | Modified: ${new Date(file.modified).toLocaleString()}
|
||||
</div>
|
||||
<button data-action="download-log" data-filename="${escapeHtml(file.filename)}"
|
||||
class="w-full px-3 py-1 bg-pm-success hover:bg-pm-success text-pm text-xs rounded transition-colors">
|
||||
📥 Download
|
||||
</button>
|
||||
</div>
|
||||
`).join('');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to load log files:", error);
|
||||
}
|
||||
}
|
||||
|
||||
async refreshLogs() {
|
||||
const level = document.getElementById("logLevel").value;
|
||||
const limit = parseInt(document.getElementById("logLimit").value);
|
||||
|
||||
try {
|
||||
const url = new URL(ApiClient.resolveUrl('/prompt_manager/logs'), window.location.origin);
|
||||
if (level) url.searchParams.set('level', level);
|
||||
url.searchParams.set('limit', limit.toString());
|
||||
|
||||
const response = await ApiClient.fetchUrl(url);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
this.displayLogs(data.logs);
|
||||
this.showNotification(`Loaded ${data.logs.length} log entries`, "success");
|
||||
} else {
|
||||
this.showNotification(`Failed to load logs: ${data.error}`, "error");
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to refresh logs:", error);
|
||||
this.showNotification("Failed to refresh logs", "error");
|
||||
}
|
||||
}
|
||||
|
||||
displayLogs(logs) {
|
||||
const container = document.getElementById("logsContainer");
|
||||
|
||||
if (logs.length === 0) {
|
||||
container.innerHTML = '<div class="text-center py-4 text-pm-muted">No logs found</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = logs.map(log => {
|
||||
const levelColors = {
|
||||
DEBUG: 'text-pm-secondary',
|
||||
INFO: 'text-pm-accent',
|
||||
WARNING: 'text-pm-warning',
|
||||
ERROR: 'text-pm-error',
|
||||
CRITICAL: 'text-pm-error'
|
||||
};
|
||||
|
||||
const levelColor = levelColors[log.level] || 'text-pm-secondary';
|
||||
const timestamp = new Date(log.timestamp).toLocaleString();
|
||||
|
||||
return `
|
||||
<div class="border-l-2 border-pm pl-3 py-1 hover:bg-pm-surface transition-colors">
|
||||
<div class="flex items-start space-x-2 text-sm">
|
||||
<span class="text-pm-muted text-xs font-mono w-24 flex-shrink-0">${escapeHtml(timestamp.split(' ')[1])}</span>
|
||||
<span class="${levelColor} font-semibold w-16 flex-shrink-0">${escapeHtml(log.level)}</span>
|
||||
<span class="text-pm-accent text-xs w-32 flex-shrink-0">${escapeHtml(log.logger)}</span>
|
||||
<span class="text-pm-secondary flex-1">${escapeHtml(log.message)}</span>
|
||||
</div>
|
||||
<div class="text-xs text-pm-muted ml-44">
|
||||
${escapeHtml(log.filename)}:${escapeHtml(log.lineno)}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
|
||||
// Auto-scroll to bottom
|
||||
container.scrollTop = container.scrollHeight;
|
||||
}
|
||||
|
||||
async downloadLogs() {
|
||||
const level = document.getElementById("logLevel").value;
|
||||
const limit = parseInt(document.getElementById("logLimit").value);
|
||||
|
||||
try {
|
||||
const url = new URL(ApiClient.resolveUrl('/prompt_manager/logs'), window.location.origin);
|
||||
if (level) url.searchParams.set('level', level);
|
||||
url.searchParams.set('limit', limit.toString());
|
||||
|
||||
const response = await ApiClient.fetchUrl(url);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
const logText = data.logs.map(log =>
|
||||
`${log.timestamp} [${log.level}] ${log.logger}: ${log.message} (${log.filename}:${log.lineno})`
|
||||
).join('\n');
|
||||
|
||||
const blob = new Blob([logText], { type: 'text/plain' });
|
||||
const url2 = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url2;
|
||||
a.download = `prompt_manager_logs_${new Date().toISOString().slice(0, 19).replace(/:/g, '-')}.txt`;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url2);
|
||||
|
||||
this.showNotification("Logs downloaded successfully", "success");
|
||||
} else {
|
||||
this.showNotification(`Failed to download logs: ${data.error}`, "error");
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to download logs:", error);
|
||||
this.showNotification("Failed to download logs", "error");
|
||||
}
|
||||
}
|
||||
|
||||
async downloadLogFile(filename) {
|
||||
try {
|
||||
const response = await ApiClient.fetchUrl(`/prompt_manager/logs/download/${encodeURIComponent(filename)}`);
|
||||
|
||||
if (response.ok) {
|
||||
const blob = await response.blob();
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
|
||||
this.showNotification(`Downloaded ${filename}`, "success");
|
||||
} else {
|
||||
this.showNotification(`Failed to download ${filename}`, "error");
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to download log file:", error);
|
||||
this.showNotification("Failed to download log file", "error");
|
||||
}
|
||||
}
|
||||
|
||||
async clearLogs() {
|
||||
if (!confirm("Are you sure you want to clear all log files? This cannot be undone.")) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/logs/truncate', {
|
||||
method: 'POST'
|
||||
});
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
this.showNotification(data.message, "success");
|
||||
this.refreshLogs();
|
||||
this.loadLogStats();
|
||||
this.loadLogFiles();
|
||||
} else {
|
||||
this.showNotification(`Failed to clear logs: ${data.error}`, "error");
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to clear logs:", error);
|
||||
this.showNotification("Failed to clear logs", "error");
|
||||
}
|
||||
}
|
||||
|
||||
async updateLogConfig() {
|
||||
const level = document.getElementById("setLogLevel").value;
|
||||
const consoleLogging = document.getElementById("consoleLogging").checked;
|
||||
|
||||
try {
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/logs/config', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
level: level,
|
||||
console_logging: consoleLogging
|
||||
})
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
this.showNotification("Log configuration updated", "success");
|
||||
this.loadLogStats();
|
||||
} else {
|
||||
this.showNotification(`Failed to update config: ${data.error}`, "error");
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to update log config:", error);
|
||||
this.showNotification("Failed to update log configuration", "error");
|
||||
}
|
||||
}
|
||||
|
||||
formatBytes(bytes) {
|
||||
if (bytes === 0) return '0 B';
|
||||
const k = 1024;
|
||||
const sizes = ['B', 'KB', 'MB', 'GB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
// Pagination methods
|
||||
updatePaginationControls() {
|
||||
const { currentPage, totalPages, total, limit } = this.pagination;
|
||||
@@ -3358,7 +3801,7 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
} else if (data.type === 'complete') {
|
||||
this.autoTagState.eventSource.close();
|
||||
this.hideModal("autoTagProgressModal");
|
||||
this.showNotification(ImageHelpers.autotagCompleteMessage(data.tagged), 'success');
|
||||
this.showNotification(`Auto tagging complete! Applied tags to ${data.tagged || 0} prompts.`, 'success');
|
||||
this.refreshList(); // Refresh the prompt list
|
||||
} else if (data.type === 'error') {
|
||||
this.autoTagState.eventSource.close();
|
||||
@@ -3376,7 +3819,7 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
console.error('EventSource readyState:', this.autoTagState.eventSource.readyState);
|
||||
this.autoTagState.eventSource.close();
|
||||
this.hideModal("autoTagProgressModal");
|
||||
this.showNotification((error && error.serverMessage) || 'Connection lost during auto tagging', 'error');
|
||||
this.showNotification('Connection lost during auto tagging', 'error');
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Auto tag error:', error);
|
||||
@@ -3451,14 +3894,10 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
}
|
||||
}
|
||||
|
||||
/** POST /autotag/single for `image` (#379: `existing` "skip" answers already_tagged for an auto-tagged image). */
|
||||
async requestReviewTags(image, existing) {
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/autotag/single', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(ImageHelpers.autotagSingleBody(image, { ...this.autoTagState, existing }))
|
||||
});
|
||||
return response.json();
|
||||
// Check if tags array has "real" tags (excluding auto-scanned and prepend:* metadata)
|
||||
getRealTags(tags) {
|
||||
if (!tags || !Array.isArray(tags)) return [];
|
||||
return tags.filter(tag => tag !== 'auto-scanned' && !tag.startsWith('prepend:') && !tag.startsWith('append:'));
|
||||
}
|
||||
|
||||
// Show the re-tag confirmation modal
|
||||
@@ -3510,48 +3949,26 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
// always provides a servable url; never rebuild it from the path.
|
||||
const imageUrl = this.getImageUrl(image);
|
||||
|
||||
this.autoTagState.currentImage = image;
|
||||
this.autoTagState.currentSha256 = null;
|
||||
this.autoTagState.tagsExpanded = false; // Reset accordion for new image
|
||||
// Check if image already has real tags (excluding auto-scanned)
|
||||
const realTags = this.getRealTags(image.prompt_tags);
|
||||
|
||||
let data;
|
||||
try {
|
||||
// Each image is first asked with "skip": an image a tagging model already tagged answers
|
||||
// already_tagged (its image tags, never its prompt's) without running the model (#379).
|
||||
data = await this.requestReviewTags(image, 'skip');
|
||||
if (data.success && data.already_tagged) {
|
||||
if (this.autoTagState.skipAllTagged) {
|
||||
this.autoTagState.reviewIndex++;
|
||||
return this.loadNextReviewImage();
|
||||
}
|
||||
const choice = await this.showRetagConfirmation(image, imageUrl, data.image_tags || []);
|
||||
if (choice === 'skip' || choice === 'skipAll') {
|
||||
this.autoTagState.reviewIndex++;
|
||||
return this.loadNextReviewImage();
|
||||
}
|
||||
// choice === 'retag': generate new tags for it
|
||||
this.showReviewImage(image, imageUrl);
|
||||
data = await this.requestReviewTags(image, 'merge');
|
||||
} else {
|
||||
this.showReviewImage(image, imageUrl);
|
||||
if (realTags.length > 0) {
|
||||
// Skip if "Skip All Tagged" was selected
|
||||
if (this.autoTagState.skipAllTagged) {
|
||||
this.autoTagState.reviewIndex++;
|
||||
return this.loadNextReviewImage();
|
||||
}
|
||||
if (data.success) {
|
||||
this.autoTagState.currentTags = data.tags;
|
||||
this.autoTagState.currentSha256 = data.sha256 || null;
|
||||
this.renderReviewTags();
|
||||
} else {
|
||||
document.getElementById('reviewTagsVisible').innerHTML =
|
||||
`<div class="text-pm-error">Error: ${escapeHtml(data.error)}</div>`;
|
||||
|
||||
// Show confirmation modal
|
||||
const choice = await this.showRetagConfirmation(image, imageUrl, realTags);
|
||||
|
||||
if (choice === 'skip' || choice === 'skipAll') {
|
||||
this.autoTagState.reviewIndex++;
|
||||
return this.loadNextReviewImage();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error generating tags:', error);
|
||||
this.showReviewImage(image, imageUrl);
|
||||
document.getElementById('reviewTagsVisible').innerHTML =
|
||||
'<div class="text-pm-error">Failed to generate tags</div>';
|
||||
// choice === 'retag' - continue with tagging
|
||||
}
|
||||
}
|
||||
|
||||
showReviewImage(image, imageUrl) {
|
||||
document.getElementById('reviewImage').src = imageUrl;
|
||||
// The record's file name, else the last segment of its path (#328: `filename` was never declared).
|
||||
const filename = image.filename || String(image.image_path || image.path || '').split(/[\\/]/).pop();
|
||||
@@ -3560,6 +3977,37 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
document.getElementById('reviewTagsHidden').innerHTML = '';
|
||||
document.getElementById('reviewTagsAccordion').classList.add('hidden');
|
||||
document.getElementById('reviewTagsToggle').classList.add('hidden');
|
||||
|
||||
// Store prompt_id from database for applying tags later
|
||||
this.autoTagState.currentPromptId = image.prompt_id;
|
||||
this.autoTagState.tagsExpanded = false; // Reset accordion for new image
|
||||
|
||||
try {
|
||||
// Review rows come from /images/all (database ids), so this resolves to image_id
|
||||
const requestBody = ImageHelpers.autotagSingleBody(image, this.autoTagState);
|
||||
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/autotag/single', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(requestBody)
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
this.autoTagState.currentTags = data.tags;
|
||||
// Use prompt_id from database, fallback to API response
|
||||
this.autoTagState.currentPromptId = image.prompt_id || data.prompt_id;
|
||||
this.renderReviewTags();
|
||||
} else {
|
||||
document.getElementById('reviewTagsVisible').innerHTML =
|
||||
`<div class="text-pm-error">Error: ${escapeHtml(data.error)}</div>`;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error generating tags:', error);
|
||||
document.getElementById('reviewTagsVisible').innerHTML =
|
||||
'<div class="text-pm-error">Failed to generate tags</div>';
|
||||
}
|
||||
}
|
||||
|
||||
/** /prompt_manager/images/all is capped per call; collect every page. */
|
||||
@@ -3678,9 +4126,15 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
}
|
||||
|
||||
async applyReviewTags() {
|
||||
// #379: the reviewed tags go to this image (linked or not), never its prompt; an empty
|
||||
// list still marks the image as reviewed by a tagging model.
|
||||
if (!this.autoTagState.currentSha256) {
|
||||
if (this.autoTagState.currentTags.length === 0) {
|
||||
this.skipReviewImage();
|
||||
return;
|
||||
}
|
||||
|
||||
const promptId = this.autoTagState.currentPromptId;
|
||||
|
||||
if (!promptId) {
|
||||
this.showNotification('No linked prompt found for this image', 'warning');
|
||||
this.skipReviewImage();
|
||||
return;
|
||||
}
|
||||
@@ -3689,8 +4143,10 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/autotag/apply', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(ImageHelpers.autotagApplyBody(
|
||||
this.autoTagState.currentImage, this.autoTagState.currentSha256, this.autoTagState.currentTags))
|
||||
body: JSON.stringify({
|
||||
prompt_id: promptId,
|
||||
tags: this.autoTagState.currentTags
|
||||
})
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
+16
-20
@@ -3560,24 +3560,19 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
} else if (data.type === 'complete') {
|
||||
this.autoTagState.eventSource.close();
|
||||
this.hideModal("autoTagProgressModal");
|
||||
this.showNotification(ImageHelpers.autotagCompleteMessage(data.tagged), 'success');
|
||||
this.showNotification(`Auto tagging complete! Applied tags to ${data.tagged || 0} prompts.`, 'success');
|
||||
this.loadImages(); // Refresh the gallery
|
||||
} else if (data.type === 'error') {
|
||||
this.autoTagState.eventSource.close();
|
||||
this.hideModal("autoTagProgressModal");
|
||||
this.showNotification(`Auto tag error: ${data.message}`, 'error');
|
||||
} else if (data.type === 'cancelled') {
|
||||
// Cancelled in Prompt Manager's Activity (#379): as its own cancel.
|
||||
this.autoTagState.eventSource.close();
|
||||
this.hideModal("autoTagProgressModal");
|
||||
this.showNotification('Auto tagging cancelled', 'info');
|
||||
}
|
||||
};
|
||||
|
||||
this.autoTagState.eventSource.onerror = (error) => {
|
||||
this.autoTagState.eventSource.onerror = () => {
|
||||
this.autoTagState.eventSource.close();
|
||||
this.hideModal("autoTagProgressModal");
|
||||
this.showNotification((error && error.serverMessage) || 'Connection lost during auto tagging', 'error');
|
||||
this.showNotification('Connection lost during auto tagging', 'error');
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Auto tag error:', error);
|
||||
@@ -3634,8 +3629,6 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
this.reviewRequest = request;
|
||||
this.autoTagState.currentTags = [];
|
||||
this.autoTagState.reviewPromptId = null;
|
||||
this.autoTagState.reviewImage = null;
|
||||
this.autoTagState.reviewSha256 = null;
|
||||
if (this.autoTagState.reviewIndex >= this.autoTagState.reviewImages.length) {
|
||||
this.hideModal("autoTagReviewModal");
|
||||
this.showNotification('Review complete!', 'success');
|
||||
@@ -3650,9 +3643,8 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
document.getElementById('reviewTagsContainer').innerHTML = '<div class="text-pm-secondary">Generating tags...</div>';
|
||||
|
||||
try {
|
||||
// Gallery entries come from /images/output: id is a path digest, so address by path.
|
||||
// The gallery review always suggests (it never skipped tagged images, #379).
|
||||
const requestBody = ImageHelpers.autotagSingleBody(image, { ...this.autoTagState, existing: 'merge' });
|
||||
// Gallery entries come from /images/output: id is a path digest, so address by path
|
||||
const requestBody = ImageHelpers.autotagSingleBody(image, this.autoTagState);
|
||||
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/autotag/single', {
|
||||
method: 'POST',
|
||||
@@ -3665,8 +3657,6 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
if (this.reviewRequest !== request) return;
|
||||
if (data.success) {
|
||||
this.autoTagState.reviewPromptId = data.prompt_id ?? null;
|
||||
this.autoTagState.reviewImage = image;
|
||||
this.autoTagState.reviewSha256 = data.sha256 || null;
|
||||
this.autoTagState.currentTags = data.tags;
|
||||
this.renderReviewTags();
|
||||
} else {
|
||||
@@ -3707,12 +3697,16 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
}
|
||||
|
||||
async applyReviewTags() {
|
||||
// #379: the reviewed tags go to this image (linked or not), never its prompt; an empty
|
||||
// list still marks the image as reviewed by a tagging model.
|
||||
if (!this.autoTagState.reviewSha256) {
|
||||
if (this.autoTagState.currentTags.length === 0) {
|
||||
this.skipReviewImage();
|
||||
return;
|
||||
}
|
||||
|
||||
const promptId = this.autoTagState.reviewPromptId;
|
||||
if (!promptId) {
|
||||
this.showNotification('This image has no linked prompt. Scan the output folder first.', 'warning');
|
||||
return;
|
||||
}
|
||||
const request = this.reviewRequest;
|
||||
if (this.reviewApplying === request) return;
|
||||
this.reviewApplying = request;
|
||||
@@ -3721,8 +3715,10 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/autotag/apply', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(ImageHelpers.autotagApplyBody(
|
||||
this.autoTagState.reviewImage, this.autoTagState.reviewSha256, this.autoTagState.currentTags))
|
||||
body: JSON.stringify({
|
||||
prompt_id: promptId,
|
||||
tags: this.autoTagState.currentTags
|
||||
})
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
+10
-44
@@ -70,28 +70,10 @@
|
||||
* path digest as id, which would 404, so those are addressed by path.
|
||||
*
|
||||
* @param {{id?: unknown, path?: unknown}} image
|
||||
* `existing` (#379): "skip" answers `already_tagged` for an image a model
|
||||
* already tagged; anything else is "merge" (always suggest).
|
||||
*
|
||||
* @param {{id?: unknown, path?: unknown}} image
|
||||
* @param {{modelType: string, generalThreshold?: number, characterThreshold?: number, prompt?: string, existing?: string}} settings
|
||||
* @param {{modelType: string, generalThreshold?: number, characterThreshold?: number, prompt?: string}} settings
|
||||
* @returns {object} JSON-serialisable request body
|
||||
*/
|
||||
function autotagSingleBody(image, settings) {
|
||||
const target = autotagTarget(image);
|
||||
const existing = settings.existing === "skip" ? "skip" : "merge";
|
||||
const modelType = String(settings.modelType || "");
|
||||
const params = modelType.startsWith("wd14")
|
||||
? {
|
||||
general_threshold: settings.generalThreshold,
|
||||
character_threshold: settings.characterThreshold,
|
||||
}
|
||||
: { prompt: settings.prompt };
|
||||
return { ...target, model_type: modelType, existing, ...params };
|
||||
}
|
||||
|
||||
/** The image a legacy single/apply body names: its row id, or its path (and root). */
|
||||
function autotagTarget(image) {
|
||||
const entry = image || {};
|
||||
const target =
|
||||
typeof entry.path === "string" && entry.path && !Number.isInteger(entry.id)
|
||||
@@ -100,22 +82,14 @@
|
||||
if ("path" in target && Number.isInteger(entry.root_index)) {
|
||||
target.root_index = entry.root_index;
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
/**
|
||||
* Body for POST /prompt_manager/autotag/apply (#379): the image, the
|
||||
* `sha256` /autotag/single answered and the reviewed tags. The tags go to
|
||||
* that image only, never to its prompt; an empty list marks it reviewed.
|
||||
*/
|
||||
function autotagApplyBody(image, sha256, tags) {
|
||||
return { ...autotagTarget(image), sha256: String(sha256 || ""), tags: Array.isArray(tags) ? [...tags] : [] };
|
||||
}
|
||||
|
||||
/** The legacy batch's completion toast (#379): images, never prompts. */
|
||||
function autotagCompleteMessage(tagged) {
|
||||
const count = Number.isSafeInteger(tagged) && tagged > 0 ? tagged : 0;
|
||||
return `Auto tagging complete! Tagged ${count} ${count === 1 ? "image" : "images"}.`;
|
||||
const modelType = String(settings.modelType || "");
|
||||
const params = modelType.startsWith("wd14")
|
||||
? {
|
||||
general_threshold: settings.generalThreshold,
|
||||
character_threshold: settings.characterThreshold,
|
||||
}
|
||||
: { prompt: settings.prompt };
|
||||
return { ...target, model_type: modelType, ...params };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -148,15 +122,7 @@
|
||||
}
|
||||
}
|
||||
|
||||
const api = {
|
||||
formatImageCaption,
|
||||
workflowDocumentHtml,
|
||||
autotagSingleBody,
|
||||
autotagApplyBody,
|
||||
autotagCompleteMessage,
|
||||
fallbackImageSource,
|
||||
metadataLookupUrl,
|
||||
};
|
||||
const api = { formatImageCaption, workflowDocumentHtml, autotagSingleBody, fallbackImageSource, metadataLookupUrl };
|
||||
if (typeof module !== "undefined" && module.exports) module.exports = api;
|
||||
else root.ImageHelpers = api;
|
||||
})(typeof window !== "undefined" ? window : globalThis);
|
||||
|
||||
+1
-15
@@ -59,20 +59,6 @@
|
||||
* @returns {{onmessage: Function|null, onerror: Function|null, onclose: Function|null,
|
||||
* readyState: number, close: Function, CONNECTING: number, OPEN: number, CLOSED: number}}
|
||||
*/
|
||||
// A non-2xx answer as an Error carrying its status and, when the body is
|
||||
// the API's JSON {error}, that sentence as serverMessage for the page.
|
||||
async function httpError(response) {
|
||||
const error = new Error(`HTTP ${response.status}`);
|
||||
error.status = response.status;
|
||||
try {
|
||||
const data = await response.json();
|
||||
if (data && typeof data.error === "string" && data.error) error.serverMessage = data.error;
|
||||
} catch (_) {
|
||||
// Not JSON: the status alone.
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
function connect(url, { method = "GET", body, fetchImpl } = {}) {
|
||||
const doFetch = fetchImpl || ((...args) => globalThis.fetch(...args));
|
||||
const controller = new AbortController();
|
||||
@@ -117,7 +103,7 @@
|
||||
if (body !== undefined) init.body = body;
|
||||
const requestUrl = root.ApiClient ? root.ApiClient.resolveUrl(url) : url;
|
||||
const response = await doFetch(requestUrl, init);
|
||||
if (!response.ok) throw await httpError(response);
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
if (source.readyState === CLOSED) return;
|
||||
source.readyState = OPEN;
|
||||
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
/**
|
||||
* File-manager selection on the Images page (#392, D135): pure helpers for the selection bar (images-selection.mjs).
|
||||
* Keys are image keys (`root_index:path`) in the listing's display order, so a range spans every loaded batch
|
||||
* whether or not its cards are mounted.
|
||||
*/
|
||||
|
||||
/** The press that turns on touch selection mode, and how far it may drift before it counts as a scroll. */
|
||||
export const LONG_PRESS_MS = 500;
|
||||
export const LONG_PRESS_SLOP = 10;
|
||||
|
||||
/**
|
||||
* The keys from `from` to `to`, inclusive, in `keys` order (either direction). Without a listed `from` (no anchor
|
||||
* yet, or it left the listing) the range is `to` alone; an unlisted `to` gives none.
|
||||
*/
|
||||
export function rangeKeys(keys, from, to) {
|
||||
const end = keys.indexOf(to);
|
||||
if (end < 0) return [];
|
||||
const start = from === null || from === undefined ? -1 : keys.indexOf(from);
|
||||
if (start < 0) return [to];
|
||||
return keys.slice(Math.min(start, end), Math.max(start, end) + 1);
|
||||
}
|
||||
|
||||
/** `selected` with `keys` added (`on`) or removed, frozen; the same set when nothing changes. */
|
||||
export function applyKeys(selected, keys, on) {
|
||||
if (keys.every(key => selected.has(key) === on)) return selected;
|
||||
const next = new Set(selected);
|
||||
for (const key of keys) {
|
||||
if (on) next.add(key); else next.delete(key);
|
||||
}
|
||||
return Object.freeze(next);
|
||||
}
|
||||
|
||||
/** Whether `keys` is non-empty and every one of them is in `selected`. */
|
||||
export function everyChecked(selected, keys) {
|
||||
return keys.length > 0 && selected.size >= keys.length && keys.every(key => selected.has(key));
|
||||
}
|
||||
|
||||
/** "Select all N matching" shows once every loaded image is checked and the listing holds more than is loaded. */
|
||||
export function offersAllMatching(selected, keys, { total, hasMore }) {
|
||||
return Boolean(hasMore) && total > keys.length && everyChecked(selected, keys);
|
||||
}
|
||||
|
||||
const grouped = count => count.toLocaleString('en-US');
|
||||
export const matchingLabel = total => `All ${grouped(total)} matching selected`;
|
||||
export const matchingOffer = total => `Select all ${grouped(total)} matching`;
|
||||
|
||||
/**
|
||||
* A touch long-press: `down` starts the timer, a `move` past the slop, `up` or `cancel` stops it, and
|
||||
* `onLongPress(target)` runs once it fires. `consumeClick()` answers whether the click the press ends in belongs to
|
||||
* the long press (so the page swallows it); it answers once. A new `down` forgets an unclaimed long press.
|
||||
*/
|
||||
export function createLongPress({ onLongPress, delay = LONG_PRESS_MS, slop = LONG_PRESS_SLOP,
|
||||
setTimer = setTimeout, clearTimer = clearTimeout }) {
|
||||
let timer = null, origin = null, fired = false;
|
||||
const stop = () => {
|
||||
if (timer !== null) clearTimer(timer);
|
||||
timer = null; origin = null;
|
||||
};
|
||||
return {
|
||||
down(x, y, target) {
|
||||
stop();
|
||||
fired = false;
|
||||
origin = { x, y };
|
||||
timer = setTimer(() => { timer = null; origin = null; fired = true; onLongPress(target); }, delay);
|
||||
},
|
||||
move(x, y) {
|
||||
if (origin && Math.hypot(x - origin.x, y - origin.y) > slop) stop();
|
||||
},
|
||||
up: stop,
|
||||
/** A press that ends without its click (a mouse press next, a scroll): its long press is forgotten too. */
|
||||
cancel() { stop(); fired = false; },
|
||||
/** Whether a touch press is held and has not yet fired (its context menu belongs to the long press). */
|
||||
pending: () => timer !== null,
|
||||
consumeClick() {
|
||||
const claimed = fired;
|
||||
fired = false;
|
||||
return claimed;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -4,16 +4,11 @@ import { inspectPath, toInspectionRecord } from './inspection-state.mjs';
|
||||
import { BUNDLE_FILENAME, metadataBundle, collectImagesMetadata, exportOutcome } from './images-export.mjs';
|
||||
import { CATEGORY_DIALOG, CATEGORY_FAILED, CATEGORY_MAX_LENGTH, CATEGORY_NONE_LINKED, categoryOutcome, categoryTargets, updatedCount } from './images-category.mjs';
|
||||
import { openInputDialog } from './workspace-dialogs.mjs';
|
||||
import { MAX_FILTER_IMAGES, TOO_MANY_MATCHING, openAddTags, openAutoTag } from './images-tagging-bulk.mjs';
|
||||
import { openAddTags, openAutoTag } from './images-tagging-bulk.mjs';
|
||||
import { showWorkspaceToast } from './toast.mjs';
|
||||
import { applyKeys, createLongPress, matchingLabel, matchingOffer, offersAllMatching, rangeKeys } from './images-multiselect.mjs';
|
||||
|
||||
const CHECKBOX = '.image-overlays input[type="checkbox"]';
|
||||
const CHECKBOX_LABEL = '.image-overlays > label';
|
||||
const CARD = 'article[data-image-key]';
|
||||
const CATEGORY_URL = '/prompt_manager/bulk/category';
|
||||
// D135: Export metadata reads every image's inspection in the browser, so it waits for loaded images.
|
||||
export const EXPORT_LOADED_ONLY = 'Export metadata reads each image in the browser, so it works on loaded images only.';
|
||||
|
||||
function el(tag, text, className) {
|
||||
const element = document.createElement(tag);
|
||||
@@ -34,196 +29,60 @@ function el(tag, text, className) {
|
||||
* - `retain(kept)` drops the checked files a refreshed listing no longer holds (a tag save under a filter, I7).
|
||||
* - With the Activity store `activity`, Add tags and Auto-tag selected (D85) follow the count, as the concept's Add tags
|
||||
* leads its actions; `onTagged(keys)` re-reads the files a finished tagging job tagged. The selection stays.
|
||||
*
|
||||
* File-manager selection (#392, D135, beyond the concept's checkbox): the anchor is the last image toggled on its own;
|
||||
* Shift+click (a checkbox or anywhere on a card) applies the anchor's range, Ctrl/Cmd+click toggles a card without
|
||||
* opening it, a touch long-press starts touch selection mode in which taps toggle. `selectAll`, `extend` and
|
||||
* `clear` serve the grid's keys (images.mjs). `getListing()` gives `{total, hasMore, positiveCount}` and
|
||||
* `getFilter()` the listing filter: once every loaded image is checked and more match, the bar offers "Select all N
|
||||
* matching", whose actions send that filter to the server.
|
||||
*/
|
||||
export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, onCategorySet, activity = null, onTagged,
|
||||
getListing, getFilter, checked: initial = Object.freeze(new Set()) }) {
|
||||
checked: initial = Object.freeze(new Set()) }) {
|
||||
let checked = initial;
|
||||
let exporting = false, categorizing = false;
|
||||
// D135: the range anchor, "all N matching" mode, touch selection mode and a pending Shift+click on a checkbox.
|
||||
let anchor = null, allMatching = false, touchMode = false, lastPointer = 'mouse', shiftBox = null;
|
||||
const element = el('div');
|
||||
element.id = 'bulkbar';
|
||||
const bulk = el('div', undefined, 'bulkbar'), count = el('strong');
|
||||
const selectMatching = createButton('');
|
||||
scope.listen(selectMatching, 'click', () => {
|
||||
allMatching = true;
|
||||
render();
|
||||
focusAfterOffer();
|
||||
});
|
||||
const setCategory = createButton('Set category');
|
||||
scope.listen(setCategory, 'click', () => openCategory(setCategory));
|
||||
const exportMetadata = createButton('Export metadata');
|
||||
scope.listen(exportMetadata, 'click', () => { if (!allMatching) exportChecked(exportMetadata); });
|
||||
scope.listen(exportMetadata, 'click', () => exportChecked(exportMetadata));
|
||||
const clearButton = createButton('Clear');
|
||||
// Clear hides the bar it sits in, so focus moves to the page heading instead of falling to the page.
|
||||
scope.listen(clearButton, 'click', () => { clear(); focusFallback.focus({ preventScroll: true }); });
|
||||
bulk.append(count);
|
||||
if (activity) {
|
||||
const addTags = createButton('Add tags');
|
||||
scope.listen(addTags, 'click', () => openAddTags({ scope, store: activity, images: checkedImages(), trigger: addTags, onTagged,
|
||||
...matchingTarget() }));
|
||||
scope.listen(addTags, 'click', () => openAddTags({ scope, store: activity, images: checkedImages(), trigger: addTags, onTagged }));
|
||||
const autoTag = createButton('Auto-tag selected'); // D85
|
||||
scope.listen(autoTag, 'click', () => openAutoTag({ api, scope, store: activity, images: checkedImages(), trigger: autoTag, onTagged,
|
||||
...matchingTarget() }));
|
||||
scope.listen(autoTag, 'click', () => openAutoTag({ api, scope, store: activity, images: checkedImages(), trigger: autoTag, onTagged }));
|
||||
bulk.append(addTags, autoTag);
|
||||
}
|
||||
bulk.append(setCategory, exportMetadata, clearButton);
|
||||
|
||||
/** The count updates in place: the same bar and `strong`, its text written only when it changes. */
|
||||
function render() {
|
||||
if (!checked.size && !allMatching) { touchMode = false; bulk.remove(); return; }
|
||||
const listing = getListing();
|
||||
const label = allMatching ? matchingLabel(listing.total) : selectionLabel(checked.size);
|
||||
if (!checked.size) { bulk.remove(); return; }
|
||||
const label = selectionLabel(checked.size);
|
||||
if (count.textContent !== label) count.textContent = label;
|
||||
// "Select all N matching" (D135) follows the count while every loaded image is checked and more match.
|
||||
if (!allMatching && offersAllMatching(checked, loadedKeys(), listing)) {
|
||||
const offer = matchingOffer(listing.total);
|
||||
if (selectMatching.textContent !== offer) selectMatching.textContent = offer;
|
||||
if (count.nextElementSibling !== selectMatching) count.after(selectMatching);
|
||||
} else selectMatching.remove();
|
||||
if (allMatching) {
|
||||
exportMetadata.setAttribute('aria-disabled', 'true');
|
||||
exportMetadata.title = EXPORT_LOADED_ONLY;
|
||||
} else if (exportMetadata.title) {
|
||||
exportMetadata.removeAttribute('title');
|
||||
if (!exporting) exportMetadata.removeAttribute('aria-disabled');
|
||||
}
|
||||
if (bulk.parentNode !== element) element.append(bulk);
|
||||
}
|
||||
/** Every rendered checkbox shows the selection (cards mounted later read `isChecked`). */
|
||||
function paint() {
|
||||
for (const box of grid.querySelectorAll(CHECKBOX)) {
|
||||
const on = isChecked(box.closest(CARD).dataset.imageKey);
|
||||
if (box.checked !== on) box.checked = on;
|
||||
}
|
||||
}
|
||||
function update(next) {
|
||||
checked = next;
|
||||
paint();
|
||||
render();
|
||||
}
|
||||
/** Leaving "all N matching": the loaded images become the explicit selection the next change applies to. */
|
||||
function settleMatching() {
|
||||
if (!allMatching) return;
|
||||
allMatching = false;
|
||||
checked = Object.freeze(new Set(loadedKeys()));
|
||||
}
|
||||
function clear() {
|
||||
anchor = null; touchMode = false;
|
||||
if (!checked.size && !allMatching) return;
|
||||
allMatching = false;
|
||||
update(Object.freeze(new Set()));
|
||||
if (!checked.size) return;
|
||||
checked = Object.freeze(new Set());
|
||||
for (const box of grid.querySelectorAll(CHECKBOX)) box.checked = false;
|
||||
render();
|
||||
}
|
||||
/** Keep only the checked files `kept(key)` still lists (a refreshed result), so the count is what the actions apply to. */
|
||||
function retain(kept) {
|
||||
if (allMatching) { render(); return; }
|
||||
const next = [...checked].filter(kept);
|
||||
if (next.length === checked.size) return;
|
||||
checked = Object.freeze(new Set(next));
|
||||
render();
|
||||
}
|
||||
function loadedKeys() {
|
||||
return getItems().map(imageKey);
|
||||
}
|
||||
function isChecked(key) {
|
||||
return allMatching || checked.has(key);
|
||||
}
|
||||
function checkedImages() {
|
||||
return allMatching ? getItems() : getItems().filter(image => checked.has(imageKey(image)));
|
||||
}
|
||||
/** The actions' images for "all N matching": the listing filter and its total, which the server resolves. */
|
||||
function matchingTarget() {
|
||||
return allMatching ? { filter: getFilter(), count: getListing().total } : {};
|
||||
}
|
||||
/** Toggle one image on its own; it becomes the anchor. */
|
||||
function toggle(key, on = !isChecked(key)) {
|
||||
settleMatching();
|
||||
anchor = key;
|
||||
update(toggleSelection(checked, key, on));
|
||||
}
|
||||
/** The anchor's range to `key` takes `on`; without an anchor `key` is toggled on and becomes it. */
|
||||
function range(key, on = true) {
|
||||
settleMatching();
|
||||
const keys = rangeKeys(loadedKeys(), anchor, key);
|
||||
if (anchor === null || keys.length === 1) anchor = key;
|
||||
update(applyKeys(checked, keys, on));
|
||||
}
|
||||
/** Shift+arrow keys: the range from the anchor (`from`, the card focus left, when there is none) to `to`. */
|
||||
function extend(from, to) {
|
||||
if (anchor === null || !loadedKeys().includes(anchor)) anchor = from;
|
||||
range(to, true);
|
||||
}
|
||||
function selectAll() {
|
||||
if (allMatching) return;
|
||||
update(applyKeys(checked, loadedKeys(), true));
|
||||
}
|
||||
function focusAfterOffer() {
|
||||
// The pressed offer left the bar: focus moves to the count's first action instead of the page.
|
||||
(bulk.querySelector('button:not([aria-disabled="true"])') ?? focusFallback).focus({ preventScroll: true });
|
||||
return getItems().filter(image => checked.has(imageKey(image)));
|
||||
}
|
||||
// Every card carries its image key; its only form control is the checkbox, so a toggle never opens or inspects.
|
||||
scope.listen(grid, 'change', event => {
|
||||
const box = event.target;
|
||||
if (!box.matches(CHECKBOX)) return;
|
||||
const key = box.closest(CARD).dataset.imageKey;
|
||||
if (shiftBox === key) range(key, box.checked);
|
||||
else toggle(key, box.checked);
|
||||
shiftBox = null;
|
||||
});
|
||||
// D135: modified clicks select instead of opening, inspecting or starring. Captured, so the page's own click
|
||||
// handler (images.mjs) never sees them; a plain click keeps its meaning.
|
||||
grid.addEventListener('click', event => {
|
||||
const card = event.target.closest(CARD);
|
||||
if (!card) return;
|
||||
const key = card.dataset.imageKey;
|
||||
if (longPress.consumeClick()) { swallow(event); return; }
|
||||
if (event.target.closest(CHECKBOX_LABEL)) {
|
||||
// The box toggles natively; its change event applies the range.
|
||||
if (event.shiftKey) shiftBox = key;
|
||||
return;
|
||||
}
|
||||
// A keyboard click (Enter, Space: `detail` 0) keeps its plain meaning in touch selection mode.
|
||||
const tapped = touchMode && lastPointer !== 'mouse' && event.detail !== 0;
|
||||
if (event.shiftKey) { swallow(event); range(key, true); }
|
||||
else if (event.ctrlKey || event.metaKey || tapped) { swallow(event); toggle(key); }
|
||||
}, { capture: true, signal: scope.signal });
|
||||
function swallow(event) {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}
|
||||
// A touch long-press selects the card and starts touch selection mode (D135).
|
||||
const longPress = createLongPress({ onLongPress: key => {
|
||||
touchMode = true;
|
||||
anchor = key;
|
||||
if (!isChecked(key)) toggle(key, true);
|
||||
} });
|
||||
scope.cleanup(() => longPress.cancel());
|
||||
scope.listen(grid, 'pointerdown', event => {
|
||||
lastPointer = event.pointerType || 'mouse';
|
||||
shiftBox = null;
|
||||
const card = event.target.closest(CARD);
|
||||
if (lastPointer === 'mouse' || !card || !event.isPrimary) { longPress.cancel(); return; }
|
||||
longPress.down(event.clientX, event.clientY, card.dataset.imageKey);
|
||||
});
|
||||
scope.listen(grid, 'pointermove', event => longPress.move(event.clientX, event.clientY));
|
||||
for (const type of ['pointerup', 'pointercancel', 'pointerleave']) scope.listen(grid, type, () => longPress.up());
|
||||
// A long-press's context menu (Android) would cover the selection it just started.
|
||||
scope.listen(grid, 'contextmenu', event => {
|
||||
if (lastPointer !== 'mouse' && (touchMode || longPress.pending())) event.preventDefault();
|
||||
});
|
||||
// Escape in the bar clears the selection; the bar hides, so focus moves to the page heading as after Clear.
|
||||
scope.listen(element, 'keydown', event => {
|
||||
if (event.key !== 'Escape' || (!checked.size && !allMatching)) return;
|
||||
event.preventDefault();
|
||||
clear();
|
||||
focusFallback.focus({ preventScroll: true });
|
||||
checked = toggleSelection(checked, box.closest('article[data-image-key]').dataset.imageKey, box.checked);
|
||||
render();
|
||||
});
|
||||
|
||||
/** Busy until it settles: `aria-disabled` now, released in a `finally` so a throw cannot leave it stuck. */
|
||||
@@ -267,11 +126,8 @@ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, o
|
||||
*/
|
||||
function openCategory(button) {
|
||||
if (categorizing) return;
|
||||
// All matching (D135): the listing's positive count says whether any match is linked; the server resolves.
|
||||
if (allMatching && getListing().total > MAX_FILTER_IMAGES) { showWorkspaceToast(TOO_MANY_MATCHING); return; }
|
||||
const targets = allMatching ? {filter: getFilter()} : categoryTargets(checkedImages());
|
||||
const linked = allMatching ? getListing().positiveCount !== 0 : targets.promptIds.length > 0;
|
||||
if (!linked) { showWorkspaceToast(CATEGORY_NONE_LINKED); return; }
|
||||
const targets = categoryTargets(checkedImages());
|
||||
if (!targets.promptIds.length) { showWorkspaceToast(CATEGORY_NONE_LINKED); return; }
|
||||
openInputDialog({ scope, trigger: button, title: CATEGORY_DIALOG.title, label: CATEGORY_DIALOG.label,
|
||||
action: CATEGORY_DIALOG.action, maxLength: CATEGORY_MAX_LENGTH, onApply: category => applyCategory(button, targets, category) });
|
||||
}
|
||||
@@ -279,12 +135,12 @@ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, o
|
||||
* The write carries no page signal: once sent it finishes even if the page is left (`writing` then tells the page
|
||||
* not to save a visit that predates it). A failure keeps the selection; a success reloads through `onCategorySet`.
|
||||
*/
|
||||
async function applyCategory(button, { promptIds, skipped, filter }, category) {
|
||||
async function applyCategory(button, { promptIds, skipped }, category) {
|
||||
categorizing = true;
|
||||
const release = busy(button);
|
||||
let result = null;
|
||||
try {
|
||||
result = await api.post(CATEGORY_URL, filter ? { images_filter: filter, category } : { prompt_ids: promptIds, category });
|
||||
result = await api.post(CATEGORY_URL, { prompt_ids: promptIds, category });
|
||||
} catch {
|
||||
// Reported below; the selection stays as it was.
|
||||
} finally {
|
||||
@@ -293,9 +149,7 @@ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, o
|
||||
}
|
||||
if (scope.signal.aborted) return;
|
||||
if (!result) { showWorkspaceToast(CATEGORY_FAILED); return; }
|
||||
if (filter && !result.prompt_count) { showWorkspaceToast(CATEGORY_NONE_LINKED); return; }
|
||||
showWorkspaceToast(filter ? categoryOutcome(updatedCount(result, result.prompt_count), Number(result.skipped) || 0)
|
||||
: categoryOutcome(updatedCount(result, promptIds.length), skipped));
|
||||
showWorkspaceToast(categoryOutcome(updatedCount(result, promptIds.length), skipped));
|
||||
const focused = document.activeElement === button;
|
||||
onCategorySet();
|
||||
// The reload hides the bar with the focused action in it, as Clear does: focus moves to the page heading.
|
||||
@@ -305,16 +159,10 @@ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, o
|
||||
render();
|
||||
return {
|
||||
element,
|
||||
isChecked,
|
||||
// A saved visit keeps "all N matching" as its loaded images; the offer then shows again.
|
||||
checked: () => (allMatching ? Object.freeze(new Set(loadedKeys())) : checked),
|
||||
hasSelection: () => allMatching || checked.size > 0,
|
||||
isChecked: key => checked.has(key),
|
||||
checked: () => checked,
|
||||
clear,
|
||||
retain,
|
||||
selectAll,
|
||||
extend,
|
||||
/** A loaded batch or a refreshed total: the count and the offer follow it. */
|
||||
listingChanged: () => { if (checked.size || allMatching) render(); },
|
||||
/** Whether a Set category write is still in flight. */
|
||||
writing: () => categorizing,
|
||||
};
|
||||
|
||||
@@ -56,16 +56,6 @@ function queryFor(filters, batchSize) {
|
||||
return query.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* The listing's filter as the bulk routes take it (#392 "Select all N matching"): the same parameters the listing
|
||||
* query carries, as an object of strings, without `mode`, `limit` and `cursor`.
|
||||
*/
|
||||
export function listingFilter(filters) {
|
||||
const query = new URLSearchParams(queryFor(filters, 1));
|
||||
query.delete('mode'); query.delete('limit');
|
||||
return Object.fromEntries(query);
|
||||
}
|
||||
|
||||
function validatePage(page, cursor) {
|
||||
if (!page || !Array.isArray(page.images) || !Number.isSafeInteger(page.total) || page.total < 0 ||
|
||||
typeof page.has_more !== 'boolean' ||
|
||||
|
||||
@@ -9,10 +9,6 @@
|
||||
* apply** (D90, ruling T3-bulk): **Apply automatically** starts the job, **Review each image** starts the browser
|
||||
* review session (images-tagging-review.mjs). A model that is not downloaded keeps Start disabled and says why.
|
||||
*
|
||||
* - **All N matching** (#392, D135): with `filter` (the listing's, images-state.mjs `listingFilter`) and `count` (its
|
||||
* total), both send the filter instead of the targets and the server resolves it, up to 10,000 images. Review each
|
||||
* image walks images in the browser, so it is not offered.
|
||||
*
|
||||
* The toast says the job started (the Activity store's "Tagging started." / "Auto-tagging started."), or why it did
|
||||
* not: a running record-changing job (named, with its link), a held model, a refused body. Activity shows progress.
|
||||
* The selection stays either way. When a started job ends having tagged files, `onTagged(keys)` re-reads them.
|
||||
@@ -30,9 +26,6 @@ import { showWorkspaceToast } from './toast.mjs';
|
||||
export const IMAGE_TAG_JOB = 'image-tags';
|
||||
export const MAX_BULK_IMAGES = 500;
|
||||
export const TOO_MANY = 'Select at most 500 images.';
|
||||
// #392: a filter start resolves on the server (py/api/images.py MAX_LISTING_SELECTION).
|
||||
export const MAX_FILTER_IMAGES = 10000;
|
||||
export const TOO_MANY_MATCHING = 'Narrow the view to at most 10,000 images.';
|
||||
export const NO_TAGS = 'Enter at least one tag.'; // D118
|
||||
export const ADD_DIALOG = Object.freeze({title: 'Tag selected images', label: 'Tags, separated by commas', action: 'Apply'});
|
||||
export const AUTO_TITLE = 'Auto-tag selected images';
|
||||
@@ -60,16 +53,6 @@ export function bulkTargets(images) {
|
||||
return images.map(imageTarget).filter(Boolean);
|
||||
}
|
||||
|
||||
/**
|
||||
* The job body's images: `{filter}` for all matching (null over MAX_FILTER_IMAGES), else `{targets}` (null over 500).
|
||||
* Returns `[images, refusal]`, the refusal being the toast to show.
|
||||
*/
|
||||
function jobImages(images, filter, count) {
|
||||
if (filter) return count > MAX_FILTER_IMAGES ? [null, TOO_MANY_MATCHING] : [{filter}, null];
|
||||
const targets = bulkTargets(images);
|
||||
return targets ? [{targets}, null] : [null, TOO_MANY];
|
||||
}
|
||||
|
||||
/** The Add tags field's names, deduplicated as the Tag image dialog's manual field is. */
|
||||
export const parseBulkTags = parseManualTags;
|
||||
|
||||
@@ -124,13 +107,13 @@ function followTagged({store, scope, job, images, onTagged}) {
|
||||
* to no names keeps the dialog and its text, and reports NO_TAGS on the field (D118). `openInput` is the dialog seam.
|
||||
*/
|
||||
export function openAddTags({scope, store, images, trigger, onTagged = () => {}, notify = showWorkspaceToast,
|
||||
openInput = openInputDialog, filter = null, count = images.length}) {
|
||||
const [chosen, refusal] = jobImages(images, filter, count);
|
||||
if (refusal) { notify(refusal); return; }
|
||||
openInput = openInputDialog}) {
|
||||
const targets = bulkTargets(images);
|
||||
if (!targets) { notify(TOO_MANY); return; }
|
||||
openInput({scope, trigger, ...ADD_DIALOG, validate: text => (parseBulkTags(text).length ? null : NO_TAGS),
|
||||
onApply: async text => {
|
||||
const tags = parseBulkTags(text);
|
||||
const job = await startTagJob({store, scope, body: {mode: 'add', ...chosen, tags}, notify});
|
||||
const job = await startTagJob({store, scope, body: {mode: 'add', targets, tags}, notify});
|
||||
if (job) followTagged({store, scope, job, images, onTagged});
|
||||
}});
|
||||
}
|
||||
@@ -158,16 +141,15 @@ function applyChoice(onChange) {
|
||||
* Auto-tag selected (D85): the model panel, How to apply (D90), Cancel and Start. Start validates the fields as
|
||||
* Generate does, then starts the job or the review session over `images` (the checked files, taken now).
|
||||
*/
|
||||
export function openAutoTag({api, scope, store, images, trigger, onTagged = () => {}, notify = showWorkspaceToast,
|
||||
filter = null, count = images.length}) {
|
||||
const [chosen, refusal] = jobImages(images, filter, count);
|
||||
if (refusal) { notify(refusal); return; }
|
||||
export function openAutoTag({api, scope, store, images, trigger, onTagged = () => {}, notify = showWorkspaceToast}) {
|
||||
const targets = bulkTargets(images);
|
||||
if (!targets) { notify(TOO_MANY); return; }
|
||||
const listed = images.filter(imageTarget);
|
||||
const local = createScope();
|
||||
const options = createTaggingOptionsController({api});
|
||||
// D85: the Tag image dialog's model panel without Generate; its Suggestions field is the job's count (D91).
|
||||
const {element: panel, validate} = renderModelPanel({scope: local, controller: options,
|
||||
heading: `Auto-tag ${countOf(filter ? count : listed.length, 'image')}`, generate: false}); // D118
|
||||
heading: `Auto-tag ${countOf(listed.length, 'image')}`, generate: false}); // D118
|
||||
let choice = APPLY_CHOICES[0];
|
||||
const actions = el('div', undefined, 'dialog-actions');
|
||||
const cancel = createButton('Cancel');
|
||||
@@ -177,8 +159,7 @@ export function openAutoTag({api, scope, store, images, trigger, onTagged = () =
|
||||
const status = el('div', undefined, 'small');
|
||||
status.id = 'tag-generation-status';
|
||||
status.setAttribute('role', 'status');
|
||||
// D90; all matching (D135) has no Review each image, which walks images in the browser.
|
||||
const choices = filter ? null : applyChoice(next => { choice = next; start.textContent = next.action; });
|
||||
const choices = applyChoice(next => { choice = next; start.textContent = next.action; }); // D90
|
||||
const render = state => {
|
||||
if (status.textContent !== state.unavailable) status.textContent = state.unavailable;
|
||||
if (state.canStart) start.removeAttribute('aria-disabled');
|
||||
@@ -197,11 +178,11 @@ export function openAutoTag({api, scope, store, images, trigger, onTagged = () =
|
||||
return;
|
||||
}
|
||||
closeWorkspaceDialog();
|
||||
startTagJob({store, scope, body: {mode: 'auto', ...chosen, ...request}, notify}).then(job => {
|
||||
startTagJob({store, scope, body: {mode: 'auto', targets, ...request}, notify}).then(job => {
|
||||
if (job) followTagged({store, scope, job, images: listed, onTagged});
|
||||
});
|
||||
});
|
||||
openContentDialog({scope, title: AUTO_TITLE, nodes: [panel, choices, status, actions].filter(Boolean), trigger, onClose: () => {
|
||||
openContentDialog({scope, title: AUTO_TITLE, nodes: [panel, choices, status, actions], trigger, onClose: () => {
|
||||
document.getElementById('modal')?.classList.remove('tagging-modal');
|
||||
local.dispose();
|
||||
options.dispose();
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createScope, createButton } from '../shared/toolkit/core.js';
|
||||
import { createImageStream, withItem, imageKey, countLine, captionDate, scopeLabel, activeRefinements, initialImagesView, listingFilter } from './images-state.mjs';
|
||||
import { createImageStream, withItem, imageKey, countLine, captionDate, scopeLabel, activeRefinements, initialImagesView } from './images-state.mjs';
|
||||
import { createImageCard } from './image-card.mjs';
|
||||
import { layoutImages, visibleImages, gridMetrics, stacksCardInfo } from './image-layout.mjs';
|
||||
import { mountFileInspector } from './file-inspector.mjs';
|
||||
@@ -254,15 +254,8 @@ export function mountImages(main, api, parentScope, {embedded = false, fetchPage
|
||||
body.append(collection, details);
|
||||
// The checked cards and the concept's selection bar (#249 D8), between the filter row and the gallery layout. The
|
||||
// selection is kept across appended batches and cleared with the collection; the Library-embedded gallery has none.
|
||||
const bar = embedded ? null : mountSelectionBar({scope, api, grid, getItems:() => state?.items ?? [], focusFallback,
|
||||
checked:visit?.checked, onCategorySet:() => { loadCategories(); reset(); }, activity, onTagged:keys => refreshTagged(keys),
|
||||
// #392 "Select all N matching": the listing's total and its filter, as the server lists them.
|
||||
getListing:() => ({total:state?.total ?? 0, hasMore:Boolean(state?.hasMore), positiveCount:state?.positiveCount ?? null}),
|
||||
// A Source filter carries its root's scope, so a renumbered source is refused (source-changed), not acted on.
|
||||
getFilter:() => {
|
||||
const filter = listingFilter(filters), scope = state?.items[0]?.root_scope;
|
||||
return filter.root !== undefined && typeof scope === 'string' ? {...filter, root_scope:scope} : filter;
|
||||
}});
|
||||
const bar = embedded ? null : mountSelectionBar({scope, api, grid, getItems:() => state.items, focusFallback,
|
||||
checked:visit?.checked, onCategorySet:() => { loadCategories(); reset(); }, activity, onTagged:keys => refreshTagged(keys)});
|
||||
if (bar) main.append(bar.element);
|
||||
main.append(body);
|
||||
const mediaViewer = createMediaViewer({api,container:main,parentScope:scope,inspectionController:inspection,sender,
|
||||
@@ -393,19 +386,8 @@ export function mountImages(main, api, parentScope, {embedded = false, fetchPage
|
||||
const columns = geometry.columns;
|
||||
const target = { Home: 0, End: state.items.length - 1, ArrowRight: index + 1,
|
||||
ArrowLeft: index - 1, ArrowDown: index + columns, ArrowUp: index - columns }[event.key];
|
||||
if (target !== undefined) {
|
||||
event.preventDefault();
|
||||
const next = Math.max(0, Math.min(state.items.length - 1, target));
|
||||
focusImage(next);
|
||||
// D135: Shift extends the selection from the anchor (this card when there is none) to the new card.
|
||||
if (event.shiftKey && bar) bar.extend(card.dataset.imageKey, imageKey(state.items[next]));
|
||||
} else if (bar && (event.ctrlKey || event.metaKey) && !event.altKey && event.key.toLowerCase() === 'a') {
|
||||
event.preventDefault();
|
||||
bar.selectAll();
|
||||
} else if (bar && event.key === 'Escape' && bar.hasSelection()) {
|
||||
event.preventDefault();
|
||||
bar.clear();
|
||||
} else if (event.key === 'Tab') {
|
||||
if (target !== undefined) { event.preventDefault(); focusImage(Math.max(0, Math.min(state.items.length - 1, target))); }
|
||||
else if (event.key === 'Tab') {
|
||||
const backward = event.shiftKey && event.target.matches('.image-open');
|
||||
const forward = !event.shiftKey && event.target.closest('.image-card-actions');
|
||||
const adjacent = backward ? index - 1 : index + 1;
|
||||
@@ -480,7 +462,6 @@ export function mountImages(main, api, parentScope, {embedded = false, fetchPage
|
||||
if (replaced && selection !== null && !itemIndexes.has(selection)) dropSelection();
|
||||
if (replaced) bar?.retain(key => itemIndexes.has(key));
|
||||
state = next;
|
||||
bar?.listingChanged();
|
||||
if (!embedded) {
|
||||
setText(count, countLine(next));
|
||||
setText(captionDay, captionDate(next.items, filters.sort));
|
||||
|
||||
@@ -53,14 +53,7 @@ export const OPERATION_TITLES = Object.freeze({
|
||||
duplicate_files: 'Duplicate image files',
|
||||
// Reviewed and applied through the API and seen in Activity; Maintenance has no row for it (the concept has none).
|
||||
workflow_storage: 'Store each workflow once',
|
||||
// S12: Diagnostics & logs › Log files & retention › Review log cleanup (the concept's `maintenance-plan` "rotated logs").
|
||||
rotated_logs: 'Rotated log copies',
|
||||
});
|
||||
/** S12: plans whose details are files, not records ("Listed files"). */
|
||||
const FILE_PLANS = new Set(['rotated_logs']);
|
||||
/** S12: a plan that changes no library record (`backup.policy: "none"`): no backup is taken or offered. */
|
||||
export const NO_BACKUP_TEXT = 'Not needed: no library record changes';
|
||||
const noBackup = plan => plan?.backup?.policy === 'none';
|
||||
// The review is gone or no longer matches the database: review it again.
|
||||
const REVIEW_AGAIN = new Set(['plan-expired', 'storage-changed', 'plan-changed']);
|
||||
// Refusals whose server message is the whole truth.
|
||||
@@ -78,7 +71,6 @@ function el(tag, text, className) {
|
||||
|
||||
/** The Backup value: what the apply will do before it changes anything. */
|
||||
export function backupText(backup) {
|
||||
if (backup?.policy === 'none') return NO_BACKUP_TEXT;
|
||||
if (backup?.policy === 'off') return 'No safety backup (the policy is off)';
|
||||
const where = backup?.destination_label ? ` (${backup.destination_label})` : '';
|
||||
if (backup?.policy === 'required') return `Always created and verified first${where}`;
|
||||
@@ -213,18 +205,6 @@ export async function applyPlan({ api, store, scope, token, reviewAgain, notify
|
||||
return { job };
|
||||
}
|
||||
|
||||
/** The details list's summary: "Listed records (N)", or "Listed files (N)" for a plan of files (S12); "(N of M)" when cut. */
|
||||
export function detailsSummary(plan, shown) {
|
||||
const what = FILE_PLANS.has(plan?.operation) ? 'Listed files' : 'Listed records';
|
||||
const count = plan?.scope?.count ?? shown;
|
||||
return shown < count ? `${what} (${number(shown)} of ${number(count)})` : `${what} (${number(shown)})`;
|
||||
}
|
||||
|
||||
/** D55's backup box for the review, or null for a plan that changes no library record (S12, `policy: "none"`). */
|
||||
export function backupBox(plan, onBackup) {
|
||||
return noBackup(plan) ? null : { copy: BACKUP_COPY, onBackup };
|
||||
}
|
||||
|
||||
/** Show the reviewed `plan` for `operation`; Apply applies exactly this token. */
|
||||
export function showPlan(plan, { api, store, scope, trigger = null, reviewAgain, notify = showWorkspaceToast,
|
||||
navigate = navigateTo, backup = () => downloadDatabaseBackup() }) {
|
||||
@@ -232,7 +212,9 @@ export function showPlan(plan, { api, store, scope, trigger = null, reviewAgain,
|
||||
const lines = detailLines(plan);
|
||||
if (lines.length) {
|
||||
const list = el('details');
|
||||
list.append(el('summary', detailsSummary(plan, lines.length)));
|
||||
const count = plan.scope?.count ?? lines.length;
|
||||
list.append(el('summary', lines.length < count ? `Listed records (${number(lines.length)} of ${number(count)})`
|
||||
: `Listed records (${number(lines.length)})`));
|
||||
const body = el('pre', lines.join('\n'));
|
||||
body.tabIndex = 0;
|
||||
list.append(body);
|
||||
@@ -256,7 +238,7 @@ export function showPlan(plan, { api, store, scope, trigger = null, reviewAgain,
|
||||
return;
|
||||
}
|
||||
openReviewDialog({ scope, trigger, title: PLAN_TITLE, heading,
|
||||
grid: planGrid(plan), options, sentence: planNote(plan), backup: backupBox(plan, backup),
|
||||
grid: planGrid(plan), options, sentence: planNote(plan), backup: { copy: BACKUP_COPY, onBackup: backup },
|
||||
cancelLabel: 'Cancel', primary: { label: APPLY_LABEL },
|
||||
onStart: () => applyPlan({ api, store, scope, token: plan.token, reviewAgain, notify, navigate }) });
|
||||
}
|
||||
|
||||
@@ -19,8 +19,8 @@ export const routes = [
|
||||
// The concept's Activity (app.js renderActivity 732–738): this server session's jobs. No href: no Overview card.
|
||||
{ id: 'activity', label: 'Activity' },
|
||||
// Native System (#249 System S2): `#/system/<section>`. Q12: Overview's card opens its Overview section (`native`:
|
||||
// a workspace hash, not a legacy page); every section is native (S12 retired D94's hand-off). The description is
|
||||
// the concept's System heading (app.js renderSystem 700).
|
||||
// a workspace hash, not a legacy page); each undelivered section hands off to its legacy tool (D94). The
|
||||
// description is the concept's System heading (app.js renderSystem 700).
|
||||
{ id: 'system', label: 'System', href: '#/system/overview', native: true,
|
||||
description: 'Configuration, maintenance and recovery—with browsing kept in the workspace.', action: 'Open System' },
|
||||
];
|
||||
|
||||
@@ -1,600 +0,0 @@
|
||||
/**
|
||||
* System › Diagnostics & logs (#249 System S12, #329): the concept's `renderSystem` "diagnostics" body (app.js 694–695)
|
||||
* with live data, built with the DOM.
|
||||
*
|
||||
* - "Diagnostics": "Run checks" opens the shared Health report (D95), as Overview and Maintenance do. The concept's
|
||||
* `div.meta-grid` reads one `GET /system/diagnostics` (Database, Sources, Viewer.js) and the send bridge (Host
|
||||
* renderers, D106's line). "Download diagnostic report" saves a fresh report (the server's listed fields, no secret,
|
||||
* no absolute path) with the renderer line added. "Image-link diagnostics" → Inspect opens the read-only Inspect a
|
||||
* file page. D110 adds "Test image link": the legacy synthetic link, confirmed first, naming the prompt it links.
|
||||
* - "Logs": the concept's toolbar (Level, Refresh, Download log) over the in-memory buffer, newest first, as
|
||||
* `HH:MM:SS LEVEL message`. D109 adds: Lines 1–500, all six minimum levels, the statistics line, CRITICAL as a
|
||||
* runtime level, the descriptions that say a runtime value lasts until the server restarts, the retention line, the
|
||||
* log files with their downloads and "Clear all logs" (confirmed). "Runtime log level" and "Console logging" are
|
||||
* runtime drafts saved by the one Save row (`POST /logs/config`). "Review log cleanup" reviews the `rotated_logs`
|
||||
* maintenance plan (D100), which deletes only the rotated copies it listed.
|
||||
*
|
||||
* Every request goes through the page's api client; every listener sits on the section's `scope`, and a read that
|
||||
* answers after the section was left paints nothing.
|
||||
*/
|
||||
import { createButton } from '../shared/toolkit/core.js';
|
||||
import { settingRow, operationRow } from './system-controls.mjs';
|
||||
import { runHealthAction } from './system-overview.mjs';
|
||||
import { reviewMissingLinks } from './system-maintenance.mjs';
|
||||
import { bridgeFactory, rendererText, RENDERER_COPY } from './system-integrations.mjs';
|
||||
import { closeWorkspaceDialog, openContentDialog } from './workspace-dialogs.mjs';
|
||||
import { reviewPlan } from './maintenance-review.mjs';
|
||||
import { backupFilename, saveResponse } from './activity-scan.mjs';
|
||||
import { showWorkspaceToast } from './toast.mjs';
|
||||
import { navigateTo } from './state.mjs';
|
||||
import { noun } from './plural.mjs';
|
||||
|
||||
export const REPORT_URL = '/prompt_manager/system/diagnostics';
|
||||
export const RECENT_URL = '/prompt_manager/recent?limit=1';
|
||||
export const TEST_LINK_URL = '/prompt_manager/diagnostics/test-link';
|
||||
export const LOGS_URL = '/prompt_manager/logs';
|
||||
export const STATS_URL = '/prompt_manager/logs/stats';
|
||||
export const FILES_URL = '/prompt_manager/logs/files';
|
||||
export const CONFIG_URL = '/prompt_manager/logs/config';
|
||||
export const TRUNCATE_URL = '/prompt_manager/logs/truncate';
|
||||
export const DOWNLOAD_URL = '/prompt_manager/logs/download/';
|
||||
export const ACTIVE_LOG = 'prompt_manager.log';
|
||||
export const TEST_LINK_FILE = 'prompt_manager_test_link.png';
|
||||
|
||||
export const LOADING = 'Loading…';
|
||||
export const UNREADABLE = 'Could not be read.';
|
||||
export const META_TERMS = Object.freeze(['Database', 'Sources', 'Viewer.js', 'Host renderers']);
|
||||
/** S12-8: a minimum level; `''` asks for every level. */
|
||||
export const LEVEL_OPTIONS = Object.freeze([['', 'All levels'], ['DEBUG', 'DEBUG'], ['INFO', 'INFO'], ['WARNING', 'WARNING'],
|
||||
['ERROR', 'ERROR'], ['CRITICAL', 'CRITICAL']]);
|
||||
/** The concept's runtime levels in its order, then CRITICAL (D109). */
|
||||
export const RUNTIME_LEVELS = Object.freeze(['INFO', 'WARNING', 'DEBUG', 'ERROR', 'CRITICAL']);
|
||||
export const DEFAULT_LIMIT = 100;
|
||||
export const MAX_LIMIT = 500;
|
||||
export const KIND_LABELS = Object.freeze({ active: 'Active log', rotated: 'Rotated copy', link: 'Link, not downloaded',
|
||||
other: 'Other file' });
|
||||
|
||||
export const COPY = Object.freeze({
|
||||
runtimeLevel: 'How much the server writes to the log. Applied until the server restarts.',
|
||||
console: 'Print Prompt Manager’s warnings and errors in the ComfyUI console (everything at DEBUG). Applied until the server restarts.',
|
||||
inspect: 'Check a file-to-prompt association without changing it.',
|
||||
testLink: 'Write a placeholder image record to check that linking works.',
|
||||
noPrompt: 'Add a prompt first.',
|
||||
promptUnread: 'The most recent prompt could not be read.',
|
||||
noEntries: 'No log entries at this level.',
|
||||
logUnread: 'The log could not be read.',
|
||||
statsUnread: 'The log statistics could not be read.',
|
||||
filesUnread: 'The log files could not be read.',
|
||||
configUnread: 'The retention settings could not be read.',
|
||||
runtimeUnread: 'The current value could not be read.',
|
||||
limitHint: 'Lines to show: a whole number from 1 to 500.',
|
||||
logDownloadFailed: 'The log could not be downloaded.',
|
||||
reportDone: 'Diagnostic report downloaded.',
|
||||
reportFailed: 'The diagnostic report could not be downloaded. Nothing was saved to this device.',
|
||||
testLinkFailed: 'The test link could not be created. Nothing was changed.',
|
||||
clearFailed: 'The logs could not be cleared. Read the log files again before retrying.',
|
||||
clearDescription: 'Empty the active log and delete its rotated copies.',
|
||||
});
|
||||
|
||||
const number = value => value.toLocaleString('en-US');
|
||||
|
||||
function el(tag, text, className) {
|
||||
const node = document.createElement(tag);
|
||||
if (text !== undefined) node.textContent = text;
|
||||
if (className) node.className = className;
|
||||
return node;
|
||||
}
|
||||
|
||||
/** A byte count as the server's plan labels it: `1000 bytes`, `2.0 KB`, `10.0 MB`, `1.6 GB` (1 KB = 1024 bytes). */
|
||||
export function sizeText(bytes) {
|
||||
const size = Number.isFinite(bytes) && bytes > 0 ? Math.floor(bytes) : 0;
|
||||
if (size < 1024) return `${number(size)} ${noun(size, 'byte')}`;
|
||||
for (const [unit, scale] of [['KB', 2 ** 10], ['MB', 2 ** 20]]) {
|
||||
if (size < scale * 1024) return `${(size / scale).toFixed(1)} ${unit}`;
|
||||
}
|
||||
return `${(size / 2 ** 30).toFixed(1)} GB`;
|
||||
}
|
||||
|
||||
/** The text for a read that was refused: a `busy` refusal names the running job (D57's naming). */
|
||||
export function refusalText(error) {
|
||||
if (error?.code === 'busy') return `Busy: ${error.running?.title || 'another job'} is running.`;
|
||||
return UNREADABLE;
|
||||
}
|
||||
|
||||
/** A `dd` text from a read's state: `{loading}`, `{error}` or `{report}` handed to `value(report)`. */
|
||||
function stateText(state, value) {
|
||||
if (!state || state.loading) return LOADING;
|
||||
if (state.error !== undefined) return refusalText(state.error);
|
||||
return value(state.report);
|
||||
}
|
||||
|
||||
/** Database: "Readable · 1.6 GB · 19,956 images", or why it is not readable. */
|
||||
export function databaseText(state) {
|
||||
return stateText(state, report => {
|
||||
const check = report?.checks?.database;
|
||||
if (check?.status !== 'ok') return `Not readable: ${check?.message || 'no answer from the database'}`;
|
||||
const images = Number.isSafeInteger(check.image_count) ? check.image_count : 0;
|
||||
return `Readable · ${sizeText(report.database_info?.database_size_bytes)} · ${number(images)} ${noun(images, 'image')}`;
|
||||
});
|
||||
}
|
||||
|
||||
/** Sources: the first match of the design's precedence table (§4). */
|
||||
export function sourcesText(state) {
|
||||
return stateText(state, report => {
|
||||
const count = key => (Number.isSafeInteger(report?.sources?.[key]) ? report.sources[key] : 0);
|
||||
const configured = count('configured');
|
||||
const watching = count('watching');
|
||||
const unavailable = count('unavailable');
|
||||
if (configured === 0) return `ComfyUI output folder · ${watching ? 'watching' : 'manual scan'}`;
|
||||
if (unavailable > 0) return `${number(configured)} ${noun(configured, 'source')} · ${number(unavailable)} not available`;
|
||||
return `${number(configured)} ${noun(configured, 'source')} · ${number(watching)} watching`;
|
||||
});
|
||||
}
|
||||
|
||||
/** Viewer.js: the pinned vendor version the server ships. */
|
||||
export function viewerText(state) {
|
||||
return stateText(state, report => `${report?.viewerjs || 'unknown'} · pinned vendor`);
|
||||
}
|
||||
|
||||
const pad = value => String(value).padStart(2, '0');
|
||||
|
||||
/** `prompt-manager-diagnostics-YYYYMMDD-HHMM.json` in local time. */
|
||||
export function reportFilename(now = new Date()) {
|
||||
const day = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}`;
|
||||
return `prompt-manager-diagnostics-${day}-${pad(now.getHours())}${pad(now.getMinutes())}.json`;
|
||||
}
|
||||
|
||||
/** The report as saved: the server's fields and the renderer line this page saw (null while it looks). */
|
||||
export function mergeReport(report, hostRenderers) {
|
||||
return { ...report, host_renderers: typeof hostRenderers === 'string' ? hostRenderers : null };
|
||||
}
|
||||
|
||||
/** One buffer entry as the concept prints it: `09:42:18 INFO message`. */
|
||||
export function logLine(entry) {
|
||||
const time = typeof entry?.timestamp === 'string' ? entry.timestamp.slice(11, 19) : '--:--:--';
|
||||
return `${time} ${String(entry?.level ?? '').padEnd(5)} ${entry?.message ?? ''}`;
|
||||
}
|
||||
|
||||
/** Lines: a whole number 1–500 after trimming, else null. */
|
||||
export function validLimit(value) {
|
||||
const text = String(value ?? '').trim();
|
||||
if (!/^\d+$/.test(text)) return null;
|
||||
const limit = Number(text);
|
||||
return limit >= 1 && limit <= MAX_LIMIT ? limit : null;
|
||||
}
|
||||
|
||||
/** The `/logs` request for a level (`''`: every level) and a valid limit. */
|
||||
export function logsUrl(level, limit) {
|
||||
const query = new URLSearchParams({ limit: String(limit) });
|
||||
if (level) query.set('level', level);
|
||||
return `${LOGS_URL}?${query}`;
|
||||
}
|
||||
|
||||
/** D109: "1,000 entries in memory · 3 errors · 12 warnings · 5 files · 41.2 MB on disk" (errors: ERROR + CRITICAL). */
|
||||
export function statsText(stats) {
|
||||
const counts = stats?.level_counts ?? {};
|
||||
const of = key => (Number.isSafeInteger(counts[key]) ? counts[key] : 0);
|
||||
const entries = Number.isSafeInteger(stats?.buffer_count) ? stats.buffer_count : 0;
|
||||
const errors = of('ERROR') + of('CRITICAL');
|
||||
const warnings = of('WARNING');
|
||||
const files = Number.isSafeInteger(stats?.log_files_count) ? stats.log_files_count : 0;
|
||||
return [`${number(entries)} ${noun(entries, 'entry', 'entries')} in memory`, `${number(errors)} ${noun(errors, 'error')}`,
|
||||
`${number(warnings)} ${noun(warnings, 'warning')}`, `${number(files)} ${noun(files, 'file')}`,
|
||||
`${sizeText(stats?.total_log_size)} on disk`].join(' · ');
|
||||
}
|
||||
|
||||
const megabytes = bytes => {
|
||||
const value = (Number.isFinite(bytes) ? bytes : 0) / 2 ** 20;
|
||||
return Number.isInteger(value) ? number(value) : value.toFixed(1);
|
||||
};
|
||||
|
||||
/** D109: the read-only retention values. */
|
||||
export function retentionText(config) {
|
||||
const copies = Number.isSafeInteger(config?.backup_count) ? config.backup_count : 0;
|
||||
const buffer = Number.isSafeInteger(config?.buffer_size) ? config.buffer_size : 0;
|
||||
return `Rotates at ${megabytes(config?.max_file_size)} MB · keeps ${number(copies)} rotated ${noun(copies, 'copy', 'copies')}`
|
||||
+ ` · file logging ${config?.file_logging === false ? 'off' : 'on'} · memory buffer ${number(buffer)} ${noun(buffer, 'entry', 'entries')}`;
|
||||
}
|
||||
|
||||
const rotatedOf = files => (Array.isArray(files) ? files : []).filter(row => row?.kind === 'rotated');
|
||||
|
||||
/** The concept's line under "Log files & retention": "prompt_manager.log · 1.4 MB · rotated copies: 3". */
|
||||
export function fileLine(files) {
|
||||
const active = (Array.isArray(files) ? files : []).find(row => row?.kind === 'active');
|
||||
const size = active ? sizeText(active.size) : 'not written yet';
|
||||
return `${ACTIVE_LOG} · ${size} · rotated copies: ${number(rotatedOf(files).length)}`;
|
||||
}
|
||||
|
||||
/** "2026-10-25 14:02" from the server's naive local ISO time. */
|
||||
export function modifiedText(iso) {
|
||||
return typeof iso === 'string' && iso.length >= 16 ? iso.slice(0, 16).replace('T', ' ') : 'unknown time';
|
||||
}
|
||||
|
||||
/** One listed file's line: "<size> · <modified> · <kind>". */
|
||||
export function fileDetail(row) {
|
||||
return `${sizeText(row?.size)} · ${modifiedText(row?.modified)} · ${KIND_LABELS[row?.kind] ?? KIND_LABELS.other}`;
|
||||
}
|
||||
|
||||
/** CUJ 7: what Clear all logs does, from the files read last (null: they could not be read). */
|
||||
export function clearSentence(files) {
|
||||
const tail = 'and clears the lines shown here. The other files in the log folder are kept. This cannot be undone.';
|
||||
if (!Array.isArray(files)) return `Empties ${ACTIVE_LOG}, deletes its rotated copies ${tail}`;
|
||||
const rotated = rotatedOf(files);
|
||||
if (!rotated.length) return `Empties ${ACTIVE_LOG} ${tail}`;
|
||||
const size = rotated.reduce((sum, row) => sum + (Number.isFinite(row.size) ? row.size : 0), 0);
|
||||
return `Empties ${ACTIVE_LOG}, deletes ${number(rotated.length)} rotated ${noun(rotated.length, 'copy', 'copies')} (${sizeText(size)}) ${tail}`;
|
||||
}
|
||||
|
||||
/** CUJ 7: the toast for `/logs/truncate`'s `results` ({truncated: [names], errors: [messages]}). */
|
||||
export function clearToast(results) {
|
||||
const truncated = Array.isArray(results?.truncated) ? results.truncated : [];
|
||||
const deleted = truncated.filter(name => typeof name === 'string' && name.toLowerCase() !== ACTIVE_LOG).length;
|
||||
const failed = Array.isArray(results?.errors) ? results.errors.length : 0;
|
||||
const parts = ['Logs cleared', `${number(deleted)} rotated ${noun(deleted, 'copy', 'copies')} deleted`];
|
||||
if (failed) parts.push(`${number(failed)} could not be deleted`);
|
||||
return `${parts.join(' · ')}.`;
|
||||
}
|
||||
|
||||
/** A prompt's short name for the test-link dialog: its first words, on one line. */
|
||||
export function promptName(prompt) {
|
||||
const text = String(prompt?.text ?? '').replace(/\s+/g, ' ').trim();
|
||||
return text.length > 60 ? `${text.slice(0, 59)}…` : text;
|
||||
}
|
||||
|
||||
/** CUJ 8: the confirmation, naming the prompt the placeholder record links to. */
|
||||
export function testLinkSentence(prompt) {
|
||||
return `Writes a placeholder image record (${TEST_LINK_FILE}) linked to prompt #${prompt.id} “${promptName(prompt)}”. `
|
||||
+ 'Maintenance › Missing-file links removes it later.';
|
||||
}
|
||||
|
||||
/** S12-9: the runtime drafts from `GET /logs/config`'s `config`. */
|
||||
export function runtimeValues(config) {
|
||||
return { runtime_level: config?.level, console_logging: config?.console_logging };
|
||||
}
|
||||
|
||||
/** S12-9: the `POST /logs/config` body for the changed runtime drafts only. */
|
||||
export function runtimeBody(changes) {
|
||||
const body = {};
|
||||
if (Object.hasOwn(changes ?? {}, 'runtime_level')) body.level = changes.runtime_level;
|
||||
if (Object.hasOwn(changes ?? {}, 'console_logging')) body.console_logging = changes.console_logging;
|
||||
return body;
|
||||
}
|
||||
|
||||
/** Fetch `url` through the api-client URL and save it under the name the server gave (or `fallback`). */
|
||||
async function download(url, fallback, fetchUrl) {
|
||||
const response = await fetchUrl(url);
|
||||
if (!response.ok) throw new Error(`Download failed with ${response.status}`);
|
||||
const blob = await response.blob();
|
||||
saveResponse(blob, backupFilename(response.headers.get('Content-Disposition'), fallback));
|
||||
}
|
||||
|
||||
/** Run `work` with `button` aria-disabled; a second press meanwhile is ignored. */
|
||||
async function pressing(button, work) {
|
||||
if (button.getAttribute('aria-disabled') === 'true') return;
|
||||
button.setAttribute('aria-disabled', 'true');
|
||||
try {
|
||||
await work();
|
||||
} finally {
|
||||
button.removeAttribute('aria-disabled');
|
||||
}
|
||||
}
|
||||
|
||||
function confirmDialog({ scope, trigger, title, sentence, label, danger = false, onConfirm }) {
|
||||
const cancel = createButton('Cancel');
|
||||
const confirm = createButton(label, danger ? 'danger' : 'ca-primary');
|
||||
cancel.addEventListener('click', () => closeWorkspaceDialog());
|
||||
confirm.addEventListener('click', () => {
|
||||
closeWorkspaceDialog();
|
||||
onConfirm();
|
||||
});
|
||||
const actions = el('div', undefined, 'dialog-actions');
|
||||
actions.append(cancel, confirm);
|
||||
openContentDialog({ scope, title, trigger, nodes: [el('p', sentence), actions] });
|
||||
}
|
||||
|
||||
/** The "Test image link" operation (D110): disabled until the most recent prompt was read. */
|
||||
function testLinkRow({ api, scope, notify }) {
|
||||
let prompt = null;
|
||||
const row = operationRow({ scope, title: 'Test image link', description: COPY.testLink, label: 'Create test link…',
|
||||
onClick: button => {
|
||||
if (button.getAttribute('aria-disabled') === 'true' || !prompt) return;
|
||||
const target = prompt;
|
||||
// The button is held until the write answered, so a second press cannot write a second record.
|
||||
confirmDialog({ scope, trigger: button, title: 'Create test link', sentence: testLinkSentence(target), label: 'Create',
|
||||
onConfirm: () => pressing(button, async () => {
|
||||
try {
|
||||
const answer = await api.post(TEST_LINK_URL, { prompt_id: target.id });
|
||||
if (!scope.signal.aborted) notify(`Test link created (image record ${answer?.result?.image_id}).`);
|
||||
} catch {
|
||||
if (!scope.signal.aborted) notify(COPY.testLinkFailed);
|
||||
}
|
||||
}) });
|
||||
} });
|
||||
const button = row.querySelector('.row-actions > button');
|
||||
const description = row.querySelector('p');
|
||||
button.setAttribute('aria-disabled', 'true');
|
||||
api.get(RECENT_URL, { signal: scope.signal, cache: 'no-store' }).then(answer => {
|
||||
if (scope.signal.aborted) return;
|
||||
const first = Array.isArray(answer?.results) ? answer.results[0] : null;
|
||||
if (first && first.id !== undefined && first.id !== null) {
|
||||
prompt = first;
|
||||
button.removeAttribute('aria-disabled');
|
||||
} else {
|
||||
description.textContent = `${COPY.testLink} ${COPY.noPrompt}`;
|
||||
}
|
||||
}, () => {
|
||||
if (!scope.signal.aborted) description.textContent = `${COPY.testLink} ${COPY.promptUnread}`;
|
||||
});
|
||||
return row;
|
||||
}
|
||||
|
||||
/** The "Diagnostics" surface. */
|
||||
function diagnosticsSurface({ api, activity, scope, navigate, notify, clientFactory }) {
|
||||
const surface = el('section', undefined, 'surface');
|
||||
const heading = el('div', undefined, 'panel-heading');
|
||||
const run = createButton('Run checks');
|
||||
heading.append(el('h2', 'Diagnostics'), run);
|
||||
const grid = el('div', undefined, 'meta-grid');
|
||||
const cells = {};
|
||||
for (const term of META_TERMS) {
|
||||
const entry = el('div');
|
||||
cells[term] = el('dd', LOADING);
|
||||
entry.append(el('dt', term), cells[term]);
|
||||
grid.append(entry);
|
||||
}
|
||||
const report = createButton('Download diagnostic report');
|
||||
const inspect = operationRow({ scope, title: 'Image-link diagnostics', description: COPY.inspect, label: 'Inspect',
|
||||
onClick: () => navigate('#/inspect') });
|
||||
surface.append(heading, grid, report, inspect, testLinkRow({ api, scope, notify }));
|
||||
|
||||
// D95: the same Health report as Overview and Maintenance; its "Review missing links" reviews a fresh plan.
|
||||
const onReviewMissing = reviewMissingLinks({ api, store: activity, scope, notify, navigate });
|
||||
scope.listen(run, 'click', () => runHealthAction({ api, activity, scope, button: run, navigate, notify, onReviewMissing }));
|
||||
|
||||
const paint = state => {
|
||||
cells.Database.textContent = databaseText(state);
|
||||
cells.Sources.textContent = sourcesText(state);
|
||||
cells['Viewer.js'].textContent = viewerText(state);
|
||||
};
|
||||
paint({ loading: true });
|
||||
api.get(REPORT_URL, { signal: scope.signal, cache: 'no-store' })
|
||||
.then(answer => { if (!scope.signal.aborted) paint({ report: answer }); },
|
||||
error => { if (!scope.signal.aborted) paint({ error }); });
|
||||
|
||||
// S12-13: the renderer line Integrations shows (D106), from the send bridge.
|
||||
let renderers = null;
|
||||
let client = null;
|
||||
scope.cleanup(() => client?.dispose());
|
||||
(async () => {
|
||||
let text;
|
||||
try {
|
||||
client = clientFactory();
|
||||
text = rendererText(await client.discover({ waitMs: 300 }));
|
||||
} catch {
|
||||
text = RENDERER_COPY.unavailable;
|
||||
}
|
||||
if (scope.signal.aborted) return;
|
||||
renderers = text;
|
||||
cells['Host renderers'].textContent = text;
|
||||
})();
|
||||
|
||||
scope.listen(report, 'click', () => pressing(report, async () => {
|
||||
try {
|
||||
const answer = await api.get(REPORT_URL, { signal: scope.signal, cache: 'no-store' });
|
||||
const body = JSON.stringify(mergeReport(answer, renderers), null, 2);
|
||||
saveResponse(new Blob([body], { type: 'application/json' }), reportFilename());
|
||||
if (!scope.signal.aborted) notify(COPY.reportDone);
|
||||
} catch (error) {
|
||||
if (scope.signal.aborted) return;
|
||||
notify(error?.code === 'busy' ? `${refusalText(error)} Try again when it ends.` : COPY.reportFailed);
|
||||
}
|
||||
}));
|
||||
return surface;
|
||||
}
|
||||
|
||||
/** The two runtime rows: drafts of the `runtime` domain (S12-9), saved by Save preferences. */
|
||||
function runtimeRows({ scope, drafts }) {
|
||||
const level = settingRow({ id: 'runtime-level', label: 'Runtime log level', description: COPY.runtimeLevel,
|
||||
control: { type: 'select', options: RUNTIME_LEVELS }, scope,
|
||||
onChange: value => drafts.set('runtime', 'runtime_level', value) });
|
||||
const consoleRow = settingRow({ id: 'console-log', label: 'Console logging', description: COPY.console,
|
||||
control: { type: 'toggle', checked: false }, scope, onChange: value => drafts.set('runtime', 'console_logging', value) });
|
||||
const select = level.querySelector('[data-setting="runtime-level"]');
|
||||
const toggle = consoleRow.querySelector('[data-setting="console-log"]');
|
||||
select.dataset.key = 'runtime_level';
|
||||
toggle.dataset.key = 'console_logging';
|
||||
const repaint = () => {
|
||||
const value = drafts.get('runtime', 'runtime_level');
|
||||
select.disabled = typeof value !== 'string';
|
||||
if (typeof value === 'string') select.value = value;
|
||||
const on = drafts.get('runtime', 'console_logging');
|
||||
toggle.disabled = typeof on !== 'boolean';
|
||||
if (typeof on === 'boolean') toggle.checked = on;
|
||||
};
|
||||
scope.cleanup(drafts.subscribe(repaint));
|
||||
repaint();
|
||||
// A failed read leaves both controls disabled; each says why (the value the server holds is not known).
|
||||
const notes = [level, consoleRow].map(row => {
|
||||
const note = el('p', COPY.runtimeUnread, 'small');
|
||||
return () => row.children[0].append(note);
|
||||
});
|
||||
const reload = () => drafts.load('runtime').then(() => {}, () => {
|
||||
if (!scope.signal.aborted) for (const show of notes) show();
|
||||
});
|
||||
reload();
|
||||
return { rows: [level, consoleRow], reload };
|
||||
}
|
||||
|
||||
/** The "Logs" surface. */
|
||||
function logsSurface({ api, activity, scope, drafts, navigate, notify, latestOnly, STALE, fetchUrl }) {
|
||||
const surface = el('section', undefined, 'surface');
|
||||
const toolbar = el('div', undefined, 'toolbar spaced-16');
|
||||
const levelLabel = el('label', 'Level', 'small');
|
||||
levelLabel.htmlFor = 'log-level';
|
||||
const level = el('select');
|
||||
level.id = 'log-level';
|
||||
for (const [value, label] of LEVEL_OPTIONS) {
|
||||
const option = el('option', label);
|
||||
option.value = value;
|
||||
level.append(option);
|
||||
}
|
||||
const limitLabel = el('label', 'Lines', 'small');
|
||||
limitLabel.htmlFor = 'log-limit';
|
||||
const limit = el('input');
|
||||
limit.id = 'log-limit';
|
||||
limit.type = 'number';
|
||||
limit.min = '1';
|
||||
limit.max = String(MAX_LIMIT);
|
||||
limit.step = '1';
|
||||
limit.value = String(DEFAULT_LIMIT);
|
||||
limit.title = COPY.limitHint;
|
||||
const refresh = createButton('Refresh');
|
||||
const downloadLog = createButton('Download log');
|
||||
toolbar.append(levelLabel, level, limitLabel, limit, refresh, downloadLog);
|
||||
|
||||
const lines = el('pre');
|
||||
lines.id = 'log-lines';
|
||||
lines.tabIndex = 0;
|
||||
const code = el('code', LOADING);
|
||||
lines.append(code);
|
||||
const stats = el('p', LOADING, 'small');
|
||||
stats.id = 'log-stats';
|
||||
stats.setAttribute('role', 'status');
|
||||
stats.setAttribute('aria-label', 'Log statistics');
|
||||
|
||||
const files = el('details');
|
||||
const line = el('p', LOADING, 'small');
|
||||
const review = createButton('Review log cleanup');
|
||||
const retention = el('p', LOADING, 'small');
|
||||
const list = el('div', undefined, 'operation-list');
|
||||
let listed = null;
|
||||
// The confirmation names what Clear deletes now: the files are read again before it opens (a read that fails
|
||||
// gives the general sentence, never an old count).
|
||||
const clear = operationRow({ scope, title: 'Clear all logs', description: COPY.clearDescription, label: 'Clear all logs…',
|
||||
danger: true, onClick: button => pressing(button, async () => {
|
||||
await readFiles();
|
||||
if (scope.signal.aborted) return;
|
||||
confirmDialog({ scope, trigger: button, title: 'Clear all logs', sentence: clearSentence(listed), label: 'Clear',
|
||||
danger: true, onConfirm: clearLogs });
|
||||
}) });
|
||||
files.append(el('summary', 'Log files & retention'), line, review, retention, list, clear);
|
||||
const runtime = runtimeRows({ scope, drafts });
|
||||
surface.append(el('h2', 'Logs'), toolbar, lines, stats, ...runtime.rows, files);
|
||||
|
||||
const logsRead = latestOnly({ lifetime: scope.signal });
|
||||
async function readLogs() {
|
||||
const count = validLimit(limit.value);
|
||||
if (count === null) {
|
||||
limit.setAttribute('aria-invalid', 'true');
|
||||
return;
|
||||
}
|
||||
limit.removeAttribute('aria-invalid');
|
||||
let answer;
|
||||
try {
|
||||
answer = await logsRead(signal => api.get(logsUrl(level.value, count), { signal, cache: 'no-store' }));
|
||||
} catch {
|
||||
if (!scope.signal.aborted) code.textContent = COPY.logUnread;
|
||||
return;
|
||||
}
|
||||
if (answer === STALE || scope.signal.aborted) return;
|
||||
const entries = Array.isArray(answer?.logs) ? answer.logs : [];
|
||||
code.textContent = entries.length ? entries.map(logLine).join('\n') : COPY.noEntries;
|
||||
}
|
||||
|
||||
/** One latest-only read of `url` painting `paint(answer)`, or `failed()` when it could not be read. */
|
||||
const reader = (url, paint, failed) => {
|
||||
const read = latestOnly({ lifetime: scope.signal });
|
||||
return async () => {
|
||||
let answer;
|
||||
try {
|
||||
answer = await read(signal => api.get(url, { signal, cache: 'no-store' }));
|
||||
} catch {
|
||||
if (!scope.signal.aborted) failed();
|
||||
return;
|
||||
}
|
||||
if (answer !== STALE && !scope.signal.aborted) paint(answer);
|
||||
};
|
||||
};
|
||||
const readStats = reader(STATS_URL, answer => { stats.textContent = statsText(answer?.stats); },
|
||||
() => { stats.textContent = COPY.statsUnread; });
|
||||
const readConfig = reader(CONFIG_URL, answer => { retention.textContent = retentionText(answer?.config); },
|
||||
() => { retention.textContent = COPY.configUnread; });
|
||||
const readFiles = reader(FILES_URL, answer => {
|
||||
listed = Array.isArray(answer?.files) ? answer.files : [];
|
||||
line.textContent = fileLine(listed);
|
||||
list.replaceChildren(...listed.map(fileRow));
|
||||
}, () => {
|
||||
listed = null;
|
||||
line.textContent = COPY.filesUnread;
|
||||
list.replaceChildren();
|
||||
});
|
||||
|
||||
function fileRow(row) {
|
||||
const item = el('div', undefined, 'operation');
|
||||
const text = el('div');
|
||||
text.append(el('h3', row.filename), el('p', fileDetail(row)));
|
||||
item.append(text);
|
||||
if (row.kind !== 'link') {
|
||||
const actions = el('div', undefined, 'row-actions');
|
||||
const button = createButton('Download');
|
||||
button.setAttribute('aria-label', `Download ${row.filename}`);
|
||||
scope.listen(button, 'click', () => saveLog(button, row.filename));
|
||||
actions.append(button);
|
||||
item.append(actions);
|
||||
}
|
||||
return item;
|
||||
}
|
||||
|
||||
function saveLog(button, name) {
|
||||
return pressing(button, async () => {
|
||||
try {
|
||||
await download(`${DOWNLOAD_URL}${encodeURIComponent(name)}`, name, fetchUrl);
|
||||
} catch {
|
||||
if (!scope.signal.aborted) notify(COPY.logDownloadFailed);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function readAll() {
|
||||
readLogs();
|
||||
readStats();
|
||||
readFiles();
|
||||
readConfig();
|
||||
}
|
||||
|
||||
async function clearLogs() {
|
||||
let answer;
|
||||
try {
|
||||
answer = await api.post(TRUNCATE_URL, {});
|
||||
} catch {
|
||||
if (!scope.signal.aborted) notify(COPY.clearFailed);
|
||||
}
|
||||
if (scope.signal.aborted) return;
|
||||
if (answer) notify(clearToast(answer.results));
|
||||
readAll();
|
||||
runtime.reload();
|
||||
}
|
||||
|
||||
scope.listen(level, 'change', readLogs);
|
||||
scope.listen(limit, 'change', readLogs);
|
||||
scope.listen(refresh, 'click', () => { readLogs(); readStats(); readFiles(); });
|
||||
scope.listen(downloadLog, 'click', () => saveLog(downloadLog, ACTIVE_LOG));
|
||||
scope.listen(review, 'click', () => reviewPlan({ api, store: activity, scope, operation: 'rotated_logs', trigger: review,
|
||||
notify, navigate }));
|
||||
readAll();
|
||||
return surface;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mount Diagnostics & logs in `container` (System's `.settings-layout`); returns its two surfaces in concept order.
|
||||
* `drafts` is the System draft store (its runtime domain holds the log level and console setting), `activity` the
|
||||
* page's Activity store.
|
||||
*/
|
||||
export function mountDiagnostics(container, { api, activity, scope, drafts, navigate = navigateTo, notify = showWorkspaceToast,
|
||||
clientFactory = bridgeFactory(), latestOnly = window.ApiClient.latestOnly, STALE = window.ApiClient.STALE,
|
||||
fetchUrl = window.ApiClient.fetchUrl }) {
|
||||
return [
|
||||
diagnosticsSurface({ api, activity, scope, navigate, notify, clientFactory }),
|
||||
logsSurface({ api, activity, scope, drafts, navigate, notify, latestOnly, STALE, fetchUrl }),
|
||||
];
|
||||
}
|
||||
@@ -345,8 +345,7 @@ function comfySurface({ scope, clientFactory }) {
|
||||
return surface;
|
||||
}
|
||||
|
||||
/** The send bridge's client factory for this page's mount (Integrations and Diagnostics & logs, S12-13). */
|
||||
export function bridgeFactory() {
|
||||
function bridgeFactory() {
|
||||
const url = new URL(window.ApiClient.resolveUrl('/prompt_manager/'), window.location.href);
|
||||
return () => createBridgeClient({ namespace: url.pathname.slice(0, -'prompt_manager/'.length), origin: window.location.origin });
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user