Compare commits
26
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8dbef14a5c | ||
|
|
95c4542c6f | ||
|
|
534c95248a | ||
|
|
942a097c45 | ||
|
|
e4dc781c26 | ||
|
|
d0339128e4 | ||
|
|
ef7e07c765 | ||
|
|
83fa80c330 | ||
|
|
e472a8bf74 | ||
|
|
be07d1e74e | ||
|
|
83bac3988f | ||
|
|
44177974cb | ||
|
|
d11b354ed6 | ||
|
|
2f8b7cb306 | ||
|
|
37e4e08327 | ||
|
|
d2ea705a44 | ||
|
|
52a00370ea | ||
|
|
bdc38422a9 | ||
|
|
a985af506b | ||
|
|
a0fc5171e3 | ||
|
|
0652beba90 | ||
|
|
f35e010f79 | ||
|
|
d868745194 | ||
|
|
1bb3678f32 | ||
|
|
2f67b98287 | ||
|
|
f3b9f61f11 |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"required_status_checks": {
|
||||
"strict": true,
|
||||
"strict": false,
|
||||
"contexts": [
|
||||
"py3.12 on ubuntu-latest",
|
||||
"py3.13 on ubuntu-latest",
|
||||
|
||||
@@ -4,6 +4,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
+44
-15
@@ -4,6 +4,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
@@ -63,16 +64,18 @@ jobs:
|
||||
comfyui:
|
||||
name: real ComfyUI compatibility
|
||||
runs-on: ubuntu-latest
|
||||
# Chromium and its system libraries come from the pinned Playwright image (tag = @playwright/test in
|
||||
# tools/vendor/package-lock.json; tests/vendor/browser-image.test.mjs checks it). The runner install
|
||||
# pulled them from Ubuntu's apt mirror on every run, and three runs (#349, #378, #382) crawled at
|
||||
# ~26 kB/s for 11-12 minutes until the job hit its limit after every test had passed.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 24
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: npm
|
||||
cache-dependency-path: tools/vendor/package-lock.json
|
||||
- name: Read the reviewed runtime pins
|
||||
id: runtime
|
||||
run: |
|
||||
@@ -97,21 +100,42 @@ jobs:
|
||||
ref: ${{ steps.runtime.outputs.kikoRevision }}
|
||||
path: .ci-kikotools
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v7
|
||||
# The image's own Python and Node 24: setup-python/setup-node install the runner's toolcache builds,
|
||||
# whose entry points name host paths that do not exist in the container. The image has no ensurepip
|
||||
# and the job has no root, so a checksum-pinned pip wheel bootstraps a user-space venv. The image
|
||||
# Python must be the reviewed host Python, so a pin or image bump that disagrees fails here.
|
||||
- name: Python environment on the reviewed host Python
|
||||
env:
|
||||
PIP_WHEEL: https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl
|
||||
PIP_SHA256: 71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e
|
||||
PINNED_PYTHON: ${{ steps.runtime.outputs.pythonVersion }}
|
||||
run: |
|
||||
image="$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])')"
|
||||
[ "$image" = "$PINNED_PYTHON" ] || { echo "::error::image Python $image is not the pinned $PINNED_PYTHON"; exit 1; }
|
||||
venv="$RUNNER_TEMP/venv" wheel="$RUNNER_TEMP/pip-26.2.1-py3-none-any.whl"
|
||||
python3 -m venv --without-pip "$venv"
|
||||
curl -fsSL --retry 3 -o "$wheel" "$PIP_WHEEL"
|
||||
echo "$PIP_SHA256 $wheel" | sha256sum -c -
|
||||
"$venv/bin/python" "$wheel/pip" install --no-index --no-cache-dir "$wheel"
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
# pip's download cache (torch alone is ~200 MB); inside the container ~ is the job's HOME (/github/home).
|
||||
- name: Cache pip downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
python-version: ${{ steps.runtime.outputs.pythonVersion }}
|
||||
cache: pip
|
||||
cache-dependency-path: |
|
||||
requirements-dev.txt
|
||||
.ci-comfyui/requirements.txt
|
||||
tools/vendor/comfyui-runtime.json
|
||||
path: ~/.cache/pip
|
||||
key: pip-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('requirements-dev.txt', '.ci-comfyui/requirements.txt', 'tools/vendor/comfyui-runtime.json') }}
|
||||
- name: Install CPU runtime and test dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}" --index-url https://download.pytorch.org/whl/cpu
|
||||
pip install -r .ci-comfyui/requirements.txt -r requirements-dev.txt "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}"
|
||||
# npm's download cache, keyed on the lockfile. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- run: npx --prefix tools/vendor playwright install --with-deps chromium
|
||||
- name: Classic and Nodes 2.0 startup and compatibility
|
||||
env:
|
||||
PM_COMFY_ROOT: ${{ github.workspace }}/.ci-comfyui
|
||||
@@ -197,7 +221,10 @@ jobs:
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install black flake8
|
||||
cache: pip
|
||||
cache-dependency-path: requirements-dev.txt
|
||||
# The exact black and flake8 pins from requirements-dev.txt, so a new release cannot turn CI red.
|
||||
- run: pip install $(grep -E '^(black|flake8)==' requirements-dev.txt)
|
||||
- run: black --check .
|
||||
- run: flake8 .
|
||||
|
||||
@@ -211,6 +238,8 @@ jobs:
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: npm
|
||||
cache-dependency-path: tools/vendor/package-lock.json
|
||||
|
||||
- name: Check shipped extension and page modules parse
|
||||
run: |
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
name: Viewer.js vendor and compatibility
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
|
||||
@@ -13,9 +13,11 @@ repos:
|
||||
language: system
|
||||
types: [python]
|
||||
|
||||
# Fails at the severity CI enforces (code-quality.yml: -lll, high only), never stricter.
|
||||
# pre-commit appends the staged files, so no shell wrapper can drop the first one.
|
||||
- id: bandit
|
||||
name: bandit
|
||||
entry: bash -c 'bandit "$@" -ll -q 2>&1; true'
|
||||
entry: bandit -lll -q
|
||||
language: system
|
||||
types: [python]
|
||||
exclude: ^(tests/|reference/)
|
||||
|
||||
+587
-54
@@ -282,11 +282,12 @@ PromptManager's own background threads stop through one package teardown,
|
||||
works against one 10 s deadline. Each step gets the smaller of its own bound
|
||||
and the time left. The steps, in order:
|
||||
|
||||
1. Detach the monitor's callbacks (Needs review recorder, preview queue,
|
||||
Images listing listener).
|
||||
2. `ImageMonitor.stop_monitoring` closes the image handler, which joins the
|
||||
image worker (5 s). It then calls watchdog's `observer.stop()`, which is
|
||||
not bounded here, and joins the observer (5 s).
|
||||
1. Refuse new previews (`refuse_auto_thumbnails()`), then detach the
|
||||
monitor's callbacks (Needs review recorder, preview queue, Images listing
|
||||
listener).
|
||||
2. `ImageMonitor.stop_monitoring` works against one 5 s deadline. Closing the
|
||||
image handler joins the image worker; watchdog's `observer.stop()` (not
|
||||
bounded here) and the observer join get what is left, which can be 0 s.
|
||||
3. `close_auto_thumbnails(final=True)` stops the preview worker (30 s) and
|
||||
kills ffmpeg.
|
||||
4. `PromptTracker.shutdown` stops the prompt cleanup thread (2 s).
|
||||
@@ -295,7 +296,11 @@ A failing step, or a second Ctrl+C, is logged with its traceback and does not
|
||||
skip the later steps. A clean run logs one INFO line to PromptManager's log:
|
||||
`Background work stopped in 0.4 s: monitor=ok thumbnails=ok tracker=ok`. If
|
||||
any part is still running, one WARNING goes to the console:
|
||||
`Background work not stopped in time (10.0 s): monitor`.
|
||||
`Background work not stopped in time (10.0 s): monitor`. A Ctrl+C that lands
|
||||
between two steps (each step catches its own) is re-raised after the result is
|
||||
stored, with the parts not reached False, and the `atexit` callback is
|
||||
removed: the later steps are skipped, no summary line is logged, and a later
|
||||
call returns that result and runs nothing.
|
||||
|
||||
ComfyUI never runs aiohttp's `on_cleanup` hooks, because it never calls
|
||||
`runner.cleanup()`. So `atexit` is the live host's path. It runs on Ctrl+C
|
||||
@@ -740,11 +745,55 @@ with a literal true `strip_newlines` is `replace("\n", "").strip()`;
|
||||
(`N` = `inputcount`) after the literal delimiter unless missing or exactly `""`, and
|
||||
ignores later inputs. A linked setting is `unsupported-node`; a list output
|
||||
(`return_list` true or linked) and `PromptSearchList` are `runtime-value` ("Text is
|
||||
decided at run time"), never one list item. Unverified text nodes (LoRAfromText,
|
||||
RegexReplace, XYZPrompt, ImpactWildcardProcessor) stay `unsupported-node`. Every
|
||||
decided at run time"), never one list item. Every
|
||||
intermediate text is length-checked and charged to the scanned-character budget. If
|
||||
the table cannot be loaded, graph inspection is `unresolved` ("Inspection node
|
||||
semantics are unavailable") and ingestion is unaffected, because it never imports the table.
|
||||
Unknown nodes pass through (#381, maintainer-approved 2026-10-27; no per-node
|
||||
inventory): an output whose class the table, its `facts`, `lineage` and the code's
|
||||
adapters never name is read through, not refused. A `text` input is its text (the
|
||||
legacy encoder rule, so RegexReplace or LoRAfromText show their input text), except
|
||||
that a string output of a node with a seed input (`seed`, `noise_seed`,
|
||||
`sampling_mode.seed`) and no linked `conditioning`/`positive`/`negative` input is
|
||||
generated at run time (a language model such as TextGenerate, a wildcard):
|
||||
`runtime-value` naming it. Else, at output slot 0 or 1 of a node with a `positive` or
|
||||
`negative` input, only `positive` (slot 1, `negative`) is followed, nothing when it
|
||||
is not linked; else a linked `conditioning` alone; else every linked input except
|
||||
the table's `pass_through.skip_inputs` (core type inputs: model, clip, vae, image,
|
||||
latent, ...), each traced with kind `any`. A followed input with no text adds
|
||||
nothing; limits and selector choices propagate; another failure (`runtime-value`,
|
||||
`composite`, `malformed`) blocks, unless a PromptManager node was reached: this
|
||||
repository's own nodes are authoritative, and then only their texts count. One
|
||||
distinct text resolves; several make the sampler `ambiguous` "Several prompts reach
|
||||
the sampler" with one candidate per distinct outcome, in input order (at most 32,
|
||||
else `work-limit`). A PromptManagerPair's execution record is what ran: while its
|
||||
inputs are checked against it, a walk through an unknown node that fails, diverges
|
||||
or passes text through is unverifiable and never overrides the record. No text
|
||||
at all is `unsupported-node` naming the outermost unknown node. An unlisted output
|
||||
of a listed class (a sampler, a loader, a lineage node) stays `unsupported-node`, so
|
||||
the walk never reads another stage's prompt. `INSPECTION_RESOLVER_VERSION` is 2 for
|
||||
this rule; ingestion keeps the default profile (`RESOLVER_VERSION` 1). The legacy
|
||||
parity gate (`tests/test_legacy_parity_gate.py`) records, per corpus (the
|
||||
generation-params vectors, the inspection parity fixtures, the local resolver golden
|
||||
and selection cases, the unknown-node shapes in `tests/unknown_node_graphs.py`),
|
||||
every graph whose inspection positive equals the legacy parser's; a graph may join
|
||||
that list (`python3 -m tests.test_legacy_parity_gate --write`), never leave it. A graph
|
||||
may read differently from legacy only when `LEGACY_WRONG` there states why (legacy's
|
||||
first-sampler pick on a branch the file name identifies).
|
||||
Saver anchoring (#381): an inspected file's name (`path.name` on the server, the chosen
|
||||
file's name locally) anchors an unanchored inspection on the one node whose
|
||||
`filename_prefix` (literal, or traced as a string) has a last path part that makes the
|
||||
file's stem `<prefix>_<digits>` with an optional trailing `_`. That node's linked inputs
|
||||
are walked like a detailer's image ancestry, through any image node, and the listed
|
||||
samplers found are the candidates. No match, several matches, a walk that fails or no
|
||||
sampler found means no anchor: nothing is guessed and the file stays "several possible
|
||||
prompts". Ingestion ignores file names. Shared-conditioning stages: in a sampler's
|
||||
latent walk, an earlier sampler reading the very same conditioning links (positive and
|
||||
negative, or a guider; compared by node ID text and slot) is the same prompt, not an
|
||||
unattributed stage, when it is a plain first stage: not a detailer, no prompt override
|
||||
set, and no listed sampler in its own latent ancestry (else it stays unattributed) (`tests/test_resolver_saver_anchor.py`, shapes in
|
||||
`tests/saver_anchor_graphs.py` from the maintainer's `gpu_unlock_bench`, dataset and
|
||||
ANIMA runs; browser parity through the `anchor-*` cases, which carry `__file_name`).
|
||||
|
||||
`tools/inspection_corpus.py --root DIR --out JSONL [--workers N] [--baseline JSONL]
|
||||
[--summary TXT] [--code TREE]` measures a media folder read-only against the #240
|
||||
@@ -1196,7 +1245,8 @@ Recorded deviations on this page (spec §8):
|
||||
`#bulkbar` shows "N selected" and only the real actions in the concept's order (Add
|
||||
tags, then D85's Auto-tag selected, Set category, Export metadata, Clear). Add to
|
||||
collection waits for collections; "Remove links…" is dropped. (Amended by #249 PR D:
|
||||
Add tags ships, see "Bulk tagging from the selection bar".) The selection survives Load more and
|
||||
Add tags ships, see "Bulk tagging from the selection bar"; #392 adds range, keyboard,
|
||||
touch and "all N matching" selection, see "Selecting many images", D135.) The selection survives Load more and
|
||||
bounded re-mounting and is cleared whenever the collection is replaced (search,
|
||||
category, sort, Filters, source change, rescan, reload). A return to Images restores
|
||||
the saved collection unreplaced and keeps its selection; a full page reload starts
|
||||
@@ -1634,6 +1684,9 @@ targets of the longest legal path (1024 code points, even escaped as `\uXXXX`) f
|
||||
unchanged, `-` skipped and `!` error.
|
||||
- The progress line is the scan's: "Processing file N of M · about X min left", the
|
||||
estimate (D56's `_time_left`) shown once two files are done, each file one batch.
|
||||
An ended job never keeps it: its status line becomes "<Add tags | Auto-tag>
|
||||
<finished | cancelled> · N files processed" (or the storage-restored sentence),
|
||||
pinned by `test_an_ended_job_shows_no_time_left` (#379 live check).
|
||||
- Download report saves `prompt-manager-image-tags-<YYYYMMDD-HHMM>.json`.
|
||||
- The start toasts are "Tagging started." / "Auto-tagging started." (D85); the end
|
||||
toasts follow the design's pattern with the job's counts, as the scan's: "<title>
|
||||
@@ -1670,6 +1723,65 @@ busy". Units:
|
||||
| Copy without a concept line: no tags, start fallbacks, the panel heading | D118 |
|
||||
| Activity item and job view for image tagging jobs | D89 (plan D66) |
|
||||
|
||||
### Selecting many images (#392)
|
||||
|
||||
**D135** (beyond the concept, which draws only the per-card checkbox): Images selects
|
||||
like a file manager, in Grid, List (where the concept hides the checkbox) and Masonry,
|
||||
over every loaded batch. The selection is a set of image keys in display order, so
|
||||
ranges include cards that bounded rendering has unmounted.
|
||||
|
||||
- Clicking a checkbox toggles it, and that image becomes the anchor. Ctrl/Cmd+click
|
||||
anywhere on a card (image, star, Info, Tag) also toggles it and makes it the anchor;
|
||||
nothing opens, inspects or changes the favorite.
|
||||
- Shift+click on a card selects the range from the anchor to it. Shift+click on a
|
||||
checkbox gives its range the box's new state, so it can clear a range. The anchor
|
||||
stays where it is. Without an anchor, the clicked image becomes the anchor.
|
||||
- In the grid, Ctrl/Cmd+A selects every loaded image; in the search box it keeps its
|
||||
text meaning. Shift+Arrow, Shift+Home and Shift+End move focus as the plain keys do,
|
||||
then select from the anchor (the starting card when there is none) to the new card.
|
||||
Escape in the grid clears the selection. Escape in the bar clears it too, and focus
|
||||
moves to the heading, as after Clear.
|
||||
- Touch: a 500 ms long-press (at most 10 px of movement) selects the card and turns on
|
||||
touch selection mode. The context menu during the press and the click after it are
|
||||
swallowed, and the card has no image callout. Taps then toggle cards until the
|
||||
selection is empty. A keyboard Enter or Space keeps its plain meaning.
|
||||
- A plain click keeps its meaning: the image opens the viewer, and Info inspects.
|
||||
- When every loaded image is checked and the listing has more, the bar offers
|
||||
**Select all N matching** right after the count. Pressing it gives **All N matching
|
||||
selected**: later batches arrive checked, and unchecking any image goes back to the
|
||||
loaded images minus that one. Clear, Escape, and a new search, filter, sort or rescan
|
||||
end it. A refresh after a tag save keeps it. Return visits keep the loaded images,
|
||||
and the offer shows again.
|
||||
- In that mode, Add tags and Auto-tag selected start the `image-tags` job with
|
||||
`filter` (the listing's own query as an object: `q`, `subfolder`, `category`,
|
||||
`tags`, `root`, `media`, `sort`, `favorites`, and with a Source the loaded root's
|
||||
`root_scope`) instead of `targets`. The server resolves the filter at start, in
|
||||
listing order, using the listing's own admission code (`_admitted_entries`). That
|
||||
means "all matching" when the action starts: an output written since then that
|
||||
matches is included. A `root_scope` that no longer names its root (the sources were
|
||||
renumbered) answers 409 `source-changed`. The limit is 10,000 images (`MAX_LISTING_SELECTION`).
|
||||
Over the limit, the client toasts "Narrow the view to at most 10,000 images." and the
|
||||
server answers 400 `too-many-images`. No match is 400 `no-images`. Auto-tag in this
|
||||
mode does not offer Review each image, because the review walks images in the
|
||||
browser. Set category posts `{images_filter, category}` to `/prompt_manager/bulk/category`.
|
||||
The server sets the category on the distinct linked positives and answers
|
||||
`prompt_count` and `skipped`. A listing whose `positive_count` is 0 toasts the
|
||||
nothing-linked sentence and opens no dialog, and over 10,000 matches it toasts the
|
||||
limit sentence. Export metadata reads every image in
|
||||
the browser, so it is `aria-disabled` in this mode, and its `title` says why.
|
||||
|
||||
Tests: `tests/browser/workspace-images.spec.mjs` ("Shift+click selects a range across
|
||||
an appended batch in Grid, List and Masonry", "Ctrl/Cmd+click toggles a card without
|
||||
opening it, and a plain click still opens the viewer", "Ctrl/Cmd+A, Select all N
|
||||
matching and Add tags tag every matching image on the server", "Shift+arrow keys extend
|
||||
the selection from the focused card, and Escape in the bar clears it", "a touch
|
||||
long-press starts selection mode, taps then toggle, and an empty selection ends it");
|
||||
units `tests/js/workspace-images-multiselect.test.js`,
|
||||
`tests/js/workspace-images-tagging-bulk.test.js` ("all matching sends the listing
|
||||
filter instead of targets"), `tests/js/workspace-images-state.test.js`
|
||||
(`listingFilter`); Python `tests/test_image_tag_jobs.py` (`TestFilterStart`) and
|
||||
`tests/test_api_workspace_images.py` (`TestListingFilterSelection`).
|
||||
|
||||
Native image tagging (#249 PR B routes, PR C UI): `POST
|
||||
/prompt_manager/image-annotations/observe` (capture one listed file, its saved set and
|
||||
`tag_revision`), `GET`/`DELETE /prompt_manager/image-annotations/capture/<token>`
|
||||
@@ -2653,7 +2765,8 @@ the connection's `total_changes` is 0 at close and the data reads back unchanged
|
||||
`-shm` bytes are not compared: every WAL reader rewrites its read marks there, so no
|
||||
reader can keep it byte-identical (`test_health_check_never_writes`).
|
||||
|
||||
Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107 and D111–D114; D121–D134 assigned):
|
||||
Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107, D109–D110 and D111–D114; D121–D134 assigned). D94,
|
||||
the per-section hand-off panel, was retired by S12 (#329): every section is native.
|
||||
|
||||
- **D92** (shell): no "Sample configuration" chip. The Save status is truthful: "No
|
||||
unsaved changes.", "Unsaved changes.", "Saved in this browser." / "Saved to the server
|
||||
@@ -2685,10 +2798,6 @@ Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107 and D111–
|
||||
"N output sources" with "Monitoring and enrichment share one source list.", prefixed
|
||||
by "M of N configured sources available." or "None configured is available; ComfyUI's
|
||||
output folder is used." when that is the case.
|
||||
- **D94** (all sections): a section not delivered yet shows the hand-off panel "Your
|
||||
current tools are ready" with a link to each legacy tool that still owns it (the
|
||||
plan's legacy coverage table); retired section by section, gone after S12. Since S11
|
||||
(#328) only Diagnostics & logs hands off (`SECTION_HANDOFFS`, to the admin page).
|
||||
- **D95** (health report): the concept's `health-check` dialog titled "Health report",
|
||||
with no sample note. It lists every check in the server's order: Integrity check,
|
||||
Foreign keys & pair references, Canonical hashes, Pair uniqueness (added), Missing
|
||||
@@ -2719,7 +2828,7 @@ Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107 and D111–
|
||||
- **D123** (Maintenance): the concept's Maintenance notice
|
||||
("Check first. Review changes. Then apply." + Run health checks) was delivered ahead of
|
||||
the rest of Maintenance (S2), as the native place that runs the health check; since S7
|
||||
the section's two surfaces follow it and its D94 hand-off is gone. A press shows "Checking…" (`aria-disabled`) and opens the report; a
|
||||
the section's two surfaces follow it and its hand-off panel is gone. A press shows "Checking…" (`aria-disabled`) and opens the report; a
|
||||
failed check toasts "The health check could not finish. Nothing was changed."
|
||||
- **D97** (Appearance): added rows for the legacy gallery settings the concept leaves
|
||||
out, after its own rows: "Show image details on cards" (`showImageInfo`; off hides a
|
||||
@@ -2840,6 +2949,11 @@ Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107 and D111–
|
||||
and the media hashes, `GET /__test__/system/digest`). `plan dialogs match the concept`
|
||||
(system-maintenance-concept.spec, both themes and sizes) compares each of the four
|
||||
operations' dialogs with the concept's, mapped by these values only.
|
||||
S12 (#329): the `rotated_logs` plan (Diagnostics & logs › Review log cleanup, `h3`
|
||||
"Rotated log copies") changes no library record: its Backup reads "Not needed: no
|
||||
library record changes" (`backup.policy: "none"`), it has no D55 backup box, and its
|
||||
`details` reads "Listed files (N)" (`a plan without a database change reads Not needed
|
||||
and shows no backup box`, `rotated_logs details read Listed files`).
|
||||
- **D101** (Maintenance, S7): Duplicate image files → Find duplicates starts the
|
||||
duplicate search (Activity, D111); its job view adds "Review groups" after Download
|
||||
report once it finished with groups. The concept's `duplicates` dialog ("Review
|
||||
@@ -3086,37 +3200,104 @@ Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107 and D111–
|
||||
again"; an Unload that found nothing loaded only reads the status again. At phone width
|
||||
the concept's 120px select cap cuts every engine label, so `#tag-model` alone keeps its
|
||||
natural width there and its row wraps the select under the text (`workspace.css`
|
||||
`/* D132 */`); Existing tags stays as the concept draws it, cut at 390 as there. The
|
||||
comparison maps `selectsFit` only for the engine select and measures each row's
|
||||
`flexWrap` and whether its control sits under the text. Pinned by `auto-tagging matches
|
||||
the concept` (both themes and sizes), `auto-tagging at 390 wraps only the engine row`,
|
||||
`auto-tagging engine select fits at 320 wide` and the mutation check (it includes a
|
||||
wrapped row) (system-tagging-concept.spec), and the model, Download, Unload and Preview
|
||||
`/* D132 */`). The concept's cap cuts Existing tags too ("Replace after review"); by the
|
||||
maintainer's decision (#398) that parity is not kept: `#tag-existing` never shrinks for
|
||||
its row text and at phone width keeps its natural width beside the text (its row does
|
||||
not wrap; the text column narrows). 1600 is as the concept draws it. The comparison
|
||||
expects `selectsFit` true for every select and measures each row's `flexWrap` and
|
||||
whether its control sits under the text. Pinned by `auto-tagging matches the concept`
|
||||
(both themes and sizes), `auto-tagging at 390 wraps only the engine row`, `auto-tagging
|
||||
selects fit every option at 390/320 wide` (every option of both selects, both themes, no
|
||||
page scroll) and the mutation check (a wrapped row, a squeezed engine and Existing tags
|
||||
select) (system-tagging-concept.spec), and the model, Download, Unload and Preview
|
||||
tests of workspace-system-tagging.spec.
|
||||
- **D133** (Auto-tagging, S11): a "Suggestions per image" row (`#tag-suggestions`,
|
||||
number 1–100, host key `autotag_suggestion_limit`) follows the two confidence rows:
|
||||
the persisted count the Tag image dialog's Suggestions field (D91) starts from. Pinned
|
||||
by the comparison's mapping (it inserts the row) and `defaults save through Save
|
||||
preferences and the Tag image dialog uses them`.
|
||||
- **D134** (Auto-tagging, S11, OQ2/OQ3/OQ7/OQ8): an `.operation` "Tag output images"
|
||||
with "Review" follows `.row-actions`: the legacy prompt-tag batch. Review is
|
||||
- **D134** (Auto-tagging, S11, OQ2/OQ3/OQ7/OQ8; amended by #379): an `.operation` "Tag
|
||||
output images" with "Review" follows `.row-actions`: the per-image batch (#379), whose
|
||||
description reads "Tag each output image from its own pixels with the model above.
|
||||
Prompts are not changed. Review the scope first.". Review is
|
||||
`aria-disabled` (the press explains) while the chosen engine is not downloaded or a
|
||||
value it uses is invalid; otherwise it posts `/autotag/batch/preview` with the drafted
|
||||
engine and policy (read-only) and opens "Review tag output images" in the concept's
|
||||
record-changing review (its `maintenancePlan` shell): `h3` "Tag output images", the
|
||||
`dl.meta-grid` Model, Existing tags, Images linked, Prompts, Already tagged, Prompts
|
||||
to tag ("up to N"), Settings ("General <g> · Character <c>" or "Template of N
|
||||
characters") and Backup, the upper-bound sentence in the `p.small`, the D55 backup box
|
||||
and Cancel / "Start tagging", which starts the cancellable `autotag` job and opens it.
|
||||
Replace after review, a model not on disk, a short disk, no linked image and nothing to
|
||||
tag each show the rows with only Cancel and the reason ("Replacing tags needs a review
|
||||
of each image. Use Preview tagging review, or choose Skip or Merge for a batch run.",
|
||||
"Every prompt with an output image is already tagged.", "Not enough free disk space:
|
||||
the safety backup needs about X MB and Y MB is free. Nothing is written until there is
|
||||
room.") in the close-only review Integrations shares (`openCloseOnlyReview`). Pinned by
|
||||
`dl.meta-grid` Model, Existing tags, Images, Already auto-tagged, Images to tag
|
||||
("about N"), Time, Settings ("General <g> · Character <c>" or "Template of N
|
||||
characters", then " · up to L tags per image" with the preview's saved
|
||||
`suggestion_limit`) and Backup, the estimate sentence ("Nothing is written until you
|
||||
start. The number to tag is an estimate: images changed since they were tagged are
|
||||
tagged again, and missing files are skipped. Activity reports what was tagged.") in the
|
||||
`p.small`, the D55 backup box and Cancel / "Start tagging", which starts the
|
||||
cancellable `autotag` job and opens it. Time (DEC11) is `m = max(1,
|
||||
ceil(estimate_seconds / 60))` read "about m min" under 90 and "about H h M min" (or
|
||||
"about H h") from 90, followed by " on a processor; a graphics card can be faster."
|
||||
or, when the preview says WD14 runs on CUDA, " on the graphics card."; the language
|
||||
models read "Language-model engines take longer per image. Activity shows the time
|
||||
left once tagging starts.". The option label "Skip already tagged" is the concept's;
|
||||
for this batch it skips images already auto-tagged (hand tags do not count). Replace
|
||||
after review, a model not on disk, a short disk, no image and nothing to tag each show
|
||||
the rows with only Cancel and the reason ("Replacing tags needs a review of each
|
||||
image. Use Preview tagging review, or choose Skip or Merge for a batch run.", "The
|
||||
output folders contain no images to tag.", "Every output image is already
|
||||
auto-tagged.", "Not enough free disk space: the safety backup needs about X MB and Y
|
||||
MB is free. Nothing is written until there is room.") in the close-only review
|
||||
Integrations shares (`openCloseOnlyReview`). Pinned by
|
||||
`tag output images review matches the concept`, its mutation check and `the
|
||||
Cancel-only review matches the concept review shell` (system-tagging-concept.spec) and
|
||||
the batch tests of workspace-system-tagging.spec.
|
||||
the batch tests of workspace-system-tagging.spec; the mutation check renames each grid
|
||||
label in turn.
|
||||
- **D109** (Logs, S12, #329): the concept's Logs surface on the live server. The toolbar
|
||||
keeps "Level", Refresh and the download (renamed "Download log": the active log,
|
||||
`/logs/download/prompt_manager.log`, through the api client), with all six minimum
|
||||
levels (All levels, DEBUG, INFO, WARNING, ERROR, CRITICAL; the concept has three) and an
|
||||
added `label` "Lines" + `input#log-limit[type=number]` 1–500 (default 100): a value that
|
||||
is not a whole number 1–500 after trimming gets `aria-invalid` and sends nothing, a
|
||||
Level change included. The `pre` shows the in-memory buffer newest first as `HH:MM:SS
|
||||
LEVEL message` ("No log entries at this level." / "The log could not be read."); a
|
||||
newer read wins. Added after it, `p.small#log-stats[role=status]` "Log statistics":
|
||||
"N entries in memory · E errors · W warnings · F files · S on disk" (errors are ERROR +
|
||||
CRITICAL, singular for 1, links not counted). "Runtime log level" adds CRITICAL after
|
||||
the concept's INFO, WARNING, DEBUG, ERROR; it and "Console logging" are runtime drafts
|
||||
saved by Save preferences (`POST /logs/config`, only the changed keys) and read the
|
||||
server's values; their descriptions say "Applied until the server restarts." (the
|
||||
concept's say a gallery toggle is replaced and output stays quiet). Inside "Log files &
|
||||
retention", after the concept's line ("prompt_manager.log · <size> · rotated copies:
|
||||
N", N the `rotated` rows) and Review log cleanup (D100's `rotated_logs` plan): a
|
||||
`p.small` "Rotates at M MB · keeps K rotated copies · file logging on|off · memory
|
||||
buffer B entries" (read-only), an `.operation-list` of `/logs/files` (`h3` name, `p`
|
||||
"<size> · <YYYY-MM-DD HH:MM> · Active log | Rotated copy | Other file | Link, not
|
||||
downloaded", "Download" named "Download <name>" except for a link), and the danger
|
||||
`.operation` "Clear all logs" / "Empty the active log and delete its rotated copies." /
|
||||
"Clear all logs…", which confirms ("Empties prompt_manager.log, deletes N rotated
|
||||
copies (S) and clears the lines shown here. The other files in the log folder are
|
||||
kept. This cannot be undone."), runs `POST /logs/truncate`, toasts "Logs cleared · N
|
||||
rotated copies deleted." (plus " · K could not be deleted") and reads the viewer,
|
||||
statistics, files and configuration again. Pinned by workspace-system-diagnostics.spec's
|
||||
Task 5 and Task 6 tests.
|
||||
- **D110** (Diagnostics, S12, #329): the concept's Diagnostics surface with live values.
|
||||
The `div.meta-grid` `dd`s read one `GET /system/diagnostics`: Database "Readable ·
|
||||
<size> · N images" or "Not readable: <message>"; Sources "N sources · M watching", "N
|
||||
sources · K not available", or with none configured "ComfyUI output folder · watching
|
||||
| manual scan"; Viewer.js "<pinned version> · pinned vendor". Host renderers is D106's
|
||||
renderer line from the send bridge. Each reads "Loading…" first, "Busy: <job> is
|
||||
running." when an exclusive job refuses the read (D57's naming) and "Could not be
|
||||
read." on any other failure. Run checks opens the shared Health report (D95). The
|
||||
buttons drop the concept's "sample": "Download diagnostic report" saves a fresh report
|
||||
plus `host_renderers` as `prompt-manager-diagnostics-YYYYMMDD-HHMM.json` (local time)
|
||||
and toasts "Diagnostic report downloaded." ("Busy: <job> is running. Try again when it
|
||||
ends." / "The diagnostic report could not be downloaded. Nothing was saved to this
|
||||
device."); "Inspect" opens `#/inspect`, which reads a file and writes nothing. An added
|
||||
`.operation` "Test image link" / "Write a placeholder image record to check that
|
||||
linking works." / "Create test link…" keeps the legacy synthetic link (S12-11): it
|
||||
reads the most recent prompt on mount (`aria-disabled` with "Add a prompt first." or
|
||||
"The most recent prompt could not be read." added to the description), confirms in
|
||||
"Create test link" naming the prompt and how Missing-file links removes the record,
|
||||
and toasts "Test link created (image record N)." or "The test link could not be
|
||||
created. Nothing was changed.". Pinned by workspace-system-diagnostics.spec's Task 4
|
||||
tests.
|
||||
- **D124** (inspector): with "Load metadata on selection" off, the Images panel and
|
||||
the viewer's details show "Metadata loads when you ask for it." and a "Load metadata"
|
||||
button instead of reading the file. The concept has no such state, so no concept
|
||||
@@ -3491,7 +3672,9 @@ workflow (README changelog).
|
||||
### LoRA reference import contract (#221, #249 System S10)
|
||||
|
||||
The LoRA import is staged (`py/lora_import.py`): discover the LoraManager
|
||||
`*.metadata.json` files and fingerprint them (path, size, modification time); stage each
|
||||
`*.metadata.json` files and fingerprint every scanned one, kept or skipped (path, size,
|
||||
modification time and whether it was kept, #362; a skipped sidecar added, removed or edited
|
||||
after the review refuses Start with `plan-changed`, `test_start_refused_when_a_skipped_sidecar_changed`); stage each
|
||||
record (CivitAI example images go into the image cache first, best effort, counted in
|
||||
`downloaded` / `download_errors`; each image is written to a temp file beside its cache
|
||||
entry and moved in place, so a half-written image is never seen as cached); the files are
|
||||
@@ -3557,25 +3740,67 @@ the kept, readable LoRAs and `skipped` the rest. Media per LoRA, without repeats
|
||||
preview (`preview_url`, else LoraManager's `PREVIEW_EXTENSIONS` order), then every file in
|
||||
`<example-images root>[/<library>]/<sha256>/` in natural order; a leading video yields to the
|
||||
first still. A file counts only with a servable extension and inside the guarded media
|
||||
roots. CivitAI downloads run only for a LoRA without local media. The folder guard keeps a
|
||||
settings folder only when it exists and is neither a filesystem root (`/`, `C:\`, a UNC
|
||||
share root) nor the home folder or one of its ancestors (realpath + `normcase`). An
|
||||
unusable layout (no install, unreadable / non-JSON / non-object / oversized settings, no
|
||||
`platformdirs`, a missing or refused centralized sidecar root) gives a review without a
|
||||
token ("LoraManager settings: not usable: <reason>. Nothing was changed."), a legacy scan
|
||||
error with that reason, the default LoRA roots for serving, and trigger words from the
|
||||
default roots in alongside mode. `tests/run_suite.py` and `tests/browser/server.py` point
|
||||
roots. Only an absolute `preview_url` is used; a relative one is skipped with a DEBUG line
|
||||
naming the sidecar. A sidecar that is not valid UTF-8 JSON is skipped like any unreadable
|
||||
one (#362). CivitAI downloads run only for a LoRA without local media; a record's
|
||||
images are linked in reverse order, so its first image (the cover) is the newest link and
|
||||
the card image. The folder guard keeps a folder from the settings, and each of ComfyUI's
|
||||
default LoRA folders (`models/loras`, `extra_model_paths.yaml`, `folder_paths`), only when
|
||||
it exists and is neither a filesystem root (`/`, `C:\`, a UNC share root) nor the home
|
||||
folder or a folder that contains it (realpath + `normcase`); a refused default folder that
|
||||
exists is listed in the layout's `refused_defaults` and logged as one warning each time the
|
||||
refused set changes (`RefusedDefaultsTests`). An unusable layout (no install, unreadable /
|
||||
non-JSON / non-object / oversized settings, no `platformdirs`, a missing or refused
|
||||
centralized sidecar root, "the sidecar folder is a filesystem root, the home folder or a
|
||||
folder that contains it") gives a review without a token ("LoraManager settings: not
|
||||
usable: <reason>. Nothing was changed."), a legacy scan error with that reason, the default
|
||||
LoRA roots for serving (no example-images root), and trigger words from the default roots
|
||||
in alongside mode. A folder discovery cannot list answers the review with 500, code
|
||||
`folder-unreadable` ("A LoRA folder could not be listed (...). Nothing was changed.").
|
||||
`TriggerWordCache.load` never raises: an `OSError` is a warning, anything else is logged
|
||||
with its traceback (#362), both keep the previous map, and a running node retries a failed
|
||||
load only after 60 s (`TRIGGER_RETRY_SECONDS`). `tests/run_suite.py` and `tests/browser/server.py` point
|
||||
`LORA_MANAGER_SETTINGS_DIR` at a disposable folder, so no test reads the developer's real
|
||||
LoraManager settings; tests that call `load_layout` inject their own folder too.
|
||||
|
||||
**Linked-only gate and the layout cache (#362).** While the integration is on, the LoRA
|
||||
allow-list (`ImagesMixin._lora_image_dirs`) is the layout's `media_roots` plus the CivitAI
|
||||
download cache, and `_media_roots(outputs)` appends it after the output folders for Library
|
||||
previews, the Library prompt detail, the per-prompt images page, `serve_output_image` for a
|
||||
root past the outputs, `inspect_image` and `reveal_image`. Under such a root only a file the
|
||||
database records as a linked image is served, inspected or revealed (`_linked_only_error`,
|
||||
by its given and its real path); anything else answers 403. Library responses mark that
|
||||
media `linked_only`: no preview file, no favorite, no Tag button. The allow-list's layout
|
||||
comes from `py/lora_layout_cache.py` (`tests/test_lora_layout_cache.py`, which injects the
|
||||
clocks, `stat` and `ntpath` and never sleeps): one entry keyed by the stripped configured
|
||||
path, reused while the plugin settings file's and the settings file's `(st_mtime_ns,
|
||||
st_size)` (or "missing") are unchanged, the install and every media root still resolve to
|
||||
themselves (a folder swapped for a symlink reloads and the guard runs again), and the entry
|
||||
is under `MAX_AGE_SECONDS` (30 s, monotonic clock) old. A load is stored only when no
|
||||
watched file was modified within `RACY_SECONDS` (3 s) of the load (git's racy rule: a write
|
||||
during the load, LoraManager's in-place write, FAT/HFS+ coarse times), every watched file
|
||||
could be checked (any `stat` error but not-found, and an offline Windows share, is "cannot
|
||||
validate"), the settings were readable, and no `clear()` ran meanwhile (generation
|
||||
counter). Hits take no lock; concurrent misses each load. A finished import (System job or
|
||||
legacy scan, both through `replace_lora_imports`) clears it. The integration-off check runs
|
||||
before it, and the configured-path rule (a configured path that is not an install under
|
||||
`custom_nodes` adds no folders) on its result; a hit also needs the same home folder, which
|
||||
the guard compares against. `serve_image` builds the allow-list in the
|
||||
executor. The cache writes one INFO line, "LoraManager layout changed: <source> settings,
|
||||
<mode> storage, <n> media folders", when a stored layout differs from the last one logged.
|
||||
The review, Start, the legacy scan and the trigger cache read the layout fresh. Image route
|
||||
tests clear the per-process cache in `ImageRouteCoverageCase.setUp`; `write_settings` in
|
||||
`TestLoraManagerMediaServing` back-dates its file 10 s so the first load is stored.
|
||||
|
||||
`tests/browser/workspace_integrations_fixture.py` (`/__test__/integrations/*`) makes a fake
|
||||
LoraManager install and LoRA folder under the server's temp root, holds or fails the
|
||||
import, saves a blocking pair and answers CivitAI downloads with a stub opener, so no
|
||||
test reaches the network.
|
||||
|
||||
```sh
|
||||
python3 -m unittest tests.test_lora_manager_layout tests.test_lora_import tests.test_api_lora \
|
||||
tests.test_lora_database tests.test_lora_utils -v
|
||||
python3 -m unittest tests.test_lora_layout_cache tests.test_lora_manager_layout \
|
||||
tests.test_lora_import tests.test_api_lora tests.test_lora_database tests.test_lora_utils \
|
||||
tests.test_api_images -v
|
||||
node --test tests/js/workspace-system-integrations.test.js tests/js/prompt-bridge-adapter.test.js
|
||||
npm run --prefix tools/vendor test:workflows -- workspace-system-integrations.spec.mjs \
|
||||
system-integrations-concept.spec.mjs \
|
||||
@@ -3587,7 +3812,8 @@ npm run --prefix tools/vendor test:workflows -- workspace-system-integrations.sp
|
||||
System › Auto-tagging (`web/js/workspace/system-tagging.mjs`, D132–D134) edits one
|
||||
server-side tagging configuration, `autotag.defaults` in config.json: `model_type`
|
||||
(`wd14-swinv2`, `wd14-vit`, `gguf`, `hf`), `existing` (`skip`, `merge`,
|
||||
`replace_after_review`), `general_threshold` and `character_threshold` (0–1),
|
||||
`replace_after_review`), `general_threshold` and `character_threshold` (0–1 in
|
||||
steps of 0.05, #369),
|
||||
`template` (non-blank, at most 4,000 characters and 16 KiB) and `suggestion_limit`
|
||||
(1–100). They are host drafts (`autotag_model_type`, `autotag_existing`,
|
||||
`autotag_general_threshold`, `autotag_character_threshold`, `autotag_template`,
|
||||
@@ -3598,7 +3824,15 @@ naming the setting by its row label ("General-tag confidence must be between 0 a
|
||||
`settingRefusal` in system.mjs only for a save that carried `autotag_*` keys; every
|
||||
other section's plain 400 keeps the generic status) and the control is highlighted.
|
||||
The template's limit counts code points, as Python's `len` does. A hand-edited bad value
|
||||
falls back to its default on load, one key at a time. `GET /autotag/models` reports the
|
||||
falls back to its default on load, one key at a time. Every page shows the thresholds in
|
||||
steps of 0.05 (the System rows, the Tag image dialogs, the legacy sliders), so a saved
|
||||
threshold off that grid is refused ("General-tag confidence must be a multiple of
|
||||
0.05.", `validate_saved_threshold`; `draftProblem` holds back Preview tagging review and
|
||||
the batch Review with the same sentence) and falls back on load; an accepted one is
|
||||
stored exactly on the grid (#369). A per-request threshold keeps its own check: `/image-annotations/suggest` and
|
||||
`/jobs/autotag` take 0–1; the legacy `/autotag/start` passes its options through
|
||||
`validate_options` and starts the `autotag` job titled "Tag output images (legacy page)";
|
||||
the legacy `/autotag/single` only requires a number. `GET /autotag/models` reports the
|
||||
saved values (`wd14_general_threshold`, `wd14_character_threshold`, `suggestion_limit`,
|
||||
`default_prompt`, `model_type`, `existing`), so the Tag image and Auto-tag selected
|
||||
dialogs start from them; a preset (the bulk review, Preview tagging review) wins.
|
||||
@@ -3624,19 +3858,96 @@ images", "Tag output images (legacy page)", "Tagging one image (legacy page)",
|
||||
`409 busy`; the legacy SSE route attaches to the same job and keeps its frames.
|
||||
- **Unload**: `POST /prompt_manager/autotag/unload` answers `{success, message}` or
|
||||
`409 model-busy`.
|
||||
- **Tag output images**: `POST /prompt_manager/autotag/batch/preview {model_type,
|
||||
existing}` writes nothing and opens no file; it answers `images`, `prompts`,
|
||||
`already_tagged`, `to_tag` (an upper bound), `model {type, label, size_label,
|
||||
downloaded}`, `existing`, `review_required`, `backup {policy}` and `disk`. `POST
|
||||
- **Tag output images** (per image since #379; prompt tags and prompt text are never
|
||||
written): its targets are every still image the output folders list (the Images
|
||||
listing of each root, linked to a prompt or not; videos are not targets; a file under
|
||||
two overlapping roots is one target). `POST /prompt_manager/autotag/batch/preview
|
||||
{model_type, existing}` writes nothing and opens or stats no image file (it may wait
|
||||
for the shared listing walk); it answers `images`, `already_tagged` (images with any
|
||||
auto-tagged byte record), `to_tag` (`images − already_tagged` under Skip, `images`
|
||||
under Merge; both estimates), `estimate_seconds` (WD14: `to_tag` × 1.6 s on the
|
||||
processor or × `WD14_GPU_SECONDS_PER_IMAGE` when ONNX Runtime offers CUDA, see
|
||||
`py/tagging_defaults.py`; `null` for GGUF and HF), `estimate_device` (`cpu`, `gpu`,
|
||||
or `null`), `suggestion_limit` (the saved count the run uses), `model {type, label,
|
||||
size_label, downloaded}`, `existing`, `review_required`, `backup {policy}` and
|
||||
`disk`. `POST
|
||||
/prompt_manager/jobs/autotag {model_type, existing, options}` (WD14: both thresholds;
|
||||
GGUF/HF: `instructions`, the template) starts the cancellable `autotag` job (202),
|
||||
title "Tag output images", admission `records`, a safety backup by the policy first.
|
||||
The job runs the `image-tags` loop (`_tag_each_file`) on each image: a capture, at
|
||||
most `suggestion_limit` model tags, a re-hash, then a union into that image's tags
|
||||
that also marks its bytes auto-tagged (`media_image_autotags`, one row per byte
|
||||
record; an empty result and an unsupported still are marked too; File changed, File
|
||||
not found, Too large to verify, read, inference and tag-limit errors are not). Phases,
|
||||
each checked for Cancel: backup, model load, listing ("Listing output images…"), a
|
||||
pre-pass that stats each file without opening it (gone: "File not found", over 256
|
||||
MiB: "Too large to verify", under Skip a marked current record: "Already
|
||||
auto-tagged"; counted with `record_counted`, rows up to the cap, no log line each),
|
||||
then the loop ("Processing file {k + n} of {T}", the time left from the model's own
|
||||
rate, D56 unchanged). It ends "Tagged N images", "Cancelled. Tags already added were
|
||||
kept.", or failed with "Prompt storage was restored; start the job again.", "The
|
||||
output folders could not be listed. Try again." or the model's refusal.
|
||||
Refusals: `400 review-required` (Replace after review never runs as a batch), `409
|
||||
model-unavailable` ("<label> is not downloaded. Download it in System ›
|
||||
Auto-tagging."), `507 insufficient-space`, `409 busy`, `409 model-busy`. The section
|
||||
toasts `model-busy` with its operation, the other three with the server's sentence,
|
||||
`busy` with the running job (D57), anything else "<what> could not start. Try again.".
|
||||
Outcomes: Tagged / Unchanged / Skipped / Errors (the `image-tags` descriptor, D111).
|
||||
`tests/test_autotag_batch.py` pins it, including two images of one prompt getting
|
||||
different tags with no `prompts` or `prompt_tags` row changed; the marks' schema group
|
||||
is pinned by `tests/test_autotag_schema.py` (forward-only: one empty `WITHOUT ROWID`
|
||||
table, optional in old backups, refused without the annotation group).
|
||||
|
||||
- **The legacy pages (#379, PR 2)** write image tags, never prompt tags.
|
||||
`POST /autotag/start` (the admin and gallery batch) reads `model_type`, `prompt`,
|
||||
`skip_tagged` (Skip or Merge), `keep_in_memory` and the thresholds (else the saved
|
||||
ones), checks them with `validate_options` (WD14: both thresholds; GGUF/HF:
|
||||
`instructions` = the non-blank `prompt`, else the saved template; the saved
|
||||
"Suggestions per image") and starts the `autotag` job "Tag output images (legacy
|
||||
page)" (gate owner `legacy-batch`) through the native start, so the safety backup and
|
||||
the disk check apply. A second legacy batch answers JSON 409 `{success: false, error:
|
||||
"Auto-tagging is already running"}` (also when another legacy start wins the race past
|
||||
that check); `SseStream` hands a non-2xx JSON `error` to `onerror` as `serverMessage`,
|
||||
which both pages show instead of "Connection lost during auto tagging". Any other refusal (a job in the way: "<title>
|
||||
already running", `model-unavailable`, `insufficient-space`, `model-busy`, a bad
|
||||
option) is one `{type: "error", status: "error", message}` frame. The stream relays
|
||||
the job: the phases as `{type: "progress", progress: 0 | 5 | 8, status: "Taking a
|
||||
safety backup…" | "Loading model…" | "Listing output images…"}`, each image as
|
||||
`{type: "progress", progress: 10 + progress × 85 / 100, status, processed, tagged,
|
||||
skipped}` (at most every 0.5 s; `processed` = Tagged + Unchanged + Errors), and the end
|
||||
as `complete` (`processed`, `tagged`, `skipped`, `errors`, `skipped_outside_roots: 0`,
|
||||
`status` "Complete" or "No still images found", `model_status`, `model_loaded`),
|
||||
`{type: "cancelled", message: "Cancelled. Tags already added were kept."}` or an
|
||||
`error` frame with the job's status line. A job quiet for 5 s repeats its last frame,
|
||||
and a closed stream cancels the job (the legacy pages have no other stop).
|
||||
`POST /autotag/single` takes `existing` (`"skip"` or `"merge"`, default merge). Under
|
||||
Skip, an image whose current bytes are auto-tagged answers `{success, already_tagged:
|
||||
true, image_tags, prompt_id, filename}` after one stat, without the model gate.
|
||||
Otherwise the image is captured and the model runs on the capture under the gate with
|
||||
the saved cap; the answer adds `tags`, `sha256` (the captured bytes) and `image_tags`,
|
||||
and `service.custom_prompt` is never set. Refusals: `409 model-unavailable`, `413
|
||||
too-large`, `404 file-not-found`, `409 file-changed`, `503 busy`, `415 not-a-still`
|
||||
(the image is marked), `409 model-busy`. `POST /autotag/apply` takes exactly
|
||||
`{image_id, sha256, tags}` or `{path, root_index?, sha256, tags}` (64 KiB), re-hashes
|
||||
the file and unions the tags into that image (`add_image_tags`, marked): `{success,
|
||||
added_tags, total_tags}`; an empty list only marks the image. A body with `prompt_id`
|
||||
is `400` "This page is out of date. Reload it."; a changed file `409 file-changed`
|
||||
("The image changed since its tags were generated."); a full set `409 tag-limit`; a
|
||||
restore `409 storage-changed`. The admin review asks each image with Skip first: an
|
||||
auto-tagged one shows "Image Already Auto-tagged" / "A tagging model already tagged
|
||||
this image. Generate new tags?" with its image tags (Skip, Skip All Tagged, Re-tag asks
|
||||
again with Merge); the gallery review always merges. Apply & Next sends the image and
|
||||
its `sha256` (linked or not). The batch copy: "Skip images already auto-tagged",
|
||||
"Auto tagging complete! Tagged N images." ("1 image"), and under both progress bars
|
||||
"Closing this window stops tagging. Tags already added are kept. Prompt Manager's
|
||||
Activity page shows the time left.". Pinned by `tests/test_autotag_legacy.py`,
|
||||
`tests/js/image-helpers.test.js` and `legacy-autotag.spec`.
|
||||
- **Tag image dialog marks (DEC4).** A save after Generate suggestions ran on that
|
||||
capture marks the image auto-tagged (also a replayed save, and a saved retry after
|
||||
`tag-conflict`); a save without Generate does not. The API keeps the suggested capture
|
||||
tokens; each new one drops the tokens whose capture is gone (`expires_in`, which never
|
||||
resets a capture's idle clock), a release or a saved receipt drops its own, and
|
||||
closing the captures clears them (`TestDialogMarks`).
|
||||
|
||||
No browser test downloads or runs a model: the injected `TestModel` answers, its
|
||||
`download_model` raises unless `/__test__/tagging/download {mode: hold | succeed |
|
||||
@@ -3645,6 +3956,203 @@ it. Every workspace-system-tagging test asserts no `/autotag/download/` request.
|
||||
Changing the routes or `py/config.py` needs a ComfyUI restart; the section itself is
|
||||
HTML, CSS and JS only.
|
||||
|
||||
### Logs and diagnostics contract (#249 System S12)
|
||||
|
||||
System › Diagnostics & logs (`web/js/workspace/system-diagnostics.mjs`, D109–D110)
|
||||
reads and deletes only through the routes below; the legacy admin Logs and Diagnostics
|
||||
modals use the same routes and keep working.
|
||||
|
||||
- **Rule R** (`utils/logging_config.py`): a file may be deleted as a rotated copy only
|
||||
when it is a direct entry of the log folder named `prompt_manager.log.<n>` (`<n>` a
|
||||
decimal integer ≥ 1 with no leading zero, compared after `normcase`) and a regular
|
||||
file that is not a link or junction (`lstat`). Every other entry is `active`, `link`
|
||||
or `other` and is never deleted. `/logs/files` rows carry that `kind`; links are listed
|
||||
with their own `lstat` size and are not counted or sized by `/logs/stats`. A download
|
||||
is served only for a regular, non-link file of the folder (a link, a folder or a name
|
||||
not listed is 404), read off the event loop.
|
||||
- **Reviewed cleanup**: `POST /maintenance/preview {operation: "rotated_logs", options:
|
||||
{}}` lists every rotated copy (basenames, size and time) and writes nothing; `POST
|
||||
/maintenance/apply {token}` starts the `maintenance` job "Rotated log copies"
|
||||
(admission `read`, not cancellable). Under the manager's files lock and every file
|
||||
handler's lock it re-scans; when any listed file is missing, no longer rotated or
|
||||
differs in size, `mtime_ns` or a non-zero inode, it deletes nothing and fails
|
||||
`plan-changed` ("The log files changed since the review; nothing was deleted. Review
|
||||
again.", Activity offers Review again). Otherwise it deletes exactly the listed files;
|
||||
each is a Changed row ("Deleted · <size>"), a file that cannot be removed an Errors
|
||||
row. Two Prompt Manager processes sharing one log folder are not locked against each
|
||||
other: each file's identity is checked just before its unlink.
|
||||
- **Clear all logs** (`POST /logs/truncate`, also the admin page's Clear) empties the
|
||||
active log through its handler, deletes the rotated copies (Rule R) and clears the
|
||||
buffer; other files and links are kept. Response keys are unchanged. It deletes or
|
||||
empties no more than its confirmation showed (#389):
|
||||
- `/logs/files` rows carry `identity`, the string `"<size>:<mtime_ns>:<ino>"` (a
|
||||
string: `mtime_ns` and many inodes are beyond a browser's exact integers). The
|
||||
workspace sends `{"expected": [{name, identity}]}` with the active and rotated rows
|
||||
its dialog was built from. Under the files lock and every handler lock each listed
|
||||
rotated copy must still match (`_matches_review`) and the listed active log must be
|
||||
the same inode, or nothing changes: 409 `logs-changed`. Only listed copies are
|
||||
deleted; one that appeared after the confirmation is kept. A malformed list is 400.
|
||||
- Without a body (the legacy admin page, whose confirm lists nothing; also the
|
||||
workspace when the file list could not be read and the dialog said "deletes its
|
||||
rotated copies") every rotated copy present at the locked rescan is deleted.
|
||||
- Either way, before anything changes (best effort, not a cross-process lock: a
|
||||
write before our own last write, or any while file logging is off here, is not
|
||||
seen): a second writer of the active log (our
|
||||
handler's flushed stream is behind the file's size, or its file is now a rotated
|
||||
copy: another ComfyUI sharing the folder) is 409 `logs-foreign-writer`; an active
|
||||
log with more than one name (`st_nlink > 1` by `lstat` or the handler's `fstat`,
|
||||
NTFS included) is 409 `logs-hard-link`, never truncated (replacing it instead would
|
||||
rename a file our handler holds open, which Windows refuses). Each copy is
|
||||
`lstat`-checked again just before its unlink; one that changed stops the clear there
|
||||
and keeps it, the copies after it and the active log, which is emptied last.
|
||||
- The browser fixture grows the active log through the handler's own stream, since a
|
||||
second handle appending to it is exactly what Clear refuses.
|
||||
- **Runtime settings** (`GET/POST /logs/config`): the level (DEBUG…CRITICAL) and console
|
||||
logging apply until the server restarts; the retention values are shown read-only in
|
||||
the workspace and the API keeps accepting them.
|
||||
- **Report** (`GET /system/diagnostics`): built on the server from an explicit list of
|
||||
fields (version, Python, platform, Viewer.js version, the four legacy checks, database
|
||||
info, source counts, log configuration and statistics, the session's health summary
|
||||
and the non-path configuration with `has_civitai_api_key`). Every path is relative to
|
||||
a ComfyUI anchor or reads "outside ComfyUI"; a backstop scrub replaces any absolute
|
||||
path or the saved key and counts it in `redacted`, which the tests require to be 0. It
|
||||
takes a registry lease (409 `busy` while an exclusive job such as a restore runs).
|
||||
- **Compressed downloads**: the gzip middleware drops a handler's declared
|
||||
`Content-Length` before it compresses a body, so the response declares the compressed
|
||||
length. A stale length made a browser wait for bytes that never came: the log download
|
||||
of any file over 1 KB hung (`test_compressed_body_declares_its_own_length`).
|
||||
|
||||
The browser server writes its logs into `<server root>/logs`
|
||||
(`tests/browser/workspace_diagnostics_fixture.py`), whose hooks seed records with fixed
|
||||
times, rotated copies, a link (POSIX) and a user's `.bak`, and roll the copies over; no
|
||||
test reads or deletes a real log folder.
|
||||
|
||||
`tests/browser/system-diagnostics-concept.spec.mjs` compares the section and its log
|
||||
cleanup review with the concept (dark and light, 1600x1100 and 390x844, no page scroll;
|
||||
both selects show their whole option at 320 wide). The fixture's `{mode: "concept"}` seed
|
||||
reproduces the concept's Logs sample: only its four buffer lines, a 1.4 MB active log with
|
||||
three rotated copies and console logging off, so those values are compared as the concept
|
||||
prints them. `mapDiagnosticsConcept` applies only D109 and D110, and `mapCleanupConcept`
|
||||
D100 (the shared `mapPlanConcept`) with D109's rule for this plan (no backup box, "Not
|
||||
needed: no library record changes", "Listed files"). The mutation check asserts that the
|
||||
unmapped difference is exactly the deviated paths. The comparison found that the Database
|
||||
line read `database_info.size_bytes`, a key the report does not have, so it always said
|
||||
"0 bytes"; it reads `database_size_bytes`, the key `get_database_info()` returns.
|
||||
|
||||
### Legacy System capability coverage
|
||||
|
||||
Every System capability of the legacy admin and gallery pages has an owner (#249 System
|
||||
S12, #329). Status is `covered` (a native control replaces it), `kept in legacy` (it stays
|
||||
on the legacy page, which keeps working) or `approved removal` (a ruling retired it). Native
|
||||
control is `` `<section>` · <role> · "<accessible name>" ``, `<section>` a System hash id
|
||||
and `<role>` one of button, combobox, switch, slider, spinbutton, textbox, group, status or
|
||||
term, followed by ` · in "<row title>"` when several rows of the section share the name.
|
||||
Every `covered` row names one, except the rows in `CONTROL_EXEMPTIONS`
|
||||
(`tools/vendor/legacy-coverage.mjs`), each with its reason: today only "Gallery per-image
|
||||
auto-tag / review", which the Images selection bar owns, not a System section. A row that
|
||||
is not `covered` names none (`—`). Test is the exact title of a test that runs, then its
|
||||
file. Anchors are line numbers on `main` at this PR.
|
||||
|
||||
`tests/vendor/legacy-coverage.test.mjs` fails on a row with any of these problems:
|
||||
- no owner, or a status outside the three;
|
||||
- a malformed Native control cell;
|
||||
- a covered row without a control (unless exempt), or a control on a row that is not
|
||||
covered;
|
||||
- a test title its file does not declare: `test(` or `it(` (`.only` and `.slow` count),
|
||||
or a `def test_` at function or method indentation;
|
||||
- a title whose test is only declared with `.skip`, `.fixme` or `.fail`.
|
||||
|
||||
A title that appears only in a comment or a Python docstring does not count. CI running
|
||||
those tests green is what proves them. `every covered legacy System capability has its
|
||||
native control` (workspace-system.spec) opens each covered row's section and finds the
|
||||
control by role and accessible name, inside its row when the cell names one. The shared
|
||||
"Review plan" of the per-operation review row and the Sources "Refresh" of the monitoring
|
||||
list stay unscoped: any row's control proves them.
|
||||
|
||||
| Capability | Legacy anchor | Owner | Status | Native control | Test |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| Image scan directories add/remove | `admin.html:377–390`; `admin.js:200,1273–1323` | S4 Sources | covered | `sources` · button · "+ Add source" | Add source validates on the server, then marks the host draft dirty; Save persists (workspace-system-sources.spec.mjs) |
|
||||
| Currently monitoring list + Refresh | `admin.html:392–400`; `admin.js:1032` | S4 Sources | covered | `sources` · button · "Refresh" | watching only with a live observer; restart pending otherwise (workspace-system-sources.spec.mjs) |
|
||||
| Monitoring enabled | `py/config.py:212,248` | S4 Sources | covered | `sources` · switch · "Watch for new outputs" | Watch for new outputs saves monitoring_enabled; a restart applies it (workspace-system-sources.spec.mjs) |
|
||||
| Search results auto-hide `result_timeout` | `admin.html:407–412` | S3 Appearance | covered | `appearance` · spinbutton · "Node search auto-hide (seconds)" | host fields round-trip through /settings with revision (workspace-system.spec.mjs) |
|
||||
| Web UI display mode | `admin.html:414–422` | S3 Appearance | covered | `appearance` · combobox · "Open from ComfyUI" | host fields round-trip through /settings with revision (workspace-system.spec.mjs) |
|
||||
| Infinite scrolling default | `admin.html:424–430`; `admin.js:689–727` | Legacy prompt list (README continuous browsing); read-only in the report | kept in legacy | — | shouldLoadMore asks for the next page only near the bottom with pages left (list-state.test.js) |
|
||||
| Worker threads | `admin.html:437–445` | S4 Sources | covered | `sources` · slider · "Worker threads" | worker range max is cpu_count (workspace-system-sources.spec.mjs) |
|
||||
| LoraManager badge, enable, path, trigger words, CivitAI key, clear key | `admin.html:448–490`; `admin.js:1052–1156` | S10 Integrations | covered | `integrations` · switch · "LoraManager integration" | LoRA rows show detected / enabled / cache-loaded separately (workspace-system-integrations.spec.mjs) |
|
||||
| LoRA data import with progress | `admin.html:492–498,517–541`; `admin.js:1157` | S10 Integrations | covered | `integrations` · button · "Review import" | Review import shows replacement, blocked items and network effects with the backup box; Start opens the job (D107) (workspace-system-integrations.spec.mjs) |
|
||||
| Statistics | `admin.html:159`; `admin.js:1627` | S2 Overview | covered | `overview` · button · "Explore" | statistics dialog lists runs by category and library totals (workspace-system.spec.mjs) |
|
||||
| Export components JSON `/export` | `admin.html:163`; `admin.js:1607`; `py/api/prompts.py:238` | S9 Backup & export | covered | `backups` · button · "Export all prompts JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Saved pairs export | `py/api/pairs.py:84` | S9 Backup & export | covered | `backups` · button · "Export saved pairs JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Library export | `py/api/library.py:223` | S9 Backup & export | covered | `backups` · button · "Export JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Diagnostics run | `admin.html:171,659`; `admin.js:1984` | S12 Diagnostics | covered | `diagnostics` · button · "Run checks" | Run checks opens the shared Health report (workspace-system-diagnostics.spec.mjs) |
|
||||
| Synthetic image-link test | `admin.html:662`; `admin.js:2101` | S12 Diagnostics | covered | `diagnostics` · button · "Create test link…" | Create test link confirms first and names the prompt (workspace-system-diagnostics.spec.mjs) |
|
||||
| Maintenance cleanup_duplicates, vacuum, cleanup_orphaned_images, prune_orphaned_prompts | `admin.html:692–748`; `admin.js:1834` | S6/S7 Maintenance | covered | `maintenance` · button · "Review plan" · in "Duplicate prompt text" | Apply reviewed operation closes, toasts, opens the job in Activity (workspace-system-maintenance.spec.mjs) |
|
||||
| Maintenance check_hash_duplicates, check_consistency | `admin.html:692–748`; `admin.js:1834` | S2 Health report (Maintenance notice, D123) | covered | `maintenance` · button · "Run health checks" | Run health checks opens Health report and updates metric 4 (workspace-system.spec.mjs) |
|
||||
| Maintenance statistics (database info) | `admin.html:692–748`; `admin.js:1834` | S12 Diagnostics | covered | `diagnostics` · term · "Database" | meta-grid shows real database and source status, the pinned Viewer.js version and the host renderers (workspace-system-diagnostics.spec.mjs) |
|
||||
| Maintenance select all / clear all / run | `admin.html:754–762`; `admin.js:1822–1833` | S7 per-operation review | covered | `maintenance` · button · "Review plan" | plan dialog shows the server's scope, protected, backup policy and media lines with the backup box (D100); Cancel writes nothing (workspace-system-maintenance.spec.mjs) |
|
||||
| Prompt cleanup `/cleanup` | `py/api/admin.py:620` | S6/S7 Maintenance | covered | `maintenance` · button · "Review plan" · in "Duplicate prompt text" | test_duplicate_prompts_plan_counts_the_legacy_groups (test_maintenance_plans.py) |
|
||||
| Database backup | `admin.html:179,824`; `admin.js:2139,2454` | S9 Backup & export | covered | `backups` · button · "Back up" | Back up downloads through the api client and revokes the object URL (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Restore from file | `admin.html:183,616–642`; `admin.js:2177–2248` | S8/S9 Backup & export | covered | `backups` · button · "Review restore…" | Review restore stages the chosen file and shows the real summary with the backup box (D105) (workspace-system-backups.spec.mjs) |
|
||||
| Metadata scan with progress, cancel, pill | `admin.html:187,805–865`; `admin.js:2336–2500` | S4 Sources | covered | `sources` · button · "Review scan" | Review scan opens the shared Review metadata scan with all sources (workspace-system-sources.spec.mjs) |
|
||||
| Rescan-required notice + dismiss | `admin.html:1367–1410`; `admin.js:991–1011` | S4 Sources | covered | `sources` · button · "Later" | notice Later persists the 3.2.0 dismissal (pm_update_notice_dismissed) without scanning (workspace-system-sources.spec.mjs) |
|
||||
| View logs with level and limit | `admin.html:195,885–907,974`; `admin.js:2579–2641` | S12 Logs | covered | `diagnostics` · combobox · "Level" | level filter is minimum-level over all six choices (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log statistics | `admin.html:921–941`; `admin.js:2512` | S12 Logs | covered | `diagnostics` · status · "Log statistics" | log statistics show entries, errors, warnings, files and size (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log files list and download | `admin.html:910,982–985`; `admin.js:2545,2642,2679` | S12 Logs | covered | `diagnostics` · button · "Download log" | each listed file downloads by its returned name and a link has no Download (workspace-system-diagnostics.spec.mjs) |
|
||||
| Clear/truncate logs | `admin.html:915`; `admin.js:2704` | S12 Logs | covered | `diagnostics` · button · "Clear all logs…" | Clear all logs states both counts, confirms first and reads everything again (workspace-system-diagnostics.spec.mjs) |
|
||||
| Rotated-log cleanup (new, #329) | — | S12 Logs | covered | `diagnostics` · button · "Review log cleanup" | Review log cleanup lists only rotated copies and keeps the rest (workspace-system-diagnostics.spec.mjs) |
|
||||
| Set log level (incl. CRITICAL), console logging | `admin.html:948–966`; `admin.js:2530,2729` | S12 Logs | covered | `diagnostics` · combobox · "Runtime log level" | runtime level (including CRITICAL) and console logging save and say they apply until the server restarts (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log file_logging / max_file_size / backup_count / buffer_size | `py/api/logging_routes.py:56,285–299` | S12 Logs (read-only) | covered | `diagnostics` · group · "Log files & retention" | each listed file downloads by its returned name and a link has no Download (workspace-system-diagnostics.spec.mjs) |
|
||||
| Auto-tag model list with status | `admin.html:1095–1148`; `admin.js:3641` | S11 Auto-tagging | covered | `tagging` · combobox · "Tagging model" | model options are the four real engines with size and the description shows live status (workspace-system-tagging.spec.mjs) |
|
||||
| Auto-tag model download | `admin.html:1413–1438`; `admin.js:3680–3747`; `gallery.js:3412` | S11 Auto-tagging | covered | `tagging` · button · "Download" | Download starts the job and Activity shows it without Cancel (workspace-system-tagging.spec.mjs) |
|
||||
| Unload model | `gallery.js:3480` | S11 Auto-tagging | covered | `tagging` · button · "Unload" | Unload is enabled only while a model is loaded, and a held gate's refusal names the operation (workspace-system-tagging.spec.mjs) |
|
||||
| Tag template, WD14 thresholds, already-tagged mode | `admin.html:1150–1198`; `admin.js:3748` | S11 Auto-tagging | covered | `tagging` · combobox · "Existing tags" | defaults save through Save preferences and the Tag image dialog uses them (workspace-system-tagging.spec.mjs) |
|
||||
| Batch auto-tag output images | `admin.html:191,1210,1279–1318`; `admin.js:3748–3840` | S11 Auto-tagging | covered | `tagging` · button · "Review" | Tag output images reviews the images, tags each one and never a prompt (#379) (workspace-system-tagging.spec.mjs) |
|
||||
| Batch review loop | `admin.html:1206,1218–1364`; `admin.js:3841–4172` | S11 + image-tagging plan | covered | `tagging` · button · "Preview tagging review" | Preview tagging review opens the Tag image dialog for the selected image (workspace-system-tagging.spec.mjs) |
|
||||
| Gallery per-image auto-tag / review | `gallery.html:601,605`; `gallery.js:3506,3674–3753` | Image-tagging plan (selection bar) | covered | — | Auto-tag selected with Apply automatically runs a job with the chosen model, thresholds and suggestion count (workspace-tagging-bulk.spec.mjs) |
|
||||
| Gallery thumbnail generation | `gallery.js:1852,1967` | S5/S7 | covered | `sources` · button · "Review options" · in "Thumbnail generation" | Start closes the dialog, toasts, and opens the job in Activity (workspace-system-sources.spec.mjs) |
|
||||
| Clear server thumbnails | `gallery.js:2175–2192` | S7 Maintenance | covered | `maintenance` · button · "Review options" · in "Thumbnail cache" | Thumbnail cache opens the S5 dialog (workspace-system-maintenance.spec.mjs) |
|
||||
| Browser cache clear | `gallery.js:2194–2207` | S7 Maintenance | covered | `maintenance` · button · "Review plan" · in "Metadata cache" | Metadata cache reviews and clears only owned browser entries when there are some (D102) (workspace-system-maintenance.spec.mjs) |
|
||||
| Refresh file listing | `gallery.js:2209` | S4 Sources | covered | `sources` · button · "Refresh" · in "Refresh files" | Refresh files calls /gallery/rescan and toasts its counts (workspace-system-sources.spec.mjs) |
|
||||
| Duplicate image scan, review, delete | `gallery.js:2234–2577` | S6/S7 Maintenance | covered | `maintenance` · button · "Find duplicates" | Find duplicates starts a job; its view links Review groups; Keep radios build a duplicate_files plan (D101) (workspace-system-maintenance.spec.mjs) |
|
||||
| Default view mode, image quality | `gallery-settings.js:26–27` | S3 Appearance | covered | `appearance` · combobox · "Default layout" | defaults apply to a new visit but not to the current view or a saved view (workspace-system.spec.mjs) |
|
||||
| Auto-load metadata, cache metadata, show file paths | `gallery-settings.js:31–33` | S3 Appearance | covered | `appearance` · switch · "Load metadata on selection" | metadata off shows an explicit load and loads once on request (workspace-system.spec.mjs) |
|
||||
| Video autoplay, mute, loop | `gallery-settings.js:40–42` | S3 Appearance | covered | `appearance` · switch · "Autoplay videos" | video defaults reach the viewer (workspace-system.spec.mjs) |
|
||||
| Image info on cards; sidebar collapsed | `gallery-settings.js:30,34` | S3 Appearance | covered | `appearance` · switch · "Show image details on cards" | added rows reach their consumers (workspace-system.spec.mjs) |
|
||||
| Reset gallery settings | `gallery.js:1813` | S3 Appearance | covered | `appearance` · button · "Reset" | reset restores defaults without touching other keys (workspace-system.spec.mjs) |
|
||||
| Lazy loading | `gallery-settings.js:25` | Legacy gallery (native always lazy) | kept in legacy | — | a legacy gallery Save keeps the workspace keys and every other stored key (gallery-settings.test.js) |
|
||||
| Results per page / grid columns / paged mode | `gallery-settings.js:28–29,35` | README continuous browsing | approved removal | — | numeric strings from <select> values are accepted and columns are clamped (gallery-settings.test.js) |
|
||||
| Startup thumbnail prompt | `gallery-settings.js:38–39`; `gallery.js:2578` | README; Overview thumbnail row | approved removal | — | thumbnail row reflects the status endpoint (workspace-system.spec.mjs) |
|
||||
| Client debug output `debugMode` | `gallery-settings.js:36` | Legacy gallery (S12-10) | kept in legacy | — | a legacy gallery Save keeps the workspace keys and every other stored key (gallery-settings.test.js) |
|
||||
| API timeout `apiTimeout` | `gallery-settings.js:37` | Legacy gallery (S12-10; read by no request) | kept in legacy | — | defaults are a complete, sane settings object (gallery-settings.test.js) |
|
||||
| config.json-only keys | `py/config.py:212–234,722–739`; `py/api/diagnostics_report.py:185` | S12 report (read-only) | covered | `diagnostics` · button · "Download diagnostic report" | test_report_lists_config_only_keys (test_api_diagnostics_report.py) |
|
||||
|
||||
### Legacy System UI removed by S13
|
||||
|
||||
System S13 (#330) points the legacy pages' System buttons at native System (a link that
|
||||
opens `workspace#/system/<id>` in a new tab) and removes the legacy copy of a capability
|
||||
only when its row above is `covered`. One row per removed capability: Capability is the
|
||||
exact Capability cell of its coverage row, Page is `admin` or `gallery`, Removed ids and
|
||||
Removed methods list what went (`—` for none), and Native test copies that row's Test
|
||||
cell. An id that stays on a System link is not listed, only its handler.
|
||||
|
||||
`tests/vendor/legacy-coverage.test.mjs` fails when a row cites a capability that is not
|
||||
`covered` or copies another test, when an id or method is listed twice for one page, or
|
||||
when a listed id is still on its page (`id="…"` in the markup or the script, a
|
||||
`getElementById`/`querySelector('#…')` lookup, an entry of a modal-id list) or a listed
|
||||
method is still defined or called (`.name(`). A deletion listed nowhere is caught by the
|
||||
kept-flow tests (`kept admin flows still work` in `legacy-system-links.spec.mjs` and the
|
||||
legacy page workflows), not by this check.
|
||||
|
||||
| Capability | Page | Removed ids | Removed methods | Native test |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Diagnostics run | admin | `#diagnosticsModal`, `#runDiagnosticsBtn`, `#diagnosticsContent` | `showDiagnosticsModal()`, `closeDiagnostics()`, `runDiagnostics()`, `renderDiagnostics()` | Run checks opens the shared Health report (workspace-system-diagnostics.spec.mjs) |
|
||||
| Synthetic image-link test | admin | `#testImageLinkBtn` | `testImageLink()` | Create test link confirms first and names the prompt (workspace-system-diagnostics.spec.mjs) |
|
||||
| View logs with level and limit | admin | `#logsModal`, `#logLevel`, `#logLimit`, `#refreshLogsBtn`, `#logsContainer` | `showLogsModal()`, `closeLogs()`, `refreshLogs()`, `displayLogs()` | level filter is minimum-level over all six choices (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log statistics | admin | `#logStatsTotal`, `#logStatsErrors`, `#logStatsWarnings`, `#logStatsLevel`, `#logStatsSize` | `loadLogStats()`, `formatBytes()` | log statistics show entries, errors, warnings, files and size (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log files list and download | admin | `#downloadLogsBtn`, `#logFilesList` | `loadLogFiles()`, `downloadLogs()`, `downloadLogFile()` | each listed file downloads by its returned name and a link has no Download (workspace-system-diagnostics.spec.mjs) |
|
||||
| Clear/truncate logs | admin | `#clearLogsBtn` | `clearLogs()` | Clear all logs states both counts, confirms first and reads everything again (workspace-system-diagnostics.spec.mjs) |
|
||||
| Set log level (incl. CRITICAL), console logging | admin | `#updateLogConfigBtn`, `#setLogLevel`, `#consoleLogging` | `loadLogConfig()`, `updateLogConfig()` | runtime level (including CRITICAL) and console logging save and say they apply until the server restarts (workspace-system-diagnostics.spec.mjs) |
|
||||
|
||||
### Browser cache ownership (#222)
|
||||
|
||||
The legacy gallery's Clear cache used to delete every localStorage key starting `pm_` or
|
||||
@@ -4039,8 +4547,17 @@ impact and command receipts report `image_count` / `images` /
|
||||
`(size, mtime_ns)` equals its verified fingerprint; an in-place rewrite that
|
||||
preserves both keeps the old set matching until the file is next observed. The
|
||||
Images Tags filter (a file matches when every tag is on its linked positive or
|
||||
its current image tag set) and `q` search stream the tagged records once and
|
||||
stat only the listed files whose records carry a filter tag or a `q` hit; the
|
||||
its current image tag set) and `q` search find their candidates through the tag
|
||||
index (#379), never by streaming every image tag row: `image_tag_ids` reads the
|
||||
tag names that tag at least one image (an `EXISTS` on `idx_media_image_tags_tag`)
|
||||
and keeps a name when it equals a Tags filter tag exactly or contains `q`
|
||||
case-folded; `image_tag_locators` reads the files carrying those ids (`tag_id IN`,
|
||||
500 ids a chunk). Only the listed ones among them are stat'ed, and only their
|
||||
current records count; a linked positive's prompt tags still combine with the
|
||||
image tags under a Tags filter. `test_indexed_candidates_equal_the_full_stream`
|
||||
compares the lookup with the old full stream (kept in the test as the reference)
|
||||
and `test_tag_search_reads_image_tags_only_by_tag_id` traces every statement; the
|
||||
|
||||
inspector's `image_tags` facet (`current` / `none` / `unavailable`) uses the
|
||||
inspection's own stable stat (`tests/test_api_workspace_images.py`
|
||||
`TestWorkspaceImageTags`, `tests/test_api_prompt_inspector.py`). The
|
||||
@@ -4129,7 +4646,23 @@ files, oldest forgotten first, keyed by 16-byte digests: 8.6 MiB measured at
|
||||
the cap). A file whose inspection raises is logged, counted in
|
||||
`review_errors` and reported `unresolved`; the scan goes on. An older version's
|
||||
pending, undecided item of bytes now untraced leaves the queue
|
||||
(`settle_stale_review_items`), so an item never stays stuck at 409. The 16-candidate limited
|
||||
(`settle_stale_review_items`), so an item never stays stuck at 409.
|
||||
Saver anchor (#390, `tests/test_review_saver_anchor.py`): the scan and Apply
|
||||
resolve with the file's name as the workspace inspector does, so the one saver
|
||||
whose `filename_prefix` names the file anchors its branch. A file is never both
|
||||
unlinked and unqueued (maintainer ruling): when only that anchor resolves it
|
||||
(the named inspection run resolves, the unnamed one does not) and neither the
|
||||
default profile nor the parser floor links it, the scan queues it with the
|
||||
anchor's pair as its one candidate (evidence `anchor: "saver"`; the page
|
||||
preselects it), never linking it. Such an item counts as a choice for the
|
||||
untraced settle. Files already linked or resolved by the default path are
|
||||
unchanged. No match, several matches or an unfinished walk anchor nothing, and
|
||||
the file is queued as before. `REVIEW_RULE_VERSION` is folded into inspection
|
||||
item versions, so older inspection items are recorded again once. Apply accepts
|
||||
the anchored pair and refuses a choice the anchor contradicts. Measured on the
|
||||
maintainer's 40,890 outputs: the queue is unchanged (249 files); the 242
|
||||
`gpu_unlock_bench` files the inspector resolves are linked by the parser
|
||||
floor. The 16-candidate limited
|
||||
rule and the 8 MiB bound apply unchanged. The queue heals itself: after each batch, the scan removes the pending
|
||||
items of every listed file (read or skipped as known) that a saved row now
|
||||
links, and the monitor does the same for each ambiguous output it links
|
||||
|
||||
+69
-62
@@ -23,24 +23,20 @@ Features:
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_version():
|
||||
"""Parse version from pyproject.toml"""
|
||||
"""Parse version from pyproject.toml (``utils/version.py``)."""
|
||||
from .utils.version import project_version
|
||||
|
||||
try:
|
||||
pyproject_path = Path(__file__).parent / "pyproject.toml"
|
||||
if pyproject_path.exists():
|
||||
content = pyproject_path.read_text()
|
||||
match = re.search(r'version\s*=\s*["\']([^"\']+)["\']', content)
|
||||
if match:
|
||||
return match.group(1)
|
||||
root = Path(__file__).parent
|
||||
except Exception:
|
||||
pass
|
||||
return "unknown"
|
||||
return "unknown"
|
||||
return project_version(root)
|
||||
|
||||
|
||||
from .prompt_manager import ( # noqa: E402
|
||||
@@ -268,78 +264,89 @@ def _stop_background_work():
|
||||
|
||||
Returns ``{"monitor", "thumbnails", "tracker"}`` booleans (True: it
|
||||
stopped), leaving out parts that never started. Later calls return a
|
||||
copy of the first result.
|
||||
copy of the first result. A Ctrl+C inside a step is logged and the
|
||||
step is False; one between steps is re-raised after the result (parts
|
||||
not reached are False) is stored and the ``atexit`` entry removed.
|
||||
"""
|
||||
global _background_work_result
|
||||
with _background_work_lock:
|
||||
if _background_work_result is not None:
|
||||
return dict(_background_work_result)
|
||||
logger = _teardown_logger()
|
||||
began = _clock()
|
||||
deadline = began + TEARDOWN_SECONDS
|
||||
|
||||
def left(own_bound):
|
||||
return max(0.0, min(own_bound, deadline - _clock()))
|
||||
|
||||
monitor, thumbnails, tracker = (
|
||||
_global_image_monitor,
|
||||
api,
|
||||
_global_prompt_tracker,
|
||||
)
|
||||
result = {}
|
||||
if monitor is not None:
|
||||
# A part whose global is set has started; False until its step says
|
||||
# otherwise, so an interrupt between steps leaves an honest result.
|
||||
started = (
|
||||
("monitor", monitor),
|
||||
("thumbnails", thumbnails),
|
||||
("tracker", tracker),
|
||||
)
|
||||
result = {name: False for name, part in started if part is not None}
|
||||
try:
|
||||
logger = _teardown_logger()
|
||||
began = _clock()
|
||||
deadline = began + TEARDOWN_SECONDS
|
||||
|
||||
def stop_monitor():
|
||||
from .utils.image_monitor import OBSERVER_JOIN_SECONDS
|
||||
def left(own_bound):
|
||||
return max(0.0, min(own_bound, deadline - _clock()))
|
||||
|
||||
# Callbacks first: an image in hand queues no preview into a
|
||||
# closing preview worker. The worker may have read the queue
|
||||
# callback already, so the API refuses new previews too (at
|
||||
# exit, Python 3.12+ refuses to start their worker thread).
|
||||
if thumbnails is not None:
|
||||
thumbnails.refuse_auto_thumbnails()
|
||||
monitor.set_review_recorder(None)
|
||||
monitor.set_thumbnail_queue(None)
|
||||
monitor.set_output_listener(None)
|
||||
return monitor.stop_monitoring(left(OBSERVER_JOIN_SECONDS))
|
||||
if monitor is not None:
|
||||
|
||||
result["monitor"] = _teardown_step(
|
||||
"the image monitor", stop_monitor, logger
|
||||
)
|
||||
if thumbnails is not None:
|
||||
def stop_monitor():
|
||||
from .utils.image_monitor import OBSERVER_JOIN_SECONDS
|
||||
|
||||
def close_previews():
|
||||
from .py.api.thumbnail_auto import CLOSE_WAIT_SECONDS
|
||||
# Callbacks first: an image in hand queues no preview into a
|
||||
# closing preview worker. The worker may have read the queue
|
||||
# callback already, so the API refuses new previews too (at
|
||||
# exit, Python 3.12+ refuses to start their worker thread).
|
||||
if thumbnails is not None:
|
||||
thumbnails.refuse_auto_thumbnails()
|
||||
monitor.set_review_recorder(None)
|
||||
monitor.set_thumbnail_queue(None)
|
||||
monitor.set_output_listener(None)
|
||||
return monitor.stop_monitoring(left(OBSERVER_JOIN_SECONDS))
|
||||
|
||||
return thumbnails.close_auto_thumbnails(
|
||||
final=True, timeout=left(CLOSE_WAIT_SECONDS)
|
||||
result["monitor"] = _teardown_step(
|
||||
"the image monitor", stop_monitor, logger
|
||||
)
|
||||
if thumbnails is not None:
|
||||
|
||||
result["thumbnails"] = _teardown_step(
|
||||
"the preview worker", close_previews, logger
|
||||
)
|
||||
if tracker is not None:
|
||||
def close_previews():
|
||||
from .py.api.thumbnail_auto import CLOSE_WAIT_SECONDS
|
||||
|
||||
def stop_tracker():
|
||||
from .utils.prompt_tracker import TRACKER_JOIN_SECONDS
|
||||
return thumbnails.close_auto_thumbnails(
|
||||
final=True, timeout=left(CLOSE_WAIT_SECONDS)
|
||||
)
|
||||
|
||||
return tracker.shutdown(left(TRACKER_JOIN_SECONDS))
|
||||
result["thumbnails"] = _teardown_step(
|
||||
"the preview worker", close_previews, logger
|
||||
)
|
||||
if tracker is not None:
|
||||
|
||||
result["tracker"] = _teardown_step(
|
||||
"the prompt cleanup thread", stop_tracker, logger
|
||||
)
|
||||
elapsed = _clock() - began
|
||||
failed = [name for name, ok in result.items() if not ok]
|
||||
if failed:
|
||||
logger.warning(
|
||||
f"Background work not stopped in time ({elapsed:.1f} s): "
|
||||
+ ", ".join(failed)
|
||||
)
|
||||
else:
|
||||
parts = " ".join(f"{name}=ok" for name in result)
|
||||
logger.info(f"Background work stopped in {elapsed:.1f} s: {parts}")
|
||||
_background_work_result = result
|
||||
atexit.unregister(_stop_background_work)
|
||||
def stop_tracker():
|
||||
from .utils.prompt_tracker import TRACKER_JOIN_SECONDS
|
||||
|
||||
return tracker.shutdown(left(TRACKER_JOIN_SECONDS))
|
||||
|
||||
result["tracker"] = _teardown_step(
|
||||
"the prompt cleanup thread", stop_tracker, logger
|
||||
)
|
||||
elapsed = _clock() - began
|
||||
failed = [name for name, ok in result.items() if not ok]
|
||||
if failed:
|
||||
logger.warning(
|
||||
f"Background work not stopped in time ({elapsed:.1f} s): "
|
||||
+ ", ".join(failed)
|
||||
)
|
||||
else:
|
||||
parts = " ".join(f"{name}=ok" for name in result)
|
||||
logger.info(f"Background work stopped in {elapsed:.1f} s: {parts}")
|
||||
finally:
|
||||
_background_work_result = result
|
||||
atexit.unregister(_stop_background_work)
|
||||
return dict(result)
|
||||
|
||||
|
||||
|
||||
+135
-45
@@ -19,6 +19,11 @@ links (``PromptDatabase._move_tag_links``).
|
||||
Cheap reads (spec amendment S1) treat a byte record as *current* only while
|
||||
the file's present ``(size, mtime_ns)`` equals the record's verified
|
||||
fingerprint; a file without a current record shows no image tags.
|
||||
|
||||
A write made from a tagging model's result passes ``mark_autotagged`` (#379):
|
||||
the byte record is then marked auto-tagged (``media_image_autotags``) in the
|
||||
same transaction, so a mark never exists without the write it records. Hand
|
||||
tags never mark.
|
||||
"""
|
||||
|
||||
import re
|
||||
@@ -34,6 +39,7 @@ from .annotation_schema import (
|
||||
MAX_LOCATOR_LENGTH,
|
||||
METADATA_TABLE,
|
||||
)
|
||||
from .autotag_schema import AUTOTAG_TABLE
|
||||
from .library_commands import _UUID, LibraryCommandError, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .vocabulary import validate_vocabulary_tag_name
|
||||
@@ -96,12 +102,24 @@ _TAGS_OF = (
|
||||
"JOIN tags t ON t.id=mit.tag_id WHERE mit.byte_id IN ({marks}) "
|
||||
"ORDER BY mit.byte_id,mit.position"
|
||||
)
|
||||
_RECORDS = (
|
||||
"SELECT f.locator,b.id,b.size,b.mtime_ns,t.name " # nosec B608
|
||||
f"FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"LEFT JOIN {IMAGE_TAGS_TABLE} mit ON mit.byte_id=b.id "
|
||||
"LEFT JOIN tags t ON t.id=mit.tag_id "
|
||||
"ORDER BY f.locator,b.first_seen_at,b.id,mit.position"
|
||||
_MARK = f"INSERT OR IGNORE INTO {AUTOTAG_TABLE}(byte_id) VALUES(?)" # nosec B608
|
||||
_MARKED_OF = (
|
||||
f"SELECT byte_id FROM {AUTOTAG_TABLE} WHERE byte_id IN ({{marks}})" # nosec B608
|
||||
)
|
||||
_MARKED_LOCATORS = (
|
||||
f"SELECT DISTINCT f.locator FROM {AUTOTAG_TABLE} m " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.id=m.byte_id JOIN {FILES_TABLE} f ON f.id=b.file_id"
|
||||
)
|
||||
# Tag search (#379): the names that tag at least one image (the tag index
|
||||
# answers the EXISTS), then the locators carrying chosen names (by tag_id).
|
||||
_IMAGE_TAG_NAMES = (
|
||||
"SELECT t.id,t.name FROM tags t WHERE EXISTS " # nosec B608
|
||||
f"(SELECT 1 FROM {IMAGE_TAGS_TABLE} mit WHERE mit.tag_id=t.id)"
|
||||
)
|
||||
_TAG_LOCATORS = (
|
||||
f"SELECT DISTINCT f.locator FROM {IMAGE_TAGS_TABLE} mit " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.id=mit.byte_id "
|
||||
f"JOIN {FILES_TABLE} f ON f.id=b.file_id WHERE mit.tag_id IN ({{marks}})"
|
||||
)
|
||||
|
||||
|
||||
@@ -301,14 +319,17 @@ class AnnotationOperations:
|
||||
expected_epoch: str,
|
||||
expected_tag_revision: int,
|
||||
tags: list,
|
||||
mark_autotagged: bool = False,
|
||||
) -> dict:
|
||||
"""Replace the set of these exact bytes, guarded by epoch and revision.
|
||||
|
||||
Creates the owner and byte record when the bytes are new (an explicit
|
||||
empty set is a real save). The identical set at the current revision,
|
||||
or one behind it (a lost-response retry), is ``replayed`` and writes
|
||||
nothing; any other revision mismatch is ``tag-conflict`` with the
|
||||
current set and no writes.
|
||||
no tag; any other revision mismatch is ``tag-conflict`` with the
|
||||
current set and no writes. ``mark_autotagged`` marks the bytes
|
||||
auto-tagged on a saved or replayed receipt (the Tag image dialog's
|
||||
save after Generate suggestions).
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
@@ -329,6 +350,8 @@ class AnnotationOperations:
|
||||
):
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
if mark_autotagged:
|
||||
conn.execute(_MARK, (byte_id,))
|
||||
return _receipt(epoch, sha256, current, revision, True)
|
||||
if expected_tag_revision != revision:
|
||||
raise AnnotationCommandError(
|
||||
@@ -337,6 +360,8 @@ class AnnotationOperations:
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
revision = self._write_image_tags(conn, byte_id, tags)
|
||||
if mark_autotagged:
|
||||
conn.execute(_MARK, (byte_id,))
|
||||
return _receipt(epoch, sha256, tags, revision, False)
|
||||
|
||||
def add_image_tags(
|
||||
@@ -348,16 +373,22 @@ class AnnotationOperations:
|
||||
mtime_ns: int,
|
||||
expected_epoch: str,
|
||||
tags: list,
|
||||
mark_autotagged: bool = False,
|
||||
) -> dict:
|
||||
"""Union ``tags`` into the set of these exact bytes, keeping stored spelling.
|
||||
|
||||
``added`` counts the names that were new (case-insensitively). A union
|
||||
over ``MAX_IMAGE_TAGS`` is ``tag-limit`` with the current set and no
|
||||
writes; a union that adds nothing to an existing record is ``replayed``.
|
||||
``mark_autotagged`` (a tagging model's result) also marks the bytes
|
||||
auto-tagged, replayed or not; only such a call may pass no tags (the
|
||||
model found none: the image is marked and its set is unchanged).
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
tags = normalize_image_tags(tags)
|
||||
if not tags and not mark_autotagged:
|
||||
raise ValueError("tags must name at least one tag")
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
now = utc_now_iso()
|
||||
@@ -375,6 +406,8 @@ class AnnotationOperations:
|
||||
epoch,
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
if mark_autotagged:
|
||||
conn.execute(_MARK, (byte_id,))
|
||||
if not created and not added:
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
@@ -403,6 +436,41 @@ class AnnotationOperations:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
@staticmethod
|
||||
def _validated_stats(stats: object) -> dict:
|
||||
if not isinstance(stats, dict):
|
||||
raise ValueError(
|
||||
"stats must map locators to (size, mtime_ns) integer pairs"
|
||||
)
|
||||
for locator, fingerprint in stats.items():
|
||||
_validate_fingerprint(locator, fingerprint)
|
||||
return stats
|
||||
|
||||
@staticmethod
|
||||
def _current_byte_records(conn: sqlite3.Connection, stats: dict) -> dict:
|
||||
"""``{locator: byte_id}`` of each locator's current byte record.
|
||||
|
||||
The one rule every cheap read uses: a record is current while its
|
||||
verified ``(size, mtime_ns)`` equals the file's present fingerprint
|
||||
in ``stats``; the most recently verified wins (ties on ``(verified_at,
|
||||
byte_id)``). A locator without a current record is left out. Lookups
|
||||
go in chunks of 500 locators.
|
||||
"""
|
||||
locators = list(stats)
|
||||
chosen: dict[str, tuple[tuple[str, str], str]] = {}
|
||||
for start in range(0, len(locators), _LOOKUP_CHUNK):
|
||||
chunk = locators[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
|
||||
_FINGERPRINTS.format(marks=marks), chunk
|
||||
):
|
||||
if (size, mtime_ns) != stats[locator]:
|
||||
continue
|
||||
key = (verified_at, byte_id)
|
||||
if locator not in chosen or key > chosen[locator][0]:
|
||||
chosen[locator] = (key, byte_id)
|
||||
return {locator: byte_id for locator, (_key, byte_id) in chosen.items()}
|
||||
|
||||
def current_image_tags(self, stats: dict) -> dict:
|
||||
"""Image tags of the files whose current ``(size, mtime_ns)`` is verified.
|
||||
|
||||
@@ -411,29 +479,11 @@ class AnnotationOperations:
|
||||
exactly that verified fingerprint; the most recently verified record
|
||||
wins when several do. Lookups go in chunks of 500 locators.
|
||||
"""
|
||||
if not isinstance(stats, dict):
|
||||
raise ValueError(
|
||||
"stats must map locators to (size, mtime_ns) integer pairs"
|
||||
)
|
||||
for locator, fingerprint in stats.items():
|
||||
_validate_fingerprint(locator, fingerprint)
|
||||
locators = list(stats)
|
||||
chosen: dict[str, tuple[tuple[str, str], str]] = {}
|
||||
result: dict[str, list[str]] = {}
|
||||
stats = self._validated_stats(stats)
|
||||
with self._annotation_read() as conn:
|
||||
for start in range(0, len(locators), _LOOKUP_CHUNK):
|
||||
chunk = locators[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
|
||||
_FINGERPRINTS.format(marks=marks), chunk
|
||||
):
|
||||
if (size, mtime_ns) != stats[locator]:
|
||||
continue
|
||||
key = (verified_at, byte_id)
|
||||
if locator not in chosen or key > chosen[locator][0]:
|
||||
chosen[locator] = (key, byte_id)
|
||||
owners = {byte_id: locator for locator, (_key, byte_id) in chosen.items()}
|
||||
result = {locator: [] for locator in chosen}
|
||||
current = self._current_byte_records(conn, stats)
|
||||
owners = {byte_id: locator for locator, byte_id in current.items()}
|
||||
result: dict[str, list[str]] = {locator: [] for locator in current}
|
||||
byte_ids = list(owners)
|
||||
for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
|
||||
chunk = byte_ids[start : start + _LOOKUP_CHUNK]
|
||||
@@ -442,20 +492,60 @@ class AnnotationOperations:
|
||||
result[owners[byte_id]].append(name)
|
||||
return result
|
||||
|
||||
def iter_image_tag_records(self) -> Iterator[tuple[str, int, int, list[str]]]:
|
||||
"""Stream every byte record as ``(locator, size, mtime_ns, tags)``.
|
||||
def autotag_state(self, stats: dict) -> dict:
|
||||
"""``{locator: bool}``: is each file's current byte record auto-tagged?
|
||||
|
||||
One SELECT ordered by locator, then by the bytes' first sighting, with
|
||||
each set's names in position order.
|
||||
``stats`` as for :meth:`current_image_tags`, whose rule picks the
|
||||
current record; a locator without one is False. One read snapshot.
|
||||
"""
|
||||
conn = self.model.get_connection()
|
||||
current: tuple | None = None
|
||||
for locator, byte_id, size, mtime_ns, name in conn.execute(_RECORDS):
|
||||
if current is None or current[0] != byte_id:
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
current = (byte_id, (locator, size, mtime_ns, []))
|
||||
if name is not None:
|
||||
current[1][3].append(name)
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
stats = self._validated_stats(stats)
|
||||
with self._annotation_read() as conn:
|
||||
current = self._current_byte_records(conn, stats)
|
||||
byte_ids = list(dict.fromkeys(current.values()))
|
||||
marked: set[str] = set()
|
||||
for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
|
||||
chunk = byte_ids[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
marked.update(
|
||||
row[0]
|
||||
for row in conn.execute(_MARKED_OF.format(marks=marks), chunk)
|
||||
)
|
||||
return {locator: current.get(locator) in marked for locator in stats}
|
||||
|
||||
def autotagged_locators(self) -> set:
|
||||
"""Locators with at least one auto-tagged byte record, current or not.
|
||||
|
||||
No fingerprint is checked (the Tag output images review's estimate).
|
||||
"""
|
||||
with self._annotation_read() as conn:
|
||||
return {row[0] for row in conn.execute(_MARKED_LOCATORS)}
|
||||
|
||||
def image_tag_ids(self, wanted, query: str) -> list[int]:
|
||||
"""Ids of the tag names on at least one image that a search wants.
|
||||
|
||||
A name counts when it is in ``wanted`` exactly (the Tags filter) or
|
||||
contains ``query`` case-folded (the text search; ``query`` is
|
||||
already folded). One read of the names that tag an image.
|
||||
"""
|
||||
wanted = set(wanted)
|
||||
with self._annotation_read() as conn:
|
||||
return [
|
||||
tag_id
|
||||
for tag_id, name in conn.execute(_IMAGE_TAG_NAMES)
|
||||
if name in wanted or (query and query in name.casefold())
|
||||
]
|
||||
|
||||
def image_tag_locators(self, tag_ids) -> set[str]:
|
||||
"""Locators with any byte record (current or not) carrying one of
|
||||
``tag_ids``, through the tag index, in chunks of 500 ids."""
|
||||
ids = list(dict.fromkeys(tag_ids))
|
||||
found: set[str] = set()
|
||||
with self._annotation_read() as conn:
|
||||
for start in range(0, len(ids), _LOOKUP_CHUNK):
|
||||
chunk = ids[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
found.update(
|
||||
row[0]
|
||||
for row in conn.execute(_TAG_LOCATORS.format(marks=marks), chunk)
|
||||
)
|
||||
return found
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Auto-tagged marks (#379): which image bytes a tagging model has handled.
|
||||
|
||||
One table, ``media_image_autotags``: one row per byte record of the image
|
||||
annotation group (``media_annotation_bytes``) means "a tagging model looked at
|
||||
these exact bytes and its result was stored", or that the bytes were found to
|
||||
be untaggable. "Skip already tagged" skips an image whose current byte record
|
||||
is marked; hand tags never mark. Rows are written only by the image tag store
|
||||
(``database/annotations.py``), in the transaction of the tag write they
|
||||
record.
|
||||
|
||||
The group installs like ``image_favorites``: optional in old backups,
|
||||
required once a database is current, and never backfilled. Like the Needs
|
||||
review group it is valid only beside a valid image annotation group, because
|
||||
every mark cites one byte record (``ON DELETE CASCADE``). Validation reads
|
||||
only: the catalog compared after whitespace compaction, the columns and the
|
||||
foreign key through the PRAGMAs, then one bounded value pass and the foreign
|
||||
key check.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
from .annotation_schema import BYTES_TABLE
|
||||
from .annotation_schema import TABLES as ANNOTATION_TABLES
|
||||
from .annotation_schema import _catalog as _annotation_catalog
|
||||
from .annotation_schema import _uuid, annotation_schema_error
|
||||
|
||||
AUTOTAG_TABLE = "media_image_autotags"
|
||||
_DDL = (
|
||||
f"CREATE TABLE {AUTOTAG_TABLE} (byte_id TEXT PRIMARY KEY "
|
||||
f"REFERENCES {BYTES_TABLE}(id) ON DELETE CASCADE) WITHOUT ROWID"
|
||||
)
|
||||
# (cid, name, declared type, not null, default, primary key position): the
|
||||
# key of a WITHOUT ROWID table is implicitly NOT NULL.
|
||||
_COLUMNS = [(0, "byte_id", "TEXT", 1, None, 1)]
|
||||
# (column, parent table, parent column, on delete)
|
||||
_FOREIGN_KEYS = {("byte_id", BYTES_TABLE, "id", "CASCADE")}
|
||||
|
||||
NEEDS_ANNOTATIONS = "Image auto-tag marks need the image annotation tables"
|
||||
INVALID_SCHEMA = "Image auto-tag marks have an invalid schema"
|
||||
INVALID_VALUES = "Image auto-tag marks contain invalid values"
|
||||
MISSING = "Missing image auto-tag marks schema"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql).lower()
|
||||
|
||||
|
||||
def _catalog(conn: sqlite3.Connection) -> list:
|
||||
return conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name=? OR tbl_name=?",
|
||||
(AUTOTAG_TABLE, AUTOTAG_TABLE),
|
||||
).fetchall()
|
||||
|
||||
|
||||
def autotag_without_annotations_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
"""Catalog only: the marks table present without its annotation tables.
|
||||
|
||||
Startup asks this before the annotation group installs, which would make
|
||||
a marks table copied without it look valid. It reads no rows.
|
||||
"""
|
||||
if not _catalog(conn):
|
||||
return None
|
||||
present = {
|
||||
name for kind, name, _sql in _annotation_catalog(conn) if kind == "table"
|
||||
}
|
||||
if present != set(ANNOTATION_TABLES):
|
||||
return NEEDS_ANNOTATIONS
|
||||
return None
|
||||
|
||||
|
||||
def autotag_schema_error(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
allow_missing: bool = True,
|
||||
annotations_valid: bool = False,
|
||||
) -> Optional[str]:
|
||||
"""Reject an unsupported marks table or invalid marks; reads only.
|
||||
|
||||
A database without the table is acceptable for backups that predate it
|
||||
unless ``allow_missing`` is false. A present table requires a present,
|
||||
valid image annotation group; ``annotations_valid`` says the caller has
|
||||
just validated it (startup's annotation install), so only its presence is
|
||||
checked again.
|
||||
"""
|
||||
objects = _catalog(conn)
|
||||
if not objects:
|
||||
return None if allow_missing else MISSING
|
||||
if annotations_valid:
|
||||
if autotag_without_annotations_error(conn):
|
||||
return NEEDS_ANNOTATIONS
|
||||
elif annotation_schema_error(conn, allow_missing=False):
|
||||
return NEEDS_ANNOTATIONS
|
||||
actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
|
||||
if actual != {("table", AUTOTAG_TABLE, _compact(_DDL))}:
|
||||
return INVALID_SCHEMA
|
||||
columns = [
|
||||
(row[0], row[1], row[2].upper(), row[3], row[4], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({AUTOTAG_TABLE})")
|
||||
]
|
||||
if columns != _COLUMNS:
|
||||
return INVALID_SCHEMA
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({AUTOTAG_TABLE})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS:
|
||||
return INVALID_SCHEMA
|
||||
# Only this module's constants are interpolated into the SQL below.
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {AUTOTAG_TABLE} " # nosec B608
|
||||
f"WHERE NOT ({_uuid('byte_id')}) LIMIT 1"
|
||||
).fetchone():
|
||||
return INVALID_VALUES
|
||||
if conn.execute(f"PRAGMA foreign_key_check({AUTOTAG_TABLE})").fetchone():
|
||||
return INVALID_VALUES
|
||||
return None
|
||||
|
||||
|
||||
def migrate_autotag_state(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table atomically; validate before and after creating it.
|
||||
|
||||
Runs after ``migrate_annotation_state`` (which has just validated the
|
||||
annotation group). Creates one empty table: no row is read or copied.
|
||||
"""
|
||||
conn.execute("SAVEPOINT media_autotag_schema")
|
||||
try:
|
||||
error = autotag_schema_error(conn, annotations_valid=True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
error = autotag_schema_error(conn, allow_missing=False, annotations_valid=True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE media_autotag_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO media_autotag_schema")
|
||||
conn.execute("RELEASE media_autotag_schema")
|
||||
raise
|
||||
@@ -23,6 +23,11 @@ from . import restore_stage
|
||||
from .library_schema import library_schema_error, migrate_library_state
|
||||
from .image_favorites import image_favorites_schema_error, migrate_image_favorites
|
||||
from .annotation_schema import annotation_schema_error, migrate_annotation_state
|
||||
from .autotag_schema import (
|
||||
autotag_schema_error,
|
||||
autotag_without_annotations_error,
|
||||
migrate_autotag_state,
|
||||
)
|
||||
from .review_schema import (
|
||||
migrate_review_state,
|
||||
review_schema_error,
|
||||
@@ -338,8 +343,13 @@ class PromptModel:
|
||||
review_error = review_without_annotations_error(conn)
|
||||
if review_error:
|
||||
raise sqlite3.DatabaseError(review_error)
|
||||
# The auto-tag marks cite annotation bytes the same way (#379).
|
||||
autotag_error = autotag_without_annotations_error(conn)
|
||||
if autotag_error:
|
||||
raise sqlite3.DatabaseError(autotag_error)
|
||||
migrate_annotation_state(conn)
|
||||
migrate_review_state(conn)
|
||||
migrate_autotag_state(conn)
|
||||
# Last: the older generated_images rebuilds run only before file_path.
|
||||
migrate_workflow_documents(conn)
|
||||
|
||||
@@ -1225,6 +1235,11 @@ class PromptModel:
|
||||
)
|
||||
if review_error:
|
||||
return False, review_error
|
||||
autotag_error = autotag_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if autotag_error:
|
||||
return False, autotag_error
|
||||
documents_error = workflow_documents_schema_error(
|
||||
conn, allow_missing=not require_library_state, documents=documents
|
||||
)
|
||||
|
||||
@@ -1808,37 +1808,6 @@ class PromptDatabase(
|
||||
reader = DocumentReader(conn)
|
||||
return [self._image_row_to_dict(row, reader) for row in cursor]
|
||||
|
||||
def autotag_batch_candidates(self) -> List[Dict[str, Any]]:
|
||||
"""The images a Tag output images batch looks at (#328).
|
||||
|
||||
``{id, image_path, prompt_id, prompt_tags}`` for every image with a
|
||||
stored path that is linked to a prompt, newest first (the
|
||||
``get_all_images`` order, ties newest row first). Never reads
|
||||
``workflow_data`` or the other documents: a batch needs none of them.
|
||||
"""
|
||||
sql = (
|
||||
"SELECT gi.id, gi.image_path, gi.prompt_id, "
|
||||
"(SELECT GROUP_CONCAT(t.name, '|||') FROM prompt_tags pt "
|
||||
"JOIN tags t ON pt.tag_id = t.id WHERE pt.prompt_id = gi.prompt_id) "
|
||||
"AS prompt_tags "
|
||||
"FROM generated_images gi "
|
||||
"INNER JOIN prompts p ON gi.prompt_id = p.id "
|
||||
"WHERE gi.image_path IS NOT NULL AND gi.image_path != '' "
|
||||
"ORDER BY gi.generation_time DESC, gi.id DESC"
|
||||
)
|
||||
with self.model.get_connection() as conn:
|
||||
return [
|
||||
{
|
||||
"id": row["id"],
|
||||
"image_path": row["image_path"],
|
||||
"prompt_id": row["prompt_id"],
|
||||
"prompt_tags": (
|
||||
row["prompt_tags"].split("|||") if row["prompt_tags"] else []
|
||||
),
|
||||
}
|
||||
for row in conn.execute(sql)
|
||||
]
|
||||
|
||||
def get_all_images(
|
||||
self,
|
||||
limit: int = 0,
|
||||
|
||||
@@ -229,13 +229,15 @@ def evidence_offers_choice(evidence: Any) -> bool:
|
||||
"""
|
||||
if not isinstance(evidence, dict):
|
||||
return False
|
||||
# The saver anchor's one pair (#390) is a suggestion to confirm.
|
||||
needed = 1 if evidence.get("anchor") == "saver" else MIN_CHOOSABLE
|
||||
if evidence.get("limited"):
|
||||
total = evidence.get("candidate_total")
|
||||
return type(total) is int and total >= MIN_CHOOSABLE
|
||||
return type(total) is int and total >= needed
|
||||
candidates = evidence.get("candidates")
|
||||
if not isinstance(candidates, list):
|
||||
return False
|
||||
return sum(1 for c in candidates if choosable(c)) >= MIN_CHOOSABLE
|
||||
return sum(1 for c in candidates if choosable(c)) >= needed
|
||||
|
||||
|
||||
def evidence_digest(text: str) -> str:
|
||||
|
||||
+14
-193
@@ -5,6 +5,8 @@ Split into domain-specific mixins for maintainability:
|
||||
- ImageRoutesMixin: gallery, thumbnails, image serving and linking
|
||||
- AdminRoutesMixin: duplicates, stats, settings, diagnostics, maintenance, backup, scan
|
||||
- LoggingRoutesMixin: log management endpoints
|
||||
- LogCleanupMixin: the reviewed rotated-log cleanup (a maintenance plan)
|
||||
- DiagnosticsReportMixin: the shareable diagnostic report, without secrets or paths
|
||||
- AutotagRoutesMixin: auto-tagging model management and tagging
|
||||
- JobRoutesMixin: the session job registry behind Activity (py/api/jobs.py)
|
||||
- ImageTagJobsMixin: bulk Add tags / Auto-tag as image-tags jobs
|
||||
@@ -65,6 +67,8 @@ from .duplicate_files import DuplicateFilesMixin
|
||||
from .maintenance_plans import PlanStore
|
||||
from .jobs import JobRegistry
|
||||
from .logging_routes import LoggingRoutesMixin
|
||||
from .log_cleanup import LogCleanupMixin
|
||||
from .diagnostics_report import DiagnosticsReportMixin
|
||||
from .autotag_routes import AutotagRoutesMixin
|
||||
from .lora_integration import LoraIntegrationMixin
|
||||
from .lora_import_routes import LoraImportRoutesMixin
|
||||
@@ -291,6 +295,10 @@ async def _gzip_middleware(request, handler):
|
||||
if len(compressed) >= len(body):
|
||||
return response
|
||||
|
||||
# A length the handler declared is the uncompressed one: drop it so the
|
||||
# compressed body's own length is sent (#329: a stale length made the
|
||||
# browser wait for bytes that never came).
|
||||
response.headers.pop("Content-Length", None)
|
||||
response.body = compressed
|
||||
response.headers["Content-Encoding"] = "gzip"
|
||||
response.headers["Vary"] = "Accept-Encoding"
|
||||
@@ -323,6 +331,8 @@ class PromptManagerAPI(
|
||||
WorkflowStorageRoutesMixin,
|
||||
DuplicateFilesMixin,
|
||||
LoggingRoutesMixin,
|
||||
LogCleanupMixin,
|
||||
DiagnosticsReportMixin,
|
||||
AutotagRoutesMixin,
|
||||
LoraIntegrationMixin,
|
||||
LoraImportRoutesMixin,
|
||||
@@ -353,6 +363,9 @@ class PromptManagerAPI(
|
||||
# Private media captures of the annotation routes (py/api/annotations.py),
|
||||
# created on first use with their restore subscription.
|
||||
self._captures = None
|
||||
# Capture tokens Generate suggestions ran on: a save of one marks the
|
||||
# image auto-tagged (#379).
|
||||
self._suggested_tokens = set()
|
||||
self._captures_unsubscribe = None
|
||||
# Set by close_annotation_captures (shutdown): no store is recreated.
|
||||
self._captures_closed = False
|
||||
@@ -678,6 +691,7 @@ class PromptManagerAPI(
|
||||
self._register_settings_routes(routes)
|
||||
self._register_job_routes(routes)
|
||||
self._register_system_routes(routes)
|
||||
self._register_diagnostics_report_routes(routes)
|
||||
self._register_library_location_routes(routes)
|
||||
self._register_maintenance_routes(routes)
|
||||
self._register_backup_restore_routes(routes)
|
||||
@@ -1093,196 +1107,3 @@ class PromptManagerAPI(
|
||||
result["parameters"][field] = metadata[field]
|
||||
|
||||
return result
|
||||
|
||||
def _extract_readable_prompt(self, parsed_data):
|
||||
"""Extract human-readable prompt text from ComfyUI/A1111 data."""
|
||||
|
||||
def safe_to_string(value):
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
elif isinstance(value, list):
|
||||
return " ".join(str(item) for item in value if item)
|
||||
elif value is not None:
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
# First check if we already extracted a positive prompt (A1111 format)
|
||||
if parsed_data.get("positive_prompt"):
|
||||
return parsed_data["positive_prompt"]
|
||||
|
||||
# Check if prompt is already a string
|
||||
if isinstance(parsed_data.get("prompt"), str):
|
||||
return parsed_data["prompt"]
|
||||
|
||||
# Check if prompt is a simple value that can be converted
|
||||
if parsed_data.get("prompt") and not isinstance(
|
||||
parsed_data.get("prompt"), dict
|
||||
):
|
||||
return safe_to_string(parsed_data["prompt"])
|
||||
|
||||
prompt_data = parsed_data.get("prompt")
|
||||
if isinstance(prompt_data, dict):
|
||||
# Use the enhanced logic from parse-metadata.py
|
||||
positive_prompt = self._extract_positive_prompt_from_comfyui_data(
|
||||
prompt_data
|
||||
)
|
||||
if positive_prompt:
|
||||
return positive_prompt
|
||||
|
||||
# Check workflow data if available
|
||||
workflow_data = parsed_data.get("workflow")
|
||||
if isinstance(workflow_data, dict):
|
||||
positive_prompt = self._extract_positive_prompt_from_comfyui_data(
|
||||
workflow_data
|
||||
)
|
||||
if positive_prompt:
|
||||
return positive_prompt
|
||||
|
||||
# Check parameters for positive prompt
|
||||
if parsed_data.get("parameters", {}).get("positive"):
|
||||
return safe_to_string(parsed_data["parameters"]["positive"])
|
||||
|
||||
return None
|
||||
|
||||
def _get_node_inputs(self, node):
|
||||
"""Safely get inputs from a node, handling both dict and list formats.
|
||||
|
||||
Old format: inputs is a dict with direct key-value pairs
|
||||
inputs = {"text": "my prompt", "seed": 123}
|
||||
|
||||
New format: inputs is a list of connection objects
|
||||
inputs = [
|
||||
{"name": "text", "type": "STRING", "link": null,
|
||||
"widget": {"name": "text"}},
|
||||
{"name": "clip", "type": "CLIP", "link": 11}
|
||||
]
|
||||
"""
|
||||
if not isinstance(node, dict):
|
||||
return {}
|
||||
|
||||
inputs = node.get("inputs", {})
|
||||
|
||||
# If inputs is already a dict, return it
|
||||
if isinstance(inputs, dict):
|
||||
return inputs
|
||||
|
||||
# If inputs is a list, convert to dict format
|
||||
if isinstance(inputs, list):
|
||||
inputs_dict = {}
|
||||
for input_item in inputs:
|
||||
if isinstance(input_item, dict) and "name" in input_item:
|
||||
name = input_item["name"]
|
||||
inputs_dict[name] = input_item
|
||||
return inputs_dict
|
||||
|
||||
return {}
|
||||
|
||||
def _find_text_in_node(self, node):
|
||||
"""Try to find text content in a node using various strategies.
|
||||
|
||||
Handles both old and new workflow formats.
|
||||
"""
|
||||
if not isinstance(node, dict):
|
||||
return None
|
||||
|
||||
# Strategy 1: Check normalized inputs for 'text' field
|
||||
inputs = self._get_node_inputs(node)
|
||||
if "text" in inputs and isinstance(inputs["text"], str):
|
||||
return inputs["text"]
|
||||
|
||||
# Strategy 2: For text encoder nodes, check widgets_values
|
||||
class_type = node.get("class_type", node.get("type", ""))
|
||||
text_encoder_types = [
|
||||
"CLIPTextEncode",
|
||||
"CLIPTextEncodeSDXL",
|
||||
"CLIPTextEncodeSDXLRefiner",
|
||||
"CLIPTextEncodeFlux",
|
||||
"PromptManager",
|
||||
"PromptManagerText",
|
||||
"BNK_CLIPTextEncoder",
|
||||
"Text Encoder",
|
||||
"CLIP Text Encode",
|
||||
]
|
||||
|
||||
if any(
|
||||
encoder_type.lower() in class_type.lower()
|
||||
for encoder_type in text_encoder_types
|
||||
):
|
||||
widgets_values = node.get("widgets_values", [])
|
||||
if widgets_values and len(widgets_values) > 0:
|
||||
if isinstance(widgets_values[0], str) and widgets_values[0].strip():
|
||||
return widgets_values[0]
|
||||
|
||||
return None
|
||||
|
||||
def _extract_positive_prompt_from_comfyui_data(self, data):
|
||||
"""Extract positive prompt from ComfyUI data.
|
||||
|
||||
Handles both old and new formats.
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
|
||||
# Build nodes dictionary
|
||||
nodes_by_id = {}
|
||||
if "nodes" in data:
|
||||
# Handle nodes array format
|
||||
for node in data["nodes"]:
|
||||
if isinstance(node, dict):
|
||||
nid = node.get("id")
|
||||
if nid is not None:
|
||||
nodes_by_id[nid] = node
|
||||
else:
|
||||
# Handle flat dictionary format (node_id -> node_data)
|
||||
for nid_str, node in data.items():
|
||||
try:
|
||||
nid = int(nid_str)
|
||||
except (ValueError, TypeError):
|
||||
nid = nid_str
|
||||
if isinstance(node, dict):
|
||||
if "id" in node:
|
||||
nid = node["id"]
|
||||
nodes_by_id[nid] = node
|
||||
|
||||
if not nodes_by_id:
|
||||
return None
|
||||
|
||||
# First, try to find positive/negative connection pattern
|
||||
pos_id = None
|
||||
for node in nodes_by_id.values():
|
||||
if isinstance(node, dict):
|
||||
inputs = self._get_node_inputs(node)
|
||||
if "positive" in inputs and "negative" in inputs:
|
||||
try:
|
||||
pos_input = inputs["positive"]
|
||||
if isinstance(pos_input, list) and len(pos_input) > 0:
|
||||
pos_id = int(pos_input[0])
|
||||
break
|
||||
except (ValueError, TypeError, IndexError):
|
||||
continue
|
||||
|
||||
# Get text from the positive node
|
||||
if pos_id is not None and pos_id in nodes_by_id:
|
||||
text_val = self._find_text_in_node(nodes_by_id[pos_id])
|
||||
if text_val:
|
||||
return text_val
|
||||
|
||||
# Fallback: find any text encoder node with text content
|
||||
text_nodes = []
|
||||
for node in nodes_by_id.values():
|
||||
if isinstance(node, dict):
|
||||
text_val = self._find_text_in_node(node)
|
||||
if text_val:
|
||||
# Try to determine if this is positive or negative
|
||||
node_title = node.get("title", "").lower()
|
||||
if "neg" not in node_title and "negative" not in node_title:
|
||||
# Prioritize non-negative prompts
|
||||
text_nodes.insert(0, text_val)
|
||||
else:
|
||||
text_nodes.append(text_val)
|
||||
|
||||
# Return the first positive-looking prompt
|
||||
if text_nodes:
|
||||
return text_nodes[0]
|
||||
|
||||
return None
|
||||
|
||||
+56
-24
@@ -47,7 +47,7 @@ try:
|
||||
extract_generation_params,
|
||||
parse_page_json,
|
||||
)
|
||||
from ...utils.prompt_pair_resolver import resolve_prompt_pair
|
||||
from ...utils.prompt_pair_resolver import resolve_prompt_pair, sampler_less_prompt
|
||||
from ...utils.review_capture import reviewable
|
||||
except ImportError:
|
||||
from database.pair_ingestion import blank_positive, ingest_resolved_pair
|
||||
@@ -62,7 +62,7 @@ except ImportError:
|
||||
extract_generation_params,
|
||||
parse_page_json,
|
||||
)
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair, sampler_less_prompt
|
||||
from utils.review_capture import reviewable
|
||||
|
||||
|
||||
@@ -489,9 +489,29 @@ def _diagnose_dependencies():
|
||||
}
|
||||
|
||||
|
||||
def _diagnose_output_dirs(configured_dirs):
|
||||
def legacy_checks(db_path, public_path=None):
|
||||
"""The four checks of ``GET /diagnostics`` (blocking).
|
||||
|
||||
Shared with the diagnostic report (``py/api/diagnostics_report.py``),
|
||||
which passes its own ``public_path`` so a folder outside ComfyUI is never
|
||||
named.
|
||||
"""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
return {
|
||||
"database": _diagnose_database(db_path),
|
||||
"dependencies": _diagnose_dependencies(),
|
||||
"comfyui_output": _diagnose_output_dirs(
|
||||
GalleryConfig.MONITORING_DIRECTORIES, public_path
|
||||
),
|
||||
"image_monitor": _diagnose_image_monitor(public_path),
|
||||
}
|
||||
|
||||
|
||||
def _diagnose_output_dirs(configured_dirs, public_path=None):
|
||||
"""Configured gallery roots, ComfyUI's output dir, or relative fallbacks."""
|
||||
_public_path, _ = _public_helpers()
|
||||
_public_path = public_path or _public_path
|
||||
candidates = list(configured_dirs)
|
||||
try:
|
||||
import folder_paths
|
||||
@@ -525,9 +545,10 @@ def _diagnose_output_dirs(configured_dirs):
|
||||
}
|
||||
|
||||
|
||||
def _diagnose_image_monitor():
|
||||
def _diagnose_image_monitor(public_path=None):
|
||||
"""Status of the running image monitor, with public directory names."""
|
||||
_public_path, _public_error = _public_helpers()
|
||||
_public_path = public_path or _public_path
|
||||
try:
|
||||
monitor = _current_image_monitor()
|
||||
if monitor is None:
|
||||
@@ -1055,16 +1076,7 @@ class AdminRoutesMixin:
|
||||
async def run_diagnostics(self, request):
|
||||
"""Run comprehensive system diagnostics and health checks."""
|
||||
try:
|
||||
from ..config import GalleryConfig
|
||||
|
||||
results = {
|
||||
"database": _diagnose_database(self.db.model.db_path),
|
||||
"dependencies": _diagnose_dependencies(),
|
||||
"comfyui_output": _diagnose_output_dirs(
|
||||
GalleryConfig.MONITORING_DIRECTORIES
|
||||
),
|
||||
"image_monitor": _diagnose_image_monitor(),
|
||||
}
|
||||
results = legacy_checks(self.db.model.db_path)
|
||||
return web.json_response({"success": True, "diagnostics": results})
|
||||
|
||||
except Exception as e:
|
||||
@@ -1618,7 +1630,14 @@ class AdminRoutesMixin:
|
||||
if self._review_file_linked(media_file, roots):
|
||||
outcome = "unchanged" # already linked: not a review case
|
||||
else:
|
||||
self._record_scanned_review(media_file, roots, review)
|
||||
# The records name the item versions that count; the
|
||||
# saver anchor (#390) is checked on the captured bytes.
|
||||
self._record_scanned_review(
|
||||
media_file,
|
||||
roots,
|
||||
review,
|
||||
scanned=(records.get("canonical"), None),
|
||||
)
|
||||
elif untraced and outcome == "unresolved":
|
||||
# Untraced by the default profile: the inspection profile may
|
||||
# still offer a choice (queued for review, never linked).
|
||||
@@ -1679,9 +1698,9 @@ class AdminRoutesMixin:
|
||||
``"ambiguous"`` (nothing linked it) is a Needs review case. An
|
||||
ambiguous result with nothing to choose (``reviewable`` is False: fewer
|
||||
than two candidates carry a prompt) that nothing links returns
|
||||
``"unresolved"`` instead, and ``media_file`` goes to ``untraced``;
|
||||
its parsed records go to ``records["canonical"]`` when ``records`` is
|
||||
a dict.
|
||||
``"unresolved"`` instead, and ``media_file`` goes to ``untraced``.
|
||||
The parsed records of either ambiguous file go to
|
||||
``records["canonical"]`` when ``records`` is a dict.
|
||||
"""
|
||||
if not metadata:
|
||||
# No prompt metadata (--disable-metadata, bare savers): only the
|
||||
@@ -1711,6 +1730,8 @@ class AdminRoutesMixin:
|
||||
records["canonical"] = canonical
|
||||
elif resolution["state"] == "ambiguous" and ambiguous is not None:
|
||||
ambiguous.append(media_file)
|
||||
if records is not None:
|
||||
records["canonical"] = canonical
|
||||
if resolution["state"] == "resolved":
|
||||
result = ingest_resolved_pair(
|
||||
self.db, str(media_file), canonical, resolution
|
||||
@@ -1742,13 +1763,24 @@ class AdminRoutesMixin:
|
||||
canonical.get("parameters"), str
|
||||
):
|
||||
return resolution["state"]
|
||||
elif canonical.get("prompt"):
|
||||
# No node takes a positive or negative input (a guider or wrapper
|
||||
# graph): the one wired text, never a guess (#252).
|
||||
found = sampler_less_prompt(canonical["prompt"])
|
||||
prompt_text = found["text"]
|
||||
if prompt_text is None and isinstance(canonical.get("parameters"), str):
|
||||
# A metadata saver also wrote A1111 text: its positive prompt.
|
||||
parsed = self._parse_comfyui_prompt(metadata)["positive_prompt"]
|
||||
prompt_text = parsed.strip() if parsed and parsed.strip() else None
|
||||
if prompt_text is None:
|
||||
linked = self._link_recorded_run(
|
||||
media_file, canonical, detailed, history
|
||||
)
|
||||
if linked is not None:
|
||||
return linked
|
||||
return "unresolved" if found["state"] == "ambiguous" else "found"
|
||||
else:
|
||||
parsed = self._parse_comfyui_prompt(metadata)
|
||||
if not (parsed.get("prompt") or parsed.get("parameters")):
|
||||
return None
|
||||
prompt_text = self._extract_readable_prompt(parsed)
|
||||
if prompt_text is not None and not isinstance(prompt_text, str):
|
||||
prompt_text = str(prompt_text)
|
||||
prompt_text = self._parse_comfyui_prompt(metadata)["positive_prompt"]
|
||||
if not (prompt_text and prompt_text.strip()):
|
||||
return "found"
|
||||
prompt_text = prompt_text.strip()
|
||||
|
||||
@@ -296,6 +296,7 @@ class AnnotationRoutesMixin:
|
||||
self._captures_closed = True
|
||||
unsubscribe, self._captures_unsubscribe = self._captures_unsubscribe, None
|
||||
store, self._captures = self._captures, None
|
||||
self._suggested_tokens = set()
|
||||
if unsubscribe is not None:
|
||||
unsubscribe()
|
||||
if store is not None:
|
||||
@@ -515,6 +516,7 @@ class AnnotationRoutesMixin:
|
||||
store = self._captures
|
||||
if store is not None and _TOKEN.fullmatch(token) is not None:
|
||||
store.release(token)
|
||||
self._suggested_tokens.discard(token)
|
||||
return web.json_response(
|
||||
{"success": True}, headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
@@ -564,6 +566,9 @@ class AnnotationRoutesMixin:
|
||||
return _coded("capture-expired", _OTHER_FILE)
|
||||
if facts.sha256 != capture.sha256:
|
||||
return _coded("file-changed")
|
||||
# A save after Generate suggestions ran on this capture marks the
|
||||
# image auto-tagged (#379), so a Skip batch leaves the review alone.
|
||||
suggested = token in self._suggested_tokens
|
||||
try:
|
||||
receipt = await self._run_in_executor(
|
||||
self.db.replace_image_tags,
|
||||
@@ -574,6 +579,7 @@ class AnnotationRoutesMixin:
|
||||
expected_epoch=expected_epoch,
|
||||
expected_tag_revision=revision,
|
||||
tags=tags,
|
||||
mark_autotagged=suggested,
|
||||
)
|
||||
except AnnotationCommandError as exc:
|
||||
current = exc.current or {}
|
||||
@@ -590,10 +596,23 @@ class AnnotationRoutesMixin:
|
||||
except Exception:
|
||||
self.logger.exception("Image tag save failed")
|
||||
return _error(500, "save-failed", "The tags could not be saved.")
|
||||
# Dropped only after a saved or replayed receipt: a retry after a
|
||||
# tag-conflict still marks.
|
||||
self._suggested_tokens.discard(token)
|
||||
return web.json_response(
|
||||
{"success": True, **receipt}, headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
|
||||
def _remember_suggested(self, store, token: str) -> None:
|
||||
"""Note that Generate ran on *token*; forget tokens whose capture is
|
||||
gone. ``expires_in`` does not reset a capture's idle clock."""
|
||||
self._suggested_tokens = {
|
||||
kept
|
||||
for kept in self._suggested_tokens
|
||||
if store.expires_in(kept) is not None
|
||||
}
|
||||
self._suggested_tokens.add(token)
|
||||
|
||||
# -- suggest ---------------------------------------------------------------
|
||||
|
||||
async def suggest_image_tags(self, request: web.Request) -> web.Response:
|
||||
@@ -675,6 +694,7 @@ class AnnotationRoutesMixin:
|
||||
finally:
|
||||
if not started:
|
||||
store.release_use(token)
|
||||
self._remember_suggested(store, token)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
|
||||
+566
-554
File diff suppressed because it is too large
Load Diff
@@ -483,13 +483,59 @@ class BackupRestoreRoutesMixin:
|
||||
self.logger.info(f"Removed {len(removed)} old safety backups")
|
||||
|
||||
async def _publish(self, model, staged):
|
||||
"""Publish ``staged``; on a page-copy failure prune and re-raise."""
|
||||
"""Publish ``staged``; on a page-copy failure prune and re-raise.
|
||||
|
||||
A cancel (server stop) waits for the publishing thread (#360): the
|
||||
caller's cleanup and its registry admission must outlive it. When the
|
||||
copy then succeeded the library was replaced, so it is adopted
|
||||
(``_after_publish``, itself run to its end) before the cancel goes
|
||||
on; a copy that failed is handled as usual, then the cancel goes on.
|
||||
"""
|
||||
future = self._in_executor_task(model.publish_staged, staged)
|
||||
cancelled = await _outlive_cancel(future)
|
||||
try:
|
||||
return await self._run_in_executor(model.publish_staged, staged)
|
||||
backup_path = future.result()
|
||||
except sqlite3.OperationalError as exc:
|
||||
if getattr(exc, "_prompt_manager_restore_backup_created", False) is True:
|
||||
await self._prune_after(model.db_path, failed=True)
|
||||
if cancelled:
|
||||
raise asyncio.CancelledError from exc
|
||||
raise
|
||||
except Exception as exc:
|
||||
if cancelled:
|
||||
raise asyncio.CancelledError from exc
|
||||
raise
|
||||
if cancelled:
|
||||
self.logger.info("The restore finished while the server was stopping")
|
||||
# A second cancel (the loop's own shutdown) must not cut it short.
|
||||
adopted = asyncio.ensure_future(self._after_publish(model.db_path))
|
||||
await _outlive_cancel(adopted)
|
||||
if not adopted.cancelled() and adopted.exception() is not None:
|
||||
self.logger.error(
|
||||
"Could not adopt the restored library",
|
||||
exc_info=adopted.exception(),
|
||||
)
|
||||
raise asyncio.CancelledError
|
||||
return backup_path
|
||||
|
||||
async def _stage_to_end(self, model, temp_path):
|
||||
"""Stage ``temp_path``; a cancel waits for the worker (#360).
|
||||
|
||||
The upload stays until the staging thread is done with it, and a
|
||||
stage it finished after the cancel is removed before the cancel goes
|
||||
on.
|
||||
"""
|
||||
future = self._in_executor_task(model.stage_restore, temp_path)
|
||||
if await _outlive_cancel(future):
|
||||
if not future.cancelled() and future.exception() is None:
|
||||
await self._discard_held([_Orphan(future.result())])
|
||||
raise asyncio.CancelledError
|
||||
return future.result()
|
||||
|
||||
def _in_executor_task(self, func, *args):
|
||||
"""``_run_in_executor(func, *args)`` as a task ``_outlive_cancel`` can
|
||||
wait for (shielded, the executor call is never cancelled)."""
|
||||
return asyncio.ensure_future(self._run_in_executor(func, *args))
|
||||
|
||||
async def _after_publish(self, db_path):
|
||||
"""Everything after the copy replaced the library: best effort, logged.
|
||||
@@ -609,7 +655,7 @@ class BackupRestoreRoutesMixin:
|
||||
{"success": False, "error": REQUESTS_BUSY}, status=409
|
||||
)
|
||||
model = self.db.model
|
||||
staged = await self._run_in_executor(model.stage_restore, temp_path)
|
||||
staged = await self._stage_to_end(model, temp_path)
|
||||
try:
|
||||
backup_path = await self._publish(model, staged)
|
||||
finally:
|
||||
@@ -927,6 +973,30 @@ class BackupRestoreRoutesMixin:
|
||||
job.update(progress=100, status_line=line)
|
||||
|
||||
|
||||
async def _outlive_cancel(future):
|
||||
"""Await ``future`` (an executor call's task) to its end, even through
|
||||
cancels (#360).
|
||||
|
||||
A cancelled task cannot stop the worker thread, so the files it uses and
|
||||
the registry admission it runs under must outlive it: the cancel is held
|
||||
until the worker returned. Returns whether the task was cancelled
|
||||
meanwhile; the caller reads the outcome from ``future`` and re-raises
|
||||
``CancelledError``. A future cancelled itself (an executor shut down) is
|
||||
not a cancel of the task: ``future.result()`` raises it. Never catches
|
||||
``GeneratorExit``: a closed loop still closes the coroutine.
|
||||
"""
|
||||
cancelled = False
|
||||
while not future.done():
|
||||
try:
|
||||
await asyncio.shield(future)
|
||||
except asyncio.CancelledError:
|
||||
if not future.cancelled():
|
||||
cancelled = True
|
||||
except Exception:
|
||||
pass # the caller reads it from the future
|
||||
return cancelled
|
||||
|
||||
|
||||
class _Orphan:
|
||||
"""A stage with no store entry, in the shape ``_discard_held`` takes."""
|
||||
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
"""The shareable diagnostic report (#329 System S12).
|
||||
|
||||
``GET /prompt_manager/system/diagnostics`` answers a report built on the
|
||||
server from an explicit list of fields: the project and Viewer.js versions,
|
||||
Python and the platform, the four checks of the legacy ``GET /diagnostics``
|
||||
(one helper, ``admin.legacy_checks``), the database's public info, source
|
||||
counts, the log configuration and statistics, the last health summary and
|
||||
the configuration keys that only ``config.json`` holds. It never copies
|
||||
``PromptManagerConfig.get_config()``: no path leaf is listed, and the CivitAI
|
||||
key is reported only as ``has_civitai_api_key``.
|
||||
|
||||
Every path in the report goes through ``report_path``: relative to the
|
||||
closest public anchor that is not a filesystem root, else ``"outside
|
||||
ComfyUI"``, never a basename that could be a user name. As a backstop,
|
||||
``scrub`` replaces any string that still looks like an absolute path, holds
|
||||
the saved key or the home folder (with or without its root), and counts each
|
||||
replacement in ``redacted`` (0 for a correct builder).
|
||||
|
||||
The report opens the database, so it takes a registry lease: it is refused
|
||||
(409 ``busy``) while an ``exclusive`` job (a restore) runs, and a restore
|
||||
waits for it. It is built off the event loop.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import ntpath
|
||||
import os
|
||||
import platform
|
||||
import posixpath
|
||||
import re
|
||||
from pathlib import Path, PurePosixPath, PureWindowsPath
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import admin as _admin
|
||||
from .job_routes import busy_response
|
||||
from .jobs import JobBusy
|
||||
from .system import health_summary
|
||||
|
||||
try:
|
||||
from ...utils.version import project_version
|
||||
except ImportError: # loaded as the top-level ``py`` package (tests)
|
||||
from utils.version import project_version
|
||||
|
||||
DIAGNOSTICS_LEASE = "Diagnostics"
|
||||
_ROOT = Path(__file__).resolve().parents[2]
|
||||
VIEWERJS_PROVENANCE = str(_ROOT / "web" / "lib" / "viewerjs" / "provenance.json")
|
||||
OUTSIDE = "outside ComfyUI"
|
||||
REMOVED = "[removed]"
|
||||
# The CivitAI key is searched for only when it is long enough to be a key.
|
||||
MIN_KEY_LENGTH = 8
|
||||
_WINDOWS_ABSOLUTE = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
|
||||
|
||||
def anchored_path(canonical, anchors, home, pathmod=os.path):
|
||||
"""``canonical`` relative to the closest usable anchor, else ``outside ComfyUI``.
|
||||
|
||||
All three are already canonical (``normcase``) in ``pathmod``'s spelling.
|
||||
A filesystem root is never usable, nor is a strict ancestor of ``home``
|
||||
(``/home``, ``C:\\Users``), whose relative paths would start with a user
|
||||
name; so a path inside the home folder needs an anchor at or below it.
|
||||
"""
|
||||
flavor = PureWindowsPath if pathmod is ntpath else PurePosixPath
|
||||
target = flavor(canonical)
|
||||
home_path = flavor(home) if home else None
|
||||
usable = []
|
||||
for anchor in map(flavor, anchors):
|
||||
if anchor.parent == anchor:
|
||||
continue
|
||||
if home_path is not None and home_path != anchor:
|
||||
if home_path.is_relative_to(anchor):
|
||||
continue
|
||||
if target.is_relative_to(anchor):
|
||||
usable.append(anchor)
|
||||
if not usable:
|
||||
return OUTSIDE
|
||||
anchor = max(usable, key=lambda a: len(a.parts))
|
||||
return target.relative_to(anchor).as_posix() or "."
|
||||
|
||||
|
||||
def _canonical(path):
|
||||
return os.path.normcase(os.path.realpath(str(path)))
|
||||
|
||||
|
||||
def report_path(path):
|
||||
"""``path`` relative to a public anchor (POSIX style), else ``outside ComfyUI``."""
|
||||
if not path:
|
||||
return ""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
try:
|
||||
canonical = _canonical(path)
|
||||
anchors = [_canonical(anchor) for anchor in GalleryConfig.path_anchors()]
|
||||
home = os.path.expanduser("~")
|
||||
home = _canonical(home) if home and home != "~" else None
|
||||
except (OSError, ValueError):
|
||||
return OUTSIDE
|
||||
return anchored_path(canonical, anchors, home)
|
||||
|
||||
|
||||
def _looks_absolute(text):
|
||||
return (
|
||||
posixpath.isabs(text)
|
||||
or ntpath.isabs(text)
|
||||
or _WINDOWS_ABSOLUTE.match(text) is not None
|
||||
or text.startswith("\\\\")
|
||||
)
|
||||
|
||||
|
||||
# The home folder is matched only as whole path parts, never inside a word.
|
||||
_PART = r"(?<![\w.-]){}(?![\w.-])"
|
||||
MIN_HOME_NAME = 3
|
||||
|
||||
|
||||
def _home_patterns():
|
||||
"""``(pattern, paths_only)`` for the home folder: in full, rootless, its name.
|
||||
|
||||
Accepted residual risk: only this process's own home folder is known, so
|
||||
another user's home (an output folder under /home/<them> while ComfyUI
|
||||
runs as a service account) can still be named by a report path.
|
||||
"""
|
||||
home = os.path.expanduser("~")
|
||||
if not home or home == "~" or len(home) < 4 or home in ("/", "\\"):
|
||||
return ()
|
||||
spellings = {home, home.replace("\\", "/")}
|
||||
relative = os.path.splitdrive(home)[1].strip("\\/")
|
||||
for spelling in (relative, relative.replace("\\", "/")):
|
||||
if "/" in spelling or "\\" in spelling:
|
||||
spellings.add(spelling)
|
||||
patterns = [
|
||||
(re.compile(_PART.format(re.escape(spelling)), re.IGNORECASE), False)
|
||||
for spelling in spellings
|
||||
]
|
||||
name = re.split(r"[\\/]", relative)[-1] if relative else ""
|
||||
if len(name) >= MIN_HOME_NAME:
|
||||
# The bare name only inside a path ("info/x"), not a word ("INFO").
|
||||
patterns.append(
|
||||
(re.compile(_PART.format(re.escape(name)), re.IGNORECASE), True)
|
||||
)
|
||||
return tuple(patterns)
|
||||
|
||||
|
||||
def scrub(report, key):
|
||||
"""``report`` with leaked paths and the key replaced, counted in ``redacted``."""
|
||||
key = key if isinstance(key, str) and len(key) >= MIN_KEY_LENGTH else None
|
||||
homes = _home_patterns()
|
||||
count = 0
|
||||
|
||||
def clean(value):
|
||||
nonlocal count
|
||||
if isinstance(value, str):
|
||||
if (
|
||||
_looks_absolute(value)
|
||||
or (key and key in value)
|
||||
or any(
|
||||
pattern.search(value)
|
||||
for pattern, paths_only in homes
|
||||
if not paths_only or "/" in value or "\\" in value
|
||||
)
|
||||
):
|
||||
count += 1
|
||||
return REMOVED
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return {name: clean(item) for name, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [clean(item) for item in value]
|
||||
return value
|
||||
|
||||
cleaned = clean(report)
|
||||
cleaned["redacted"] = int(cleaned.get("redacted") or 0) + count
|
||||
return cleaned
|
||||
|
||||
|
||||
def viewerjs_version(path=None):
|
||||
"""The pinned Viewer.js version from its provenance file, or ``unknown``."""
|
||||
try:
|
||||
with open(path or VIEWERJS_PROVENANCE, encoding="utf-8") as handle:
|
||||
version = json.load(handle).get("version")
|
||||
except (OSError, ValueError, AttributeError):
|
||||
return "unknown"
|
||||
return version if isinstance(version, str) and version else "unknown"
|
||||
|
||||
|
||||
def public_config():
|
||||
"""The configuration keys only ``config.json`` holds, from an explicit list."""
|
||||
from ..config import GalleryConfig, IntegrationConfig, PromptManagerConfig
|
||||
|
||||
return {
|
||||
"database": {
|
||||
"enable_duplicate_detection": (
|
||||
PromptManagerConfig.ENABLE_DUPLICATE_DETECTION
|
||||
),
|
||||
"enable_auto_save": PromptManagerConfig.ENABLE_AUTO_SAVE,
|
||||
},
|
||||
"web_ui": {
|
||||
"result_timeout": PromptManagerConfig.RESULT_TIMEOUT,
|
||||
"show_test_button": PromptManagerConfig.SHOW_TEST_BUTTON,
|
||||
"webui_display_mode": PromptManagerConfig.WEBUI_DISPLAY_MODE,
|
||||
"infinite_scroll": PromptManagerConfig.INFINITE_SCROLL,
|
||||
},
|
||||
"performance": {
|
||||
"max_search_results": PromptManagerConfig.MAX_SEARCH_RESULTS,
|
||||
"enable_fuzzy_search": PromptManagerConfig.ENABLE_FUZZY_SEARCH,
|
||||
"auto_backup_interval": PromptManagerConfig.AUTO_BACKUP_INTERVAL,
|
||||
"worker_threads": PromptManagerConfig.WORKER_THREADS,
|
||||
},
|
||||
"gallery": {
|
||||
"processing_delay": GalleryConfig.PROCESSING_DELAY,
|
||||
"prompt_timeout": GalleryConfig.PROMPT_TIMEOUT,
|
||||
"cleanup_interval": GalleryConfig.CLEANUP_INTERVAL,
|
||||
"auto_cleanup_missing_files": GalleryConfig.AUTO_CLEANUP_MISSING_FILES,
|
||||
"max_image_age_days": GalleryConfig.MAX_IMAGE_AGE_DAYS,
|
||||
"max_concurrent_processing": GalleryConfig.MAX_CONCURRENT_PROCESSING,
|
||||
"metadata_extraction_timeout": GalleryConfig.METADATA_EXTRACTION_TIMEOUT,
|
||||
},
|
||||
"integrations": {
|
||||
"lora_manager_enabled": bool(IntegrationConfig.LORA_MANAGER_ENABLED),
|
||||
"has_civitai_api_key": bool(IntegrationConfig.CIVITAI_API_KEY),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _source_counts(sources):
|
||||
configured = sources.get("configured") or []
|
||||
effective = sources.get("effective") or []
|
||||
return {
|
||||
"configured": len(configured),
|
||||
"effective": len(effective),
|
||||
"watching": sum(1 for row in effective if row.get("watch") == "watching"),
|
||||
"unavailable": sum(1 for row in configured if row.get("valid") is False),
|
||||
}
|
||||
|
||||
|
||||
class DiagnosticsReportMixin:
|
||||
"""Mixin providing ``GET /prompt_manager/system/diagnostics``."""
|
||||
|
||||
def _register_diagnostics_report_routes(self, routes):
|
||||
@routes.get("/prompt_manager/system/diagnostics")
|
||||
async def diagnostics_report_route(request):
|
||||
return await self.get_diagnostics_report(request)
|
||||
|
||||
async def get_diagnostics_report(self, request):
|
||||
"""The report, built off the loop under a lease (409 ``busy`` in a restore)."""
|
||||
from ..config import IntegrationConfig
|
||||
|
||||
try:
|
||||
async with self._jobs.lease(DIAGNOSTICS_LEASE):
|
||||
report = await self._run_in_executor(self.build_report)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
except Exception:
|
||||
self.logger.exception("The diagnostic report could not be built")
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": None,
|
||||
"error": "The diagnostic report could not be built.",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
report = scrub(report, IntegrationConfig.CIVITAI_API_KEY)
|
||||
return web.json_response(report, headers={"Cache-Control": "no-store"})
|
||||
|
||||
def build_report(self):
|
||||
"""The report's fields (blocking: reads the database and the log folder)."""
|
||||
db = self.db
|
||||
manager = self._get_logger_manager()
|
||||
stats = dict(manager.get_log_stats())
|
||||
stats["log_directory"] = report_path(stats.get("log_directory"))
|
||||
info = dict(db.model.get_database_info() or {})
|
||||
if "database_path" in info:
|
||||
info["database_path"] = report_path(info["database_path"])
|
||||
sources, _monitor = self._source_status()
|
||||
return {
|
||||
"success": True,
|
||||
"generated_at": datetime.datetime.now().replace(microsecond=0).isoformat(),
|
||||
"version": project_version(_ROOT),
|
||||
"python": platform.python_version(),
|
||||
"platform": {"system": platform.system(), "release": platform.release()},
|
||||
"viewerjs": viewerjs_version(VIEWERJS_PROVENANCE),
|
||||
"checks": _admin.legacy_checks(db.model.db_path, report_path),
|
||||
"database_info": info,
|
||||
"sources": _source_counts(sources),
|
||||
"logging": {"config": dict(manager.get_config()), "stats": stats},
|
||||
"health": health_summary(self._current_health_report()),
|
||||
"config": public_config(),
|
||||
"redacted": 0,
|
||||
}
|
||||
+464
-33
@@ -2,7 +2,9 @@
|
||||
|
||||
``POST /prompt_manager/jobs/image-tags`` starts one ``image-tags`` job over
|
||||
1..500 listed images, named by the listing identity ``{root_index,
|
||||
root_scope, path}``. The job is record-changing (it writes image tag sets and
|
||||
root_scope, path}``, or over every image a listing ``filter`` admits (at most
|
||||
10,000, resolved at start in the listing's order; "Select all N matching",
|
||||
#392). The job is record-changing (it writes image tag sets and
|
||||
new vocabulary names), so it never runs beside the metadata scan.
|
||||
|
||||
- **Add tags** (``mode: "add"``) hashes each current file and unions a fixed
|
||||
@@ -18,13 +20,21 @@ the suggestion worker and the storage call ``add_image_tags``. Files are
|
||||
handled one at a time and the job stops at a file boundary when cancelled,
|
||||
keeping what it already wrote. Rows and log lines carry root-relative paths
|
||||
and public root labels only.
|
||||
|
||||
Auto-tag marks each image it ran the model on as auto-tagged (#379), with the
|
||||
union of its tags. The ``autotag`` job ("Tag output images", started by
|
||||
``POST /jobs/autotag``) runs the same per-file loop over every still image the
|
||||
output folders list (:meth:`ImageTagJobsMixin._run_autotag_batch`): image
|
||||
tags only, never a prompt tag or prompt text.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import functools
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import NamedTuple
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -45,10 +55,24 @@ from .annotations import (
|
||||
_submit,
|
||||
_target,
|
||||
)
|
||||
from .autotag_routes import model_busy_response
|
||||
from .images import _public_path
|
||||
from .autotag_routes import BATCH_CANCELLED, MODEL_BUSY_ERROR, model_busy_response
|
||||
from .images import (
|
||||
MAX_LISTING_SELECTION,
|
||||
ListingSourceChanged,
|
||||
_current_fingerprint,
|
||||
_listing_posix,
|
||||
_public_path,
|
||||
parse_listing_filter,
|
||||
)
|
||||
from .job_routes import busy_response
|
||||
from .jobs import IMAGE_TAG_GROUP_NAMES, IMAGE_TAG_JOB_KIND, JobBusy
|
||||
from .jobs import (
|
||||
AUTOTAG_BATCH_JOB_KIND,
|
||||
AUTOTAG_GROUP_NAMES,
|
||||
IMAGE_TAG_GROUP_NAMES,
|
||||
IMAGE_TAG_JOB_KIND,
|
||||
JobBusy,
|
||||
)
|
||||
from .maintenance import disk_plan, short_space_message
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
@@ -75,6 +99,11 @@ except ImportError:
|
||||
from utils.video_metadata import is_video_path
|
||||
|
||||
MAX_JOB_TARGETS = 500
|
||||
# A filter start resolves on the server, so the request body does not bound it.
|
||||
MAX_FILTER_TARGETS = MAX_LISTING_SELECTION
|
||||
NO_IMAGES = "No images match this view."
|
||||
FILTER_LISTING_FAILED = "The matching images could not be listed. Try again."
|
||||
TOO_MANY_IMAGES = "Narrow the view to at most {cap:,} images."
|
||||
# 500 maximal targets plus the options: a path of MAX_TARGET_PATH code points
|
||||
# takes at most 6 bytes each on the wire (a \uXXXX escape; raw UTF-8 is at most
|
||||
# 4), and the root index and scope fit in 256. The annotation routes' 64 KiB
|
||||
@@ -88,9 +117,10 @@ ADD_TITLE = "Add tags to images"
|
||||
AUTO_TITLE = "Auto-tag images"
|
||||
GATE_OWNER_KIND = "image-tag-job"
|
||||
|
||||
# Each mode's fields besides the images, named by ``targets`` or ``filter``.
|
||||
_FIELDS = {
|
||||
"add": frozenset({"mode", "targets", "tags"}),
|
||||
"auto": frozenset({"mode", "targets", "model_type", "options"}),
|
||||
"add": frozenset({"mode", "tags"}),
|
||||
"auto": frozenset({"mode", "model_type", "options"}),
|
||||
}
|
||||
_VERBS = {"add": "Add tags", "auto": "Auto-tag"}
|
||||
_LOG_PREFIXES = {"tagged": "+", "unchanged": "=", "skipped": "-", "errors": "!"}
|
||||
@@ -107,6 +137,14 @@ READ_FAILED = "The file could not be read."
|
||||
SOURCE_CHANGED = "The source changed. Refresh the image collection."
|
||||
STORAGE_RESTORED = "Prompt storage was restored; start the job again."
|
||||
BUSY_RETRY = "The file could not be read now. Run the job again for it."
|
||||
# Tag output images (#379): the per-image batch.
|
||||
ALREADY_AUTOTAGGED = "Already auto-tagged"
|
||||
LISTING_FAILED = "The output folders could not be listed. Try again."
|
||||
LISTING_STATUS = "Listing output images…"
|
||||
_INTERNAL_ERROR = "An internal error occurred. Check server logs for details."
|
||||
# The pre-pass stats and looks up this many targets at a time, checking for a
|
||||
# cancel between chunks.
|
||||
PREPASS_CHUNK = 500
|
||||
|
||||
_monotonic = time.monotonic # the job's clock (a test can fake it)
|
||||
|
||||
@@ -141,6 +179,8 @@ class _Spec:
|
||||
tags: tuple = ()
|
||||
model_type: str | None = None
|
||||
options: dict | None = None
|
||||
# A listing filter the start resolves into ``targets`` (#392).
|
||||
filter: dict | None = None
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
@@ -199,21 +239,35 @@ def parse_image_tag_request(data: dict):
|
||||
"""A :class:`_Spec` for a start body, or ``(None, error response)``.
|
||||
|
||||
``{mode:"add", targets, tags}`` or ``{mode:"auto", targets, model_type,
|
||||
options}`` exactly; options follow the Tag dialog's Generate rules.
|
||||
options}`` exactly; options follow the Tag dialog's Generate rules. A
|
||||
``filter`` (a listing query object, parse_listing_filter) may stand in for
|
||||
``targets``; the start resolves it.
|
||||
"""
|
||||
fields = _FIELDS.get(data.get("mode"))
|
||||
if fields is None:
|
||||
return None, bad_request('mode must be "add" or "auto"')
|
||||
fields = fields | {"filter" if "filter" in data else "targets"}
|
||||
if set(data) != fields:
|
||||
return None, bad_request(
|
||||
"Request must contain exactly " + ", ".join(sorted(fields))
|
||||
)
|
||||
targets, error = _parse_targets(data["targets"])
|
||||
if error is not None:
|
||||
return None, error
|
||||
listing_filter, targets = None, ()
|
||||
if "filter" in data:
|
||||
try:
|
||||
listing_filter = parse_listing_filter(data["filter"])
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
else:
|
||||
targets, error = _parse_targets(data["targets"])
|
||||
if error is not None:
|
||||
return None, error
|
||||
if data["mode"] == "add":
|
||||
return _parse_add(targets, data)
|
||||
return _parse_auto(targets, data)
|
||||
spec, error = _parse_add(targets, data)
|
||||
else:
|
||||
spec, error = _parse_auto(targets, data)
|
||||
if spec is not None and listing_filter is not None:
|
||||
spec = dataclasses.replace(spec, filter=listing_filter)
|
||||
return spec, error
|
||||
|
||||
|
||||
def _capture_outcome(exc: CaptureError, root, relative: str):
|
||||
@@ -246,6 +300,92 @@ async def _capacity_retried(job, attempt):
|
||||
return await attempt()
|
||||
|
||||
|
||||
class Refusal(NamedTuple):
|
||||
"""Why a Tag output images start was refused.
|
||||
|
||||
``response`` is the native route's JSON answer; ``message`` the sentence
|
||||
a legacy stream frame carries.
|
||||
"""
|
||||
|
||||
status: int
|
||||
code: str
|
||||
message: str
|
||||
response: web.Response
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AutotagRun:
|
||||
"""What one Tag output images job runs with (validated at its start)."""
|
||||
|
||||
model_type: str
|
||||
existing: str
|
||||
options: dict
|
||||
keep_in_memory: bool
|
||||
backup_on: bool
|
||||
owner: object
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _BatchTarget:
|
||||
"""One still image of the batch: the job target and its private keys."""
|
||||
|
||||
target: tuple # (root_index, root_scope, relative): the image-tags target
|
||||
path: str # the listed file, for the pre-pass stat (never sent out)
|
||||
locator: str # the image tag key: normcase(join(realpath(root), relative))
|
||||
label: str # the root's public label
|
||||
|
||||
|
||||
def _still_targets(api, index, root, files, seen: set) -> list:
|
||||
"""Worker thread: one root's listed still images as _BatchTargets.
|
||||
|
||||
Newest first (the listing's order). A locator an earlier root already
|
||||
produced (overlapping roots) is dropped; ``seen`` collects them.
|
||||
"""
|
||||
scope = api._gallery_root_scope(root, index)
|
||||
label = _public_path(root)
|
||||
real = os.path.realpath(root)
|
||||
prefix = os.path.join(str(root), "")
|
||||
found = []
|
||||
for path, _mtime in files:
|
||||
relative = _listing_posix(root, prefix, path)
|
||||
if is_video_path(relative):
|
||||
continue
|
||||
locator = os.path.normcase(os.path.join(real, *relative.split("/")))
|
||||
if locator in seen:
|
||||
continue
|
||||
seen.add(locator)
|
||||
found.append(_BatchTarget((index, scope, relative), str(path), locator, label))
|
||||
return found
|
||||
|
||||
|
||||
def _prepass_chunk(db, chunk, skip_marked: bool, max_file: int):
|
||||
"""Worker thread: ``(remaining, skips)`` for a chunk of _BatchTargets.
|
||||
|
||||
Stats each file (no open, no hash): gone is "File not found", over the
|
||||
capture limit "Too large to verify"; under "Skip already tagged" a file
|
||||
whose current bytes are marked is "Already auto-tagged". ``skips`` are
|
||||
``(target, detail)`` pairs in target order.
|
||||
"""
|
||||
stats, skips, alive = {}, [], []
|
||||
for item in chunk:
|
||||
fingerprint = _current_fingerprint(item.path)
|
||||
if fingerprint is None:
|
||||
skips.append((item, FILE_NOT_FOUND))
|
||||
elif fingerprint[0] > max_file:
|
||||
skips.append((item, TOO_LARGE))
|
||||
else:
|
||||
stats[item.locator] = fingerprint
|
||||
alive.append(item)
|
||||
marked = db.autotag_state(stats) if skip_marked and stats else {}
|
||||
remaining = []
|
||||
for item in alive:
|
||||
if marked.get(item.locator):
|
||||
skips.append((item, ALREADY_AUTOTAGGED))
|
||||
else:
|
||||
remaining.append(item)
|
||||
return remaining, skips
|
||||
|
||||
|
||||
def _load_model(service, model_type: str, use_gpu: bool) -> None:
|
||||
"""Worker thread: load (or swap to) *model_type*; never downloads."""
|
||||
if service.is_model_loaded() and service.get_loaded_model_type() == model_type:
|
||||
@@ -254,6 +394,25 @@ def _load_model(service, model_type: str, use_gpu: bool) -> None:
|
||||
raise RuntimeError(f"{model_type} could not be loaded")
|
||||
|
||||
|
||||
def _refusal(status: int, code: str, message: str) -> Refusal:
|
||||
response = web.json_response(
|
||||
{"success": False, "code": code, "error": message}, status=status
|
||||
)
|
||||
return Refusal(status, code, message, response)
|
||||
|
||||
|
||||
def _busy_refusal(busy: JobBusy) -> Refusal:
|
||||
title = getattr(busy.running, "title", "") or "Another job"
|
||||
return Refusal(409, "busy", f"{title} already running", busy_response(busy))
|
||||
|
||||
|
||||
def _batch_cancelled(job) -> str:
|
||||
"""Say the batch was cancelled; its end state (the runner ends the job)."""
|
||||
job.update(status_line=BATCH_CANCELLED)
|
||||
job.log_line(BATCH_CANCELLED)
|
||||
return "cancelled"
|
||||
|
||||
|
||||
class ImageTagJobsMixin:
|
||||
"""The ``image-tags`` job kind: start route and runner."""
|
||||
|
||||
@@ -270,6 +429,10 @@ class ImageTagJobsMixin:
|
||||
spec, error = parse_image_tag_request(data)
|
||||
if error is not None:
|
||||
return error
|
||||
if spec.filter is not None:
|
||||
spec, error = await self._resolve_job_filter(spec)
|
||||
if error is not None:
|
||||
return error
|
||||
roots = await self._run_in_executor(self._job_roots, spec.targets)
|
||||
if any(root is None for root, _label in roots.values()):
|
||||
return _coded("source-changed")
|
||||
@@ -301,6 +464,246 @@ class ImageTagJobsMixin:
|
||||
job.task.add_done_callback(lambda _task: model_gate.release(owner))
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
# -- Tag output images (#379) ---------------------------------------------
|
||||
|
||||
async def _autotag_targets(self) -> list:
|
||||
"""Every still image the output folders list, as _BatchTargets.
|
||||
|
||||
Root by root in configured order, each newest first (the shared
|
||||
listing, ``_get_gallery_files``, used as it is even when stale). Videos
|
||||
are not targets; a file under two overlapping roots is one target.
|
||||
"""
|
||||
seen: set = set()
|
||||
found = []
|
||||
for index, root in enumerate(list(self._get_all_output_dirs())):
|
||||
files = await self._get_gallery_files(root)
|
||||
found.extend(
|
||||
await self._run_in_executor(
|
||||
_still_targets, self, index, root, files, seen
|
||||
)
|
||||
)
|
||||
return found
|
||||
|
||||
async def _start_autotag(
|
||||
self, *, model_type, existing, options, keep_in_memory, title, owner_kind
|
||||
):
|
||||
"""Register a Tag output images job: ``(job, None)`` or ``(None, Refusal)``.
|
||||
|
||||
``options`` are already validated (``validate_options``). The checks
|
||||
run in this order: 409 ``model-unavailable``; 507
|
||||
``insufficient-space`` (only when the safety backup is on); 409
|
||||
``busy`` (another records job); 409 ``model-busy``. No await follows
|
||||
the busy check. The gate is taken before the job registers and
|
||||
released when the task ends, also before its runner ever starts.
|
||||
"""
|
||||
from ..autotag import MODELS, get_autotag_service
|
||||
|
||||
backup_on = self._safety_backup_policy() == "on"
|
||||
status = await self._run_in_executor(get_autotag_service().get_models_status)
|
||||
_disk, short, _size = await self._run_in_executor(
|
||||
functools.partial(
|
||||
disk_plan, self.db.model.db_path, backup=backup_on, vacuum=False
|
||||
)
|
||||
)
|
||||
label = MODELS[model_type]["label"]
|
||||
if not (status.get(model_type) or {}).get("downloaded"):
|
||||
message = (
|
||||
f"{label} is not downloaded. Download it in System › Auto-tagging."
|
||||
)
|
||||
return None, _refusal(409, "model-unavailable", message)
|
||||
if backup_on and short is not None:
|
||||
return None, _refusal(507, "insufficient-space", short_space_message(short))
|
||||
busy = self._jobs.blocker(AUTOTAG_BATCH_JOB_KIND, admission="records")
|
||||
if busy is not None:
|
||||
return None, _busy_refusal(JobBusy(busy))
|
||||
owner = model_gate.new_owner(owner_kind)
|
||||
if not model_gate.try_acquire(owner):
|
||||
response = model_busy_response()
|
||||
return None, Refusal(409, "model-busy", MODEL_BUSY_ERROR, response)
|
||||
run = AutotagRun(
|
||||
model_type, existing, options, keep_in_memory, backup_on, owner
|
||||
)
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
AUTOTAG_BATCH_JOB_KIND,
|
||||
title,
|
||||
admission="records",
|
||||
cancellable=True,
|
||||
scope={"model_type": model_type, "existing": existing, "label": label},
|
||||
groups=AUTOTAG_GROUP_NAMES,
|
||||
runner=functools.partial(self._run_autotag_batch, run),
|
||||
)
|
||||
except BaseException as refused:
|
||||
model_gate.release(owner)
|
||||
if isinstance(refused, JobBusy):
|
||||
return None, _busy_refusal(refused)
|
||||
raise
|
||||
job.task.add_done_callback(lambda _task: model_gate.release(owner))
|
||||
return job, None
|
||||
|
||||
async def _run_autotag_batch(self, run: AutotagRun, job) -> None:
|
||||
"""The ``autotag`` job: backup, model, listing, pre-pass, per-image loop.
|
||||
|
||||
``job.cancelled`` is checked between phases (and per pre-pass chunk
|
||||
and per image). Each phase publishes ``{"type": "phase", "phase"}``.
|
||||
With ``keep_in_memory`` false the model is unloaded under the gate.
|
||||
The job ends only after that cleanup and the gate release, so a job
|
||||
that reads as ended never still holds the gate.
|
||||
"""
|
||||
state = "failed"
|
||||
try:
|
||||
state = await self._autotag_batch_phases(run, job)
|
||||
except Exception:
|
||||
self.logger.exception("Tag output images failed")
|
||||
job.update(status_line=_INTERNAL_ERROR)
|
||||
job.log_line(f"Tagging failed · {_INTERNAL_ERROR}")
|
||||
finally:
|
||||
try:
|
||||
if not run.keep_in_memory:
|
||||
await self._unload_batch_model(run.owner)
|
||||
if run.backup_on:
|
||||
await self._prune_backup_pool(job, self.db)
|
||||
finally:
|
||||
model_gate.release(run.owner)
|
||||
job.publish({"type": state, "progress": job.progress})
|
||||
job.finish(state)
|
||||
|
||||
async def _unload_batch_model(self, owner) -> None:
|
||||
from ..autotag import get_autotag_service
|
||||
|
||||
service = get_autotag_service()
|
||||
try:
|
||||
if service.is_model_loaded():
|
||||
await model_gate.run(owner, service.unload_model)
|
||||
except Exception:
|
||||
self.logger.exception("The tagging model could not be unloaded")
|
||||
|
||||
async def _autotag_batch_phases(self, run: AutotagRun, job) -> str:
|
||||
"""Run the phases; the job's end state (the caller ends the job)."""
|
||||
job.publish({"type": "phase", "phase": "backup"})
|
||||
ok, _backup = await self._safety_backup_step(job, self.db, run.backup_on)
|
||||
if not ok:
|
||||
return "failed"
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
job.publish({"type": "phase", "phase": "model"})
|
||||
job.update(status_line="Loading the tagging model")
|
||||
spec = _Spec("auto", (), model_type=run.model_type, options=run.options)
|
||||
service = await self._prepare_tagging_model(job, spec, run.owner)
|
||||
if service is None:
|
||||
return "failed"
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
job.publish({"type": "phase", "phase": "listing"})
|
||||
job.update(status_line=LISTING_STATUS)
|
||||
epoch = await self._run_in_executor(self.db.get_execution_epoch)
|
||||
try:
|
||||
targets = await self._autotag_targets()
|
||||
except Exception:
|
||||
self.logger.exception("Tag output images could not list the outputs")
|
||||
job.update(status_line=LISTING_FAILED)
|
||||
job.log_line(f"! {LISTING_FAILED}")
|
||||
return "failed"
|
||||
total = len(targets)
|
||||
job.scope["images"] = total
|
||||
job.update(total=total)
|
||||
policy = (
|
||||
"skip images already auto-tagged" if run.existing == "skip" else "merge"
|
||||
)
|
||||
job.log_line(
|
||||
f"Tagging started · {job.scope.get('label') or run.model_type} · "
|
||||
f"{_plural(total, 'image')} · {policy}"
|
||||
)
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
remaining = await self._autotag_prepass(job, targets, run.existing == "skip")
|
||||
if remaining is None:
|
||||
return _batch_cancelled(job)
|
||||
done = total - len(remaining)
|
||||
spec = dataclasses.replace(
|
||||
spec, targets=tuple(item.target for item in remaining)
|
||||
)
|
||||
loop_run = _Run(job, spec, epoch, run.owner, service)
|
||||
if not await self._tag_each_file(loop_run, done_before=done):
|
||||
job.update(status_line=STORAGE_RESTORED)
|
||||
job.log_line(f"Tagging stopped · {STORAGE_RESTORED}")
|
||||
return "failed"
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
finished = f"Tagged {_plural(job.counters.get('tagged', 0), 'image')}"
|
||||
job.update(status_line=finished, progress=100)
|
||||
job.log_line(finished)
|
||||
return "finished"
|
||||
|
||||
async def _autotag_prepass(self, job, targets, skip_marked: bool):
|
||||
"""The targets left for the model, or None when cancelled meanwhile.
|
||||
|
||||
Opens and hashes nothing: one stat per file and one indexed read per
|
||||
chunk. Skips are counted with ``record_counted`` (exact counters, rows
|
||||
up to the cap, no log line each).
|
||||
"""
|
||||
max_file = self._capture_store().max_file
|
||||
remaining, skipped = [], 0
|
||||
total = len(targets)
|
||||
for start in range(0, total, PREPASS_CHUNK):
|
||||
if job.cancelled:
|
||||
return None
|
||||
chunk = targets[start : start + PREPASS_CHUNK]
|
||||
left, skips = await self._run_in_executor(
|
||||
_prepass_chunk, self.db, chunk, skip_marked, max_file
|
||||
)
|
||||
remaining.extend(left)
|
||||
if skips:
|
||||
job.record_counted(
|
||||
"skipped",
|
||||
len(skips),
|
||||
[
|
||||
{
|
||||
"path": item.target[2],
|
||||
"root": item.label,
|
||||
"root_index": item.target[0],
|
||||
"detail": detail,
|
||||
}
|
||||
for item, detail in skips
|
||||
],
|
||||
)
|
||||
skipped += len(skips)
|
||||
job.update(processed=skipped, progress=skipped * 100 // total)
|
||||
job.publish({"type": "progress", "progress": job.progress})
|
||||
if skipped:
|
||||
job.log_line(f"Skipped {_plural(skipped, 'image')} before tagging")
|
||||
return remaining
|
||||
|
||||
async def _resolve_job_filter(self, spec: _Spec):
|
||||
"""``spec`` with its filter resolved into targets, or ``(None, error)``.
|
||||
|
||||
The listing's own identities, in its order, kept only where a target
|
||||
could have named them (a media path the job can tag).
|
||||
"""
|
||||
cap = MAX_FILTER_TARGETS
|
||||
try:
|
||||
selection = await self.resolve_listing_filter(spec.filter, cap)
|
||||
except ListingSourceChanged:
|
||||
return None, _coded("source-changed")
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
except Exception:
|
||||
# Never echo it: it can carry SQL or database text.
|
||||
self.logger.error("A tagging filter could not be resolved", exc_info=True)
|
||||
return None, _error(500, "listing-failed", FILTER_LISTING_FAILED)
|
||||
if selection.truncated:
|
||||
return None, _error(400, "too-many-images", TOO_MANY_IMAGES.format(cap=cap))
|
||||
targets = []
|
||||
for index, scope, relative in selection.targets:
|
||||
target, error = _target(
|
||||
{"root_index": index, "root_scope": scope, "path": relative}
|
||||
)
|
||||
if error is None:
|
||||
targets.append(target)
|
||||
if not targets:
|
||||
return None, _error(400, "no-images", NO_IMAGES)
|
||||
return dataclasses.replace(spec, targets=tuple(targets)), None
|
||||
|
||||
# -- runner ----------------------------------------------------------------
|
||||
|
||||
def _job_roots(self, targets) -> dict:
|
||||
@@ -331,28 +734,39 @@ class ImageTagJobsMixin:
|
||||
job.finish("failed")
|
||||
return
|
||||
if not await self._tag_each_file(run):
|
||||
job.update(status_line=STORAGE_RESTORED)
|
||||
job.log_line(f"{verb} stopped · {STORAGE_RESTORED}")
|
||||
job.finish("failed")
|
||||
return
|
||||
state = "cancelled" if job.cancelled else "finished"
|
||||
job.log_line(f"{verb} {state} · {_plural(job.processed, 'file')} processed")
|
||||
# An ended job never keeps the loop's "about X min left" (#379).
|
||||
ended = f"{verb} {state} · {_plural(job.processed, 'file')} processed"
|
||||
job.update(status_line=ended)
|
||||
job.log_line(ended)
|
||||
job.publish({"type": state, "progress": job.progress})
|
||||
job.finish(state)
|
||||
finally:
|
||||
if owner is not None:
|
||||
model_gate.release(owner)
|
||||
|
||||
async def _tag_each_file(self, run: _Run) -> bool:
|
||||
"""Tag the targets in order; False when the job had to stop."""
|
||||
job, total = run.job, len(run.spec.targets)
|
||||
async def _tag_each_file(self, run: _Run, *, done_before: int = 0) -> bool:
|
||||
"""Tag the targets in order; False when the job had to stop.
|
||||
|
||||
``done_before`` files were already counted (the batch's pre-pass): file
|
||||
n of this loop is "file done_before + n" of the job, and the time left
|
||||
uses this loop's rate only.
|
||||
"""
|
||||
job, count = run.job, len(run.spec.targets)
|
||||
total = done_before + count
|
||||
run.roots = await self._run_in_executor(self._job_roots, run.spec.targets)
|
||||
started = _monotonic()
|
||||
for number, target in enumerate(run.spec.targets, 1):
|
||||
if job.cancelled:
|
||||
break
|
||||
# The scan's estimate (D56), with each finished file as one batch.
|
||||
left = _time_left(number - 1, total, number - 1, _monotonic() - started)
|
||||
job.update(status_line=f"Processing file {number:,} of {total:,}{left}")
|
||||
left = _time_left(number - 1, count, number - 1, _monotonic() - started)
|
||||
shown = done_before + number
|
||||
job.update(status_line=f"Processing file {shown:,} of {total:,}{left}")
|
||||
job.publish({"type": "progress", "progress": job.progress})
|
||||
stopped = False
|
||||
try:
|
||||
@@ -363,7 +777,7 @@ class ImageTagJobsMixin:
|
||||
self.logger.exception("Image tagging failed for one file")
|
||||
group, detail = "errors", SAVE_FAILED
|
||||
self._record_tag_outcome(run, target, group, detail)
|
||||
job.update(processed=number, progress=number * 100 // total)
|
||||
job.update(processed=shown, progress=shown * 100 // total)
|
||||
if stopped:
|
||||
return False
|
||||
return True
|
||||
@@ -439,9 +853,17 @@ class ImageTagJobsMixin:
|
||||
store.release(capture.token)
|
||||
|
||||
async def _auto_tag_capture(self, run: _Run, store, capture, root, relative):
|
||||
"""Suggest on the capture, then union only onto the same current bytes."""
|
||||
"""Suggest on the capture, then union only onto the same current bytes.
|
||||
|
||||
The bytes are marked auto-tagged with the union, also when the model
|
||||
returned no names; captured bytes no model can read are marked too
|
||||
(and stay skipped), so a "Skip already tagged" run passes them by.
|
||||
"""
|
||||
if not capture.inferable:
|
||||
return "skipped", NOT_A_STILL
|
||||
group, detail = await self._union_image_tags(
|
||||
capture, [], run.epoch, mark_autotagged=True
|
||||
)
|
||||
return (group, detail) if group == "errors" else ("skipped", NOT_A_STILL)
|
||||
if store.acquire(capture.token) is None:
|
||||
raise _JobStopped # every capture was invalidated (restore)
|
||||
spec = run.spec
|
||||
@@ -467,21 +889,30 @@ class ImageTagJobsMixin:
|
||||
if facts.locator != capture.locator or facts.sha256 != capture.sha256:
|
||||
return "skipped", FILE_CHANGED
|
||||
names = [str(name) for name, _score in pairs]
|
||||
if not names:
|
||||
return "unchanged", NO_NEW_TAGS
|
||||
return await self._union_image_tags(facts, names, run.epoch)
|
||||
return await self._union_image_tags(
|
||||
facts, names, run.epoch, mark_autotagged=True
|
||||
)
|
||||
|
||||
async def _union_image_tags(self, facts, tags: list, epoch: str):
|
||||
"""``add_image_tags`` on the verified bytes, as a (group, detail)."""
|
||||
async def _union_image_tags(
|
||||
self, facts, tags: list, epoch: str, *, mark_autotagged: bool = False
|
||||
):
|
||||
"""``add_image_tags`` on the verified bytes, as a (group, detail).
|
||||
|
||||
``mark_autotagged`` (a model's result) marks the bytes auto-tagged in
|
||||
the same write; then ``tags`` may be empty.
|
||||
"""
|
||||
try:
|
||||
receipt = await self._run_in_executor(
|
||||
self.db.add_image_tags,
|
||||
locator=facts.locator,
|
||||
sha256=facts.sha256,
|
||||
size=facts.size,
|
||||
mtime_ns=facts.mtime_ns,
|
||||
expected_epoch=epoch,
|
||||
tags=tags,
|
||||
functools.partial(
|
||||
self.db.add_image_tags,
|
||||
locator=facts.locator,
|
||||
sha256=facts.sha256,
|
||||
size=facts.size,
|
||||
mtime_ns=facts.mtime_ns,
|
||||
expected_epoch=epoch,
|
||||
tags=tags,
|
||||
mark_autotagged=mark_autotagged,
|
||||
)
|
||||
)
|
||||
except LibraryCommandError as exc:
|
||||
if exc.code == "storage-changed":
|
||||
|
||||
+215
-75
@@ -575,12 +575,6 @@ def _current_fingerprint(path: Path) -> tuple[int, int] | None:
|
||||
return observed.st_size, observed.st_mtime_ns
|
||||
|
||||
|
||||
def _image_tags_relevant(names: list, wanted: set, query: str) -> bool:
|
||||
if wanted and not wanted.isdisjoint(names):
|
||||
return True
|
||||
return bool(query) and any(query in name.casefold() for name in names)
|
||||
|
||||
|
||||
def _workspace_key(entry: tuple, order: str) -> tuple:
|
||||
"""The keyset sort key of *entry* (a _Listed or a 4-tuple entry)."""
|
||||
if not isinstance(entry, _Listed):
|
||||
@@ -1105,37 +1099,94 @@ def _workspace_filters(query: Mapping) -> dict:
|
||||
}
|
||||
|
||||
|
||||
# The listing parameters a bulk request may name as its filter (#392): the
|
||||
# workspace query without mode, limit, cursor and the Activity-only ``path``,
|
||||
# plus the chosen root's ``root_scope`` (a renumbered source is refused).
|
||||
LISTING_FILTER_KEYS = frozenset(
|
||||
{
|
||||
"q",
|
||||
"subfolder",
|
||||
"category",
|
||||
"tags",
|
||||
"root",
|
||||
"root_scope",
|
||||
"media",
|
||||
"sort",
|
||||
"favorites",
|
||||
}
|
||||
)
|
||||
MAX_LISTING_FILTER_VALUE = 2048
|
||||
# The most images "Select all N matching" acts on in one request (#392).
|
||||
MAX_LISTING_SELECTION = 10_000
|
||||
|
||||
|
||||
def parse_listing_filter(raw: object) -> dict:
|
||||
"""A bulk request's listing filter (#392): the listing's own query as an object.
|
||||
|
||||
Keys are a subset of LISTING_FILTER_KEYS, each value a string of at most
|
||||
MAX_LISTING_FILTER_VALUE characters, validated as the listing validates its
|
||||
query. The root's bounds need the current roots, so resolve_listing_filter
|
||||
checks those. Raises ValueError.
|
||||
"""
|
||||
if not isinstance(raw, dict) or not set(raw) <= LISTING_FILTER_KEYS:
|
||||
raise ValueError("filter must be an object of listing parameters")
|
||||
if any(
|
||||
not isinstance(value, str) or len(value) > MAX_LISTING_FILTER_VALUE
|
||||
for value in raw.values()
|
||||
):
|
||||
raise ValueError("filter values must be short strings")
|
||||
if "root_scope" in raw and "root" not in raw:
|
||||
raise ValueError("root_scope needs root")
|
||||
_workspace_filters(raw)
|
||||
return dict(raw)
|
||||
|
||||
|
||||
class ListingSourceChanged(Exception):
|
||||
"""A listing filter's ``root_scope`` no longer names its root (#392)."""
|
||||
|
||||
|
||||
class ListingSelection(NamedTuple):
|
||||
"""Every image a listing filter admits, in the listing's order (#392).
|
||||
|
||||
``targets`` are listing identities ``(root_index, root_scope, path)``;
|
||||
``positive_ids`` the distinct linked positives (first seen), ``unlinked``
|
||||
the admitted images without one, and ``truncated`` that more than the cap
|
||||
matched (the lists then stop at the cap).
|
||||
"""
|
||||
|
||||
targets: list
|
||||
positive_ids: list
|
||||
unlinked: int
|
||||
truncated: bool
|
||||
|
||||
|
||||
# A plain listing's cursor page leaves positive_count out (see
|
||||
# _select_workspace_page); None instead means the count could not be read.
|
||||
_COUNT_OMITTED = object()
|
||||
|
||||
|
||||
def _filtered_page(
|
||||
def _admitted_entries(
|
||||
entries: list,
|
||||
start: int,
|
||||
size: int,
|
||||
filters: dict,
|
||||
links: dict | None,
|
||||
favorites: frozenset | None = None,
|
||||
image_tags: dict | None = None,
|
||||
) -> tuple[int, list, int | None]:
|
||||
"""Count the *entries* the filters admit and take *size* from *start* on.
|
||||
):
|
||||
"""Yield ``(position, entry, link match)`` for each entry the filters admit.
|
||||
|
||||
*links* maps normalized paths to (hit, rating, positive id) for every
|
||||
unambiguously linked image (see PromptDatabase.get_image_link_matches); a
|
||||
category or tag filter admits only those, and ``q`` admits a filename match
|
||||
or a linked-positive hit. Returns the total, the page and the number of
|
||||
distinct linked positives among all admitted entries (None without links).
|
||||
*favorites*, when given, admits only those listed paths (``favorites=1``).
|
||||
*image_tags* maps normalized paths to their _ImageTagMatch: a tag filter
|
||||
also admits a file whose current image tags (with its linked positive's
|
||||
tags) carry every tag, and ``q`` also admits an image tag name hit.
|
||||
or a linked-positive hit. *favorites*, when given, admits only those listed
|
||||
paths (``favorites=1``). *image_tags* maps normalized paths to their
|
||||
_ImageTagMatch: a tag filter also admits a file whose current image tags
|
||||
(with its linked positive's tags) carry every tag, and ``q`` also admits an
|
||||
image tag name hit.
|
||||
"""
|
||||
query, media, folder = filters["query"], filters["media"], filters["folder"]
|
||||
members_only = filters["category"] is not None or bool(filters["tags"])
|
||||
native_folder = folder.replace("/", os.sep)
|
||||
nested = native_folder + os.sep
|
||||
total, page, positives = 0, [], set()
|
||||
exact = filters.get("path", "")
|
||||
for position, entry in enumerate(entries):
|
||||
if exact and entry.folded != exact:
|
||||
@@ -1159,6 +1210,28 @@ def _filtered_page(
|
||||
and not (image and image.query)
|
||||
):
|
||||
continue
|
||||
yield position, entry, match
|
||||
|
||||
|
||||
def _filtered_page(
|
||||
entries: list,
|
||||
start: int,
|
||||
size: int,
|
||||
filters: dict,
|
||||
links: dict | None,
|
||||
favorites: frozenset | None = None,
|
||||
image_tags: dict | None = None,
|
||||
) -> tuple[int, list, int | None]:
|
||||
"""Count the *entries* the filters admit and take *size* from *start* on.
|
||||
|
||||
Admission is _admitted_entries'. Returns the total, the page and the number
|
||||
of distinct linked positives among all admitted entries (None without
|
||||
links).
|
||||
"""
|
||||
total, page, positives = 0, [], set()
|
||||
for position, entry, match in _admitted_entries(
|
||||
entries, filters, links, favorites, image_tags
|
||||
):
|
||||
total += 1
|
||||
if match is not None:
|
||||
positives.add(match[2])
|
||||
@@ -2091,8 +2164,9 @@ class ImageRoutesMixin:
|
||||
def _image_tag_matches(self, listings: list[tuple], filters: dict) -> dict:
|
||||
"""Normalized path -> _ImageTagMatch for files whose image tags count.
|
||||
|
||||
Streams the tagged byte records once and keeps only listed locators
|
||||
carrying a filter tag or a ``q`` hit; only those candidates are
|
||||
Looks the relevant tag names up (exact under Tags, a case-folded
|
||||
substring for ``q``) and their files through the tag index, keeping
|
||||
only listed locators; only those candidates are
|
||||
stat'ed (never every listed file), and a candidate's tags count only
|
||||
while a byte record is verified at its present ``(size, mtime_ns)``
|
||||
(spec amendment S1). Under a Tags filter the candidates' linked
|
||||
@@ -2101,16 +2175,15 @@ class ImageRoutesMixin:
|
||||
tags, query, category = filters["tags"], filters["query"], filters["category"]
|
||||
wanted = set(tags)
|
||||
located = self._workspace_locators(listings)
|
||||
# The tag index answers which files carry a relevant name (#379); no
|
||||
# read streams every image tag row.
|
||||
tag_ids = self.db.image_tag_ids(wanted, query)
|
||||
candidates = {}
|
||||
records = self.db.iter_image_tag_records()
|
||||
try:
|
||||
for locator, _size, _mtime_ns, names in records:
|
||||
if tag_ids:
|
||||
for locator in self.db.image_tag_locators(tag_ids):
|
||||
entry = located.get(locator)
|
||||
if entry is not None and locator not in candidates:
|
||||
if _image_tags_relevant(names, wanted, query):
|
||||
candidates[locator] = entry
|
||||
finally:
|
||||
records.close()
|
||||
if entry is not None:
|
||||
candidates[locator] = entry
|
||||
stats = {}
|
||||
for locator, entry in candidates.items():
|
||||
fingerprint = _current_fingerprint(entry.path)
|
||||
@@ -2137,35 +2210,23 @@ class ImageRoutesMixin:
|
||||
)
|
||||
return matches
|
||||
|
||||
def _select_workspace_page(
|
||||
self,
|
||||
listings: list[tuple],
|
||||
filters: dict,
|
||||
after: tuple | None,
|
||||
limit: int,
|
||||
scope: str,
|
||||
def _workspace_admission(
|
||||
self, listings: list[tuple], filters: dict, *, read_links: bool, strict=False
|
||||
) -> tuple:
|
||||
"""One keyset page of the filtered listing (blocking; run in executor).
|
||||
"""The ordered entries and what admits them (blocking; run in executor).
|
||||
|
||||
One SELECT (get_image_link_matches) reads every linked positive of the
|
||||
listing: the filters, the rating sort and the positive count share it.
|
||||
A plain listing (no ``q``, category, tags or rating sort) reads it only
|
||||
for its first page; its cursor pages read no links and the returned
|
||||
count is _COUNT_OMITTED, so the response leaves ``positive_count`` out
|
||||
and the frontend keeps the first page's value. On that plain first
|
||||
page a failed read leaves the listing usable and the count unknown
|
||||
(None); so does a failed read under ``q`` alone, whose search then
|
||||
matches filenames only (as before the linked search existed).
|
||||
Returns ``(entries, key, links, image_tags, favorites)``: the listing in
|
||||
the filters' order, its keyset key, and the link, image-tag and favorite
|
||||
maps _admitted_entries reads. One SELECT (get_image_link_matches) reads
|
||||
every linked positive when *read_links*. A failed read raises when the
|
||||
filters need it (category, tags, the rating sort) or *strict*; else the
|
||||
listing stays usable without links (a search then matches filenames
|
||||
only). A failed image-tag read raises under a Tags filter; a search then
|
||||
matches without image tags.
|
||||
"""
|
||||
order, category, tags = filters["order"], filters["category"], filters["tags"]
|
||||
needed = (
|
||||
bool(filters["query"])
|
||||
or category is not None
|
||||
or bool(tags)
|
||||
or order == "rating"
|
||||
)
|
||||
links = None
|
||||
if needed or after is None:
|
||||
if read_links:
|
||||
try:
|
||||
links = self.db.get_image_link_matches(
|
||||
filters["text"],
|
||||
@@ -2175,10 +2236,7 @@ class ImageRoutesMixin:
|
||||
wanted=self._workspace_wanted(listings),
|
||||
)
|
||||
except Exception:
|
||||
# Category, tags and the rating sort cannot be served without
|
||||
# the links. A plain listing and a filename search can: the
|
||||
# search then matches filenames only, and the count is None.
|
||||
if category is not None or tags or order == "rating":
|
||||
if strict or category is not None or tags or order == "rating":
|
||||
raise
|
||||
self.logger.warning(
|
||||
"Workspace prompt links could not be read; %s",
|
||||
@@ -2194,8 +2252,6 @@ class ImageRoutesMixin:
|
||||
try:
|
||||
image_tags = self._image_tag_matches(listings, filters)
|
||||
except Exception:
|
||||
# A Tags filter cannot be served without them (as without the
|
||||
# links); a search then matches without image tags.
|
||||
if tags:
|
||||
raise
|
||||
self.logger.warning(
|
||||
@@ -2234,6 +2290,41 @@ class ImageRoutesMixin:
|
||||
def key(entry):
|
||||
return _workspace_key(entry, order)
|
||||
|
||||
favorites = (
|
||||
self._workspace_favorite_paths(listings) if filters["favorites"] else None
|
||||
)
|
||||
return entries, key, links, image_tags, favorites
|
||||
|
||||
def _select_workspace_page(
|
||||
self,
|
||||
listings: list[tuple],
|
||||
filters: dict,
|
||||
after: tuple | None,
|
||||
limit: int,
|
||||
scope: str,
|
||||
) -> tuple:
|
||||
"""One keyset page of the filtered listing (blocking; run in executor).
|
||||
|
||||
The filters, the rating sort and the positive count share one link
|
||||
read (_workspace_admission). A plain listing (no ``q``, category, tags
|
||||
or rating sort) reads it only for its first page; its cursor pages read
|
||||
no links and the returned count is _COUNT_OMITTED, so the response
|
||||
leaves ``positive_count`` out and the frontend keeps the first page's
|
||||
value. On that plain first page a failed read leaves the listing usable
|
||||
and the count unknown (None); so does a failed read under ``q`` alone,
|
||||
whose search then matches filenames only (as before the linked search
|
||||
existed).
|
||||
"""
|
||||
order = filters["order"]
|
||||
needed = (
|
||||
bool(filters["query"])
|
||||
or filters["category"] is not None
|
||||
or bool(filters["tags"])
|
||||
or order == "rating"
|
||||
)
|
||||
entries, key, links, image_tags, favorites = self._workspace_admission(
|
||||
listings, filters, read_links=needed or after is None
|
||||
)
|
||||
start = 0
|
||||
if after is not None:
|
||||
if order == "name-desc":
|
||||
@@ -2241,9 +2332,6 @@ class ImageRoutesMixin:
|
||||
start = len(entries) - bisect_left(ascending, after, key=key)
|
||||
else:
|
||||
start = bisect_right(entries, after, key=key)
|
||||
favorites = (
|
||||
self._workspace_favorite_paths(listings) if filters["favorites"] else None
|
||||
)
|
||||
total, page, positive_count = _filtered_page(
|
||||
entries, start, limit + 1, filters, links, favorites, image_tags
|
||||
)
|
||||
@@ -2260,6 +2348,61 @@ class ImageRoutesMixin:
|
||||
positive_count,
|
||||
)
|
||||
|
||||
def _select_workspace_matches(
|
||||
self, listings: list[tuple], filters: dict, cap: int
|
||||
) -> ListingSelection:
|
||||
"""Every admitted entry, up to *cap*, as a ListingSelection (blocking).
|
||||
|
||||
The links are read strictly: the positives must be the listing's own.
|
||||
"""
|
||||
entries, _key, links, image_tags, favorites = self._workspace_admission(
|
||||
listings, filters, read_links=True, strict=True
|
||||
)
|
||||
targets, positives, unlinked, scopes = [], {}, 0, {}
|
||||
for _position, entry, match in _admitted_entries(
|
||||
entries, filters, links, favorites, image_tags
|
||||
):
|
||||
if len(targets) == cap:
|
||||
return ListingSelection(targets, list(positives), unlinked, True)
|
||||
scope = scopes.get(entry.index)
|
||||
if scope is None:
|
||||
scope = scopes[entry.index] = self._gallery_root_scope(
|
||||
entry.root, entry.index
|
||||
)
|
||||
relative = entry.path.relative_to(entry.root).as_posix()
|
||||
targets.append((entry.index, scope, relative))
|
||||
if match is None or match[2] is None:
|
||||
unlinked += 1
|
||||
else:
|
||||
positives.setdefault(match[2])
|
||||
return ListingSelection(targets, list(positives), unlinked, False)
|
||||
|
||||
async def resolve_listing_filter(
|
||||
self, params: Mapping, cap: int = MAX_LISTING_SELECTION
|
||||
) -> ListingSelection:
|
||||
"""The images a parse_listing_filter result admits right now (#392).
|
||||
|
||||
Raises ValueError for a root outside the current roots, and
|
||||
ListingSourceChanged when the filter's ``root_scope`` no longer names
|
||||
its root (the sources were renumbered since the listing).
|
||||
"""
|
||||
output_dirs = list(self._get_all_output_dirs())
|
||||
root_index = _workspace_root(params, output_dirs)
|
||||
error = self._gallery_scope_error(params, output_dirs)
|
||||
if error is not None:
|
||||
if error.status == 409:
|
||||
raise ListingSourceChanged()
|
||||
raise ValueError("Invalid gallery root scope")
|
||||
filters = _workspace_filters(params)
|
||||
listings = [
|
||||
(index, root, await self._get_gallery_files(root))
|
||||
for index, root in enumerate(output_dirs)
|
||||
if root_index is None or index == root_index
|
||||
]
|
||||
return await self._run_in_executor(
|
||||
self._select_workspace_matches, listings, filters, cap
|
||||
)
|
||||
|
||||
async def serve_image(self, request):
|
||||
"""Serve the actual image file using streamed FileResponse."""
|
||||
try:
|
||||
@@ -2272,7 +2415,8 @@ class ImageRoutesMixin:
|
||||
)
|
||||
|
||||
image_path = Path(image["image_path"])
|
||||
allowed_dirs = list(self._get_all_output_dirs()) + self._lora_image_dirs()
|
||||
lora_dirs = await self._run_in_executor(self._lora_image_dirs)
|
||||
allowed_dirs = list(self._get_all_output_dirs()) + lora_dirs
|
||||
|
||||
denied = _media_access_error(image_path, allowed_dirs)
|
||||
if denied is not None:
|
||||
@@ -2320,7 +2464,10 @@ class ImageRoutesMixin:
|
||||
LoraManager's guarded LoRA roots, sidecar root and example-images
|
||||
root (``lora_manager_layout.media_roots``, #355) and PromptManager's
|
||||
CivitAI download cache. An unusable LoraManager layout gives the
|
||||
default LoRA roots and the cache.
|
||||
default LoRA roots and the cache. The layout comes from the layout
|
||||
cache (``lora_utils.cached_lora_manager_layout``, #362), which reloads
|
||||
when LoraManager's settings files change; blocking, so the routes
|
||||
call it in the executor.
|
||||
"""
|
||||
try:
|
||||
from ..config import IntegrationConfig
|
||||
@@ -2328,25 +2475,18 @@ class ImageRoutesMixin:
|
||||
if not IntegrationConfig.LORA_MANAGER_ENABLED:
|
||||
return []
|
||||
from ..lora_utils import (
|
||||
cached_lora_manager_layout,
|
||||
get_lora_image_cache_dir,
|
||||
layout_for_install,
|
||||
lora_manager_layout,
|
||||
)
|
||||
|
||||
lora_root = IntegrationConfig.LORA_MANAGER_PATH or ""
|
||||
if lora_root.strip():
|
||||
configured = IntegrationConfig.LORA_MANAGER_PATH or ""
|
||||
install, layout = cached_lora_manager_layout(configured)
|
||||
if configured.strip() and install is None:
|
||||
# A configured path comes from config.json, which can be hand
|
||||
# edited: only a LoraManager install under custom_nodes may
|
||||
# widen the serving allow-list. An empty path keeps the
|
||||
# auto-detection, which itself only scans custom_nodes.
|
||||
from ..lora_utils import resolve_lora_manager_path
|
||||
|
||||
lora_root = resolve_lora_manager_path(lora_root)
|
||||
if not lora_root:
|
||||
return []
|
||||
layout = layout_for_install(lora_root)
|
||||
else:
|
||||
layout = lora_manager_layout("")
|
||||
return []
|
||||
|
||||
dirs = [Path(d) for d in layout.media_roots]
|
||||
# Named, not made: listing and serving never create the cache.
|
||||
|
||||
@@ -129,8 +129,9 @@ def inspect_current_file(
|
||||
profile = (
|
||||
PREVIEW_INSPECTION_PROFILE if _in_host_temp(path) else INSPECTION_PROFILE
|
||||
)
|
||||
# The file name anchors on the saver that wrote it (#381).
|
||||
evidence = inspect_prompt_evidence(
|
||||
inspection.metadata, role_saved, profile=profile
|
||||
inspection.metadata, role_saved, profile=profile, file_name=path.name
|
||||
)
|
||||
return {
|
||||
"file": {
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Reviewed rotated-log cleanup as a maintenance plan (#329 System S12).
|
||||
|
||||
``POST /prompt_manager/maintenance/preview {operation: "rotated_logs",
|
||||
options: {}}`` lists the rotated copies of the active log that Rule R allows
|
||||
(``utils/logging_config.py``: ``prompt_manager.log.<n>`` regular files that
|
||||
are not links, directly in the log folder) and writes nothing. The plan keeps
|
||||
each copy's name, size, ``mtime_ns`` and inode under its token; the response
|
||||
is the maintenance ``Plan`` shape with basenames only, a ``none`` backup
|
||||
policy (no library record changes) and no ``disk`` key.
|
||||
|
||||
``POST /prompt_manager/maintenance/apply {token}`` starts a ``maintenance``
|
||||
job titled "Rotated log copies" with admission ``read``: it may run beside a
|
||||
scan and is refused only beside another maintenance job or an ``exclusive``
|
||||
one. The job deletes exactly the reviewed copies under the logger's files
|
||||
lock and the file handler's lock, or nothing, failing ``plan-changed``, when
|
||||
any of them is missing, is no longer a rotated copy or differs in size,
|
||||
``mtime_ns`` or inode. A copy that cannot be deleted (open in another program
|
||||
on Windows) is an ``errors`` row; the others are deleted.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import json
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
try:
|
||||
from ...utils.logging_config import RotatedChanged, rotated_sort_key
|
||||
except ImportError: # loaded as the top-level ``py`` package (tests)
|
||||
from utils.logging_config import RotatedChanged, rotated_sort_key
|
||||
|
||||
from .jobs import MAINTENANCE_JOB_KIND
|
||||
from .maintenance_plans import ScopeTooLarge
|
||||
|
||||
ROTATED_LOGS = "rotated_logs"
|
||||
LOG_CLEANUP_TITLE = "Rotated log copies"
|
||||
LOGS_CHANGED = (
|
||||
"The log files changed since the review; nothing was deleted. Review again."
|
||||
)
|
||||
LOG_CLEANUP_FAILED = "The log cleanup failed."
|
||||
TOO_MANY_LOG_FILES = "Too many log files to review at once."
|
||||
LOG_CLEANUP_MEDIA = (
|
||||
"Deletes only the listed log files. No image, video or library record changes."
|
||||
)
|
||||
_GROUPS = ("changed", "protected", "errors")
|
||||
|
||||
|
||||
def size_label(size):
|
||||
"""``1000 bytes``, ``2.0 KB``, ``10.0 MB`` or ``1.2 GB`` (1 KB = 1024 bytes)."""
|
||||
size = max(0, int(size or 0))
|
||||
if size < 1024:
|
||||
return f"{size} {'byte' if size == 1 else 'bytes'}"
|
||||
for unit, scale in (("KB", 2**10), ("MB", 2**20)):
|
||||
if size < scale * 1024:
|
||||
return f"{size / scale:.1f} {unit}"
|
||||
return f"{size / 2**30:.1f} GB"
|
||||
|
||||
|
||||
def time_label(mtime_ns):
|
||||
"""The local modification time as ``YYYY-MM-DD HH:MM``."""
|
||||
try:
|
||||
moment = datetime.datetime.fromtimestamp(mtime_ns / 1e9)
|
||||
except (OSError, OverflowError, ValueError): # a time before 1970 on Windows
|
||||
return "unknown time"
|
||||
return moment.strftime("%Y-%m-%d %H:%M")
|
||||
|
||||
|
||||
def _count(count, singular, plural):
|
||||
return f"{count} {singular if count == 1 else plural}"
|
||||
|
||||
|
||||
def _protected_reasons(rows):
|
||||
"""The plan's Protected reasons: what the cleanup keeps, zero counts left out."""
|
||||
active = sum(1 for row in rows if row["kind"] == "active")
|
||||
links = sum(1 for row in rows if row["kind"] == "link")
|
||||
other = sum(1 for row in rows if row["kind"] == "other")
|
||||
reasons = []
|
||||
if active:
|
||||
reasons.append("Kept: the active log")
|
||||
if links:
|
||||
reasons.append(f"Kept: {_count(links, 'link', 'links')}")
|
||||
if other:
|
||||
reasons.append(f"Kept: {_count(other, 'other file', 'other files')}")
|
||||
return active + links + other, reasons
|
||||
|
||||
|
||||
def _scope_label(files):
|
||||
if not files:
|
||||
return "No rotated log copies"
|
||||
total = sum(row["size"] for row in files)
|
||||
return (
|
||||
f"{_count(len(files), 'rotated log copy', 'rotated log copies')}"
|
||||
f" · {size_label(total)}"
|
||||
)
|
||||
|
||||
|
||||
class LogCleanupMixin:
|
||||
"""The ``rotated_logs`` maintenance plan and its job.
|
||||
|
||||
The logger manager is reached only through ``_get_logger_manager``
|
||||
(``logging_routes.py``), so the tests can point it at a temporary folder.
|
||||
"""
|
||||
|
||||
async def preview_log_cleanup(self):
|
||||
"""The ``rotated_logs`` plan (runs inside the preview's lease)."""
|
||||
from . import maintenance
|
||||
|
||||
manager = self._get_logger_manager()
|
||||
try:
|
||||
rows = await self._run_in_executor(manager.scan_log_folder)
|
||||
except Exception:
|
||||
self.logger.exception("Rotated log review failed")
|
||||
return maintenance._failure("Failed to review maintenance", 500)
|
||||
rotated = sorted(
|
||||
(row for row in rows if row["kind"] == "rotated"),
|
||||
key=rotated_sort_key,
|
||||
)
|
||||
files = [
|
||||
{
|
||||
"name": row["name"],
|
||||
"size": row["size"],
|
||||
"mtime_ns": row["mtime_ns"],
|
||||
"ino": row["ino"],
|
||||
}
|
||||
for row in rotated
|
||||
]
|
||||
protected, reasons = _protected_reasons(rows)
|
||||
store = self._maintenance_plans
|
||||
try:
|
||||
token = store.issue(
|
||||
ROTATED_LOGS,
|
||||
json.dumps(files, sort_keys=True),
|
||||
{"files": files, "protected": protected},
|
||||
(),
|
||||
)
|
||||
except ScopeTooLarge:
|
||||
return maintenance._failure(TOO_MANY_LOG_FILES, 413, "too-large")
|
||||
plan = store.peek(token, ROTATED_LOGS)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"token": token,
|
||||
"operation": ROTATED_LOGS,
|
||||
"expires_at": maintenance._iso(plan.expires_at),
|
||||
"scope": {"count": len(files), "label": _scope_label(files)},
|
||||
"protected": {"count": protected, "reasons": reasons},
|
||||
"backup": {
|
||||
"policy": "none",
|
||||
"destination_label": None,
|
||||
"expected_bytes": None,
|
||||
},
|
||||
"media": {"deletes_files": True, "label": LOG_CLEANUP_MEDIA},
|
||||
"details": [
|
||||
{
|
||||
"label": row["name"],
|
||||
"detail": (
|
||||
f"{size_label(row['size'])} · {time_label(row['mtime_ns'])}"
|
||||
),
|
||||
}
|
||||
for row in files
|
||||
],
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
def _start_log_cleanup(self, plan):
|
||||
"""Register the cleanup job (on the loop); raises JobBusy when refused."""
|
||||
return self._jobs.start(
|
||||
MAINTENANCE_JOB_KIND,
|
||||
LOG_CLEANUP_TITLE,
|
||||
admission="read",
|
||||
cancellable=False,
|
||||
scope={"operation": ROTATED_LOGS},
|
||||
groups=_GROUPS,
|
||||
runner=lambda job: self._run_log_cleanup(job, plan),
|
||||
)
|
||||
|
||||
async def _run_log_cleanup(self, job, plan):
|
||||
from . import maintenance
|
||||
|
||||
files = plan.payload["files"]
|
||||
sizes = {row["name"]: row["size"] for row in files}
|
||||
job.update(total=len(files), status_line="Deleting rotated log copies")
|
||||
manager = self._get_logger_manager()
|
||||
try:
|
||||
result = await self._run_in_executor(manager.delete_rotated, files)
|
||||
except RotatedChanged:
|
||||
return maintenance._fail_job(job, "plan-changed", LOGS_CHANGED)
|
||||
except Exception:
|
||||
self.logger.exception("Rotated log cleanup failed")
|
||||
return maintenance._fail_job(job, None, LOG_CLEANUP_FAILED)
|
||||
for name in result["removed"]:
|
||||
detail = f"Deleted · {size_label(sizes.get(name, 0))}"
|
||||
job.record(
|
||||
"changed", {"path": name, "detail": detail}, f"- {name} · {detail}"
|
||||
)
|
||||
for name, error in result["failed"]:
|
||||
detail = self._public_error(error)
|
||||
job.record(
|
||||
"errors", {"path": name, "detail": detail}, f"! {name} · {detail}"
|
||||
)
|
||||
job.record_counted("protected", plan.payload["protected"])
|
||||
removed, failed = len(result["removed"]), len(result["failed"])
|
||||
line = f"Deleted {_count(removed, 'rotated log copy', 'rotated log copies')}"
|
||||
if failed:
|
||||
line += f", {failed} could not be deleted"
|
||||
job.log_line(line)
|
||||
job.update(processed=len(files), progress=100, status_line=line)
|
||||
+90
-10
@@ -1,9 +1,21 @@
|
||||
"""Logging API routes for PromptManager."""
|
||||
"""Logging API routes for PromptManager.
|
||||
|
||||
Every route that names a log file goes through Rule R (#329,
|
||||
``utils/logging_config.py``): a download is served only for a regular,
|
||||
non-link file listed directly in the log folder, and Clear deletes only
|
||||
rotated copies. File work runs off the event loop.
|
||||
"""
|
||||
|
||||
import os
|
||||
import stat
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
try:
|
||||
from ...utils.logging_config import ClearRefused, parse_confirmed
|
||||
except ImportError: # loaded as the top-level ``py`` package (tests)
|
||||
from utils.logging_config import ClearRefused, parse_confirmed
|
||||
|
||||
# Keys PromptManagerLogger.update_config() understands. Anything else is
|
||||
# rejected so a client cannot inject arbitrary entries into the logging config.
|
||||
LOG_CONFIG_KEYS = frozenset(
|
||||
@@ -85,6 +97,41 @@ def _validate_log_config(data):
|
||||
return clean, None
|
||||
|
||||
|
||||
def _read_log_bytes(path):
|
||||
"""The whole file at ``path``, opened without following a link (blocking).
|
||||
|
||||
The file is checked again through the open handle: a regular file with
|
||||
one name and, when ``path`` carries it, the inode it was listed with. A
|
||||
name swapped for a link or another file after it was listed, or a hard
|
||||
link to a file elsewhere, is refused as missing. The handle is closed
|
||||
before this returns.
|
||||
"""
|
||||
flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
||||
# A FIFO swapped in after the listing must not block the executor thread.
|
||||
flags |= getattr(os, "O_NONBLOCK", 0)
|
||||
try:
|
||||
fd = os.open(str(path), flags)
|
||||
except OSError as error:
|
||||
if getattr(os, "O_NOFOLLOW", 0) and os.path.islink(str(path)):
|
||||
raise FileNotFoundError("Log file not found") from error
|
||||
raise
|
||||
listed_ino = getattr(path, "listed_ino", 0)
|
||||
with os.fdopen(fd, "rb") as handle:
|
||||
opened = os.fstat(handle.fileno())
|
||||
if (
|
||||
not stat.S_ISREG(opened.st_mode)
|
||||
or opened.st_nlink > 1 # a hard link to a file elsewhere
|
||||
or (listed_ino and opened.st_ino and opened.st_ino != listed_ino)
|
||||
):
|
||||
raise FileNotFoundError("Log file not found")
|
||||
return handle.read()
|
||||
|
||||
|
||||
def _download(logger_manager, filename):
|
||||
"""``bytes`` of a downloadable log file; FileNotFoundError otherwise."""
|
||||
return _read_log_bytes(logger_manager.log_file_entry(filename))
|
||||
|
||||
|
||||
class LoggingRoutesMixin:
|
||||
"""Mixin providing logging-related API endpoints."""
|
||||
|
||||
@@ -195,16 +242,16 @@ class LoggingRoutesMixin:
|
||||
{"success": False, "error": "Invalid filename"}, status=400
|
||||
)
|
||||
|
||||
log_file_path = logger_manager.log_dir / filename
|
||||
|
||||
if not log_file_path.exists():
|
||||
try:
|
||||
file_content = await self._run_in_executor(
|
||||
_download, logger_manager, filename
|
||||
)
|
||||
except FileNotFoundError:
|
||||
# Missing, a link, a folder or not listed: all read as absent.
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Log file not found"}, status=404
|
||||
)
|
||||
|
||||
with open(log_file_path, "rb") as f:
|
||||
file_content = f.read()
|
||||
|
||||
return web.Response(
|
||||
body=file_content,
|
||||
content_type="text/plain",
|
||||
@@ -221,10 +268,43 @@ class LoggingRoutesMixin:
|
||||
)
|
||||
|
||||
async def truncate_logs(self, request):
|
||||
"""Truncate all log files."""
|
||||
"""Clear all logs: what ``{"expected": [...]}`` lists, or all (#389).
|
||||
|
||||
``expected`` holds the ``{name, identity}`` rows of ``/logs/files`` the
|
||||
confirmation showed. Without a body (the legacy admin page, whose
|
||||
confirmation lists nothing) every rotated copy present now is deleted.
|
||||
A malformed list is 400; a clear that would empty or delete what was
|
||||
not confirmed changes nothing and is 409 with the reason's ``code``.
|
||||
"""
|
||||
try:
|
||||
args = ()
|
||||
if request.can_read_body:
|
||||
data, error_response = await self._read_json_body(request)
|
||||
if error_response is not None:
|
||||
return error_response
|
||||
if "expected" in data:
|
||||
try:
|
||||
parse_confirmed(data["expected"])
|
||||
except ValueError:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "expected must list name and identity"
|
||||
" rows from /logs/files",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
args = (data["expected"],)
|
||||
logger_manager = self._get_logger_manager()
|
||||
results = logger_manager.truncate_logs()
|
||||
try:
|
||||
results = await self._run_in_executor(
|
||||
logger_manager.truncate_logs, *args
|
||||
)
|
||||
except ClearRefused as refused:
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(refused), "code": refused.code},
|
||||
status=409,
|
||||
)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
@@ -266,7 +346,7 @@ class LoggingRoutesMixin:
|
||||
return web.json_response({"success": False, "error": error}, status=400)
|
||||
|
||||
logger_manager = self._get_logger_manager()
|
||||
logger_manager.update_config(clean)
|
||||
await self._run_in_executor(logger_manager.update_config, clean)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
|
||||
@@ -194,8 +194,13 @@ class LoraImportRoutesMixin:
|
||||
return False
|
||||
|
||||
async def replace_lora_imports(self, db, staged, *, expected_ids, allow_empty):
|
||||
"""Swap the collection for ``staged.records`` in one transaction."""
|
||||
return await self._run_in_executor(
|
||||
"""Swap the collection for ``staged.records`` in one transaction.
|
||||
|
||||
Then the image routes' layout cache is dropped (#362): the import may
|
||||
have linked files under a folder LoraManager created after the cache
|
||||
stored its layout (its first example-image download creates one).
|
||||
"""
|
||||
result = await self._run_in_executor(
|
||||
lambda: db.replace_category_prompts(
|
||||
CATEGORY,
|
||||
staged.records,
|
||||
@@ -203,6 +208,8 @@ class LoraImportRoutesMixin:
|
||||
allow_empty=allow_empty,
|
||||
)
|
||||
)
|
||||
lora_utils.clear_layout_cache()
|
||||
return result
|
||||
|
||||
# ── preview ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
+19
-2
@@ -2,7 +2,9 @@
|
||||
|
||||
``POST /prompt_manager/maintenance/preview {operation, options:{}}`` reviews
|
||||
one operation (``duplicate_prompts``, ``optimize``, ``unused_components`` or
|
||||
``missing_links``; ``database/maintenance.py``) off the event loop and
|
||||
``missing_links``; ``database/maintenance.py``; ``duplicate_files``,
|
||||
``py/api/duplicate_files.py``; ``rotated_logs``, the rotated log copies,
|
||||
``py/api/log_cleanup.py``, which has its own apply job) off the event loop and
|
||||
answers a ``Plan``: ``{success, token, operation, expires_at, scope:{count,
|
||||
label}, protected:{count, reasons}, backup:{policy, destination_label},
|
||||
media:{deletes_files, label}, details:[≤50 {label, detail}]}``. The preview
|
||||
@@ -54,6 +56,7 @@ from aiohttp import web
|
||||
from .admin import _prune_safety_backups, _scan_location, _scan_roots
|
||||
from .job_routes import busy_response
|
||||
from .jobs import MAINTENANCE_JOB_KIND, JobBusy
|
||||
from .log_cleanup import ROTATED_LOGS
|
||||
from .maintenance_plans import PlanExpired, PlanUsed, ScopeTooLarge
|
||||
|
||||
try:
|
||||
@@ -83,7 +86,8 @@ MAINTENANCE_PREVIEW_LEASE = "Maintenance preview"
|
||||
DUPLICATE_PREVIEW_BODY_MAX = 1024 * 1024
|
||||
DUPLICATE_FILES = "duplicate_files"
|
||||
WORKFLOW_STORAGE = "workflow_storage" # applied by py/api/workflow_storage.py
|
||||
PREVIEW_OPERATIONS = MAINTENANCE_OPERATIONS + (DUPLICATE_FILES,)
|
||||
# rotated_logs (#329, py/api/log_cleanup.py) reviews log files, not records.
|
||||
PREVIEW_OPERATIONS = MAINTENANCE_OPERATIONS + (DUPLICATE_FILES, ROTATED_LOGS)
|
||||
# Room left over beyond what a step writes, so a disk is never filled to 0.
|
||||
DISK_MARGIN_BYTES = 16 * 2**20
|
||||
# The concept's Maintenance row titles, which name each operation's job.
|
||||
@@ -391,6 +395,8 @@ class MaintenanceRoutesMixin:
|
||||
return await self.preview_duplicate_files(options)
|
||||
if not isinstance(options, dict) or options:
|
||||
return _invalid(f"{operation} takes no options")
|
||||
if operation == ROTATED_LOGS:
|
||||
return await self.preview_log_cleanup()
|
||||
db = self.db
|
||||
identity = self._database_identity(db)
|
||||
try:
|
||||
@@ -492,6 +498,17 @@ class MaintenanceRoutesMixin:
|
||||
if refused is not None:
|
||||
return refused
|
||||
store = self._maintenance_plans
|
||||
if plan.operation == ROTATED_LOGS:
|
||||
# Log files only: no database identity, backup or disk check.
|
||||
try:
|
||||
job = self._start_log_cleanup(plan)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy) # the token stays usable
|
||||
# Same loop turn as the peek and the start: the runner has not run.
|
||||
store.consume(token, plan.operation, job_id=job.id)
|
||||
return web.json_response(
|
||||
{"success": True, "job": job.summary()}, status=202
|
||||
)
|
||||
db = self.db
|
||||
if plan.db_identity != self._database_identity(db):
|
||||
store.discard(token)
|
||||
|
||||
@@ -1178,6 +1178,8 @@ class PromptRoutesMixin:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Expected a JSON object"}, status=400
|
||||
)
|
||||
if "images_filter" in data:
|
||||
return await self._bulk_category_by_filter(data)
|
||||
if not data.get("prompt_ids"):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "No prompt IDs provided"}, status=400
|
||||
@@ -1214,6 +1216,66 @@ class PromptRoutesMixin:
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def _bulk_category_by_filter(self, data: dict) -> web.Response:
|
||||
"""Set the category on every linked positive a listing filter admits.
|
||||
|
||||
#392 "Select all N matching": ``{images_filter, category}``, the filter
|
||||
being the Images listing's own query (parse_listing_filter), resolved
|
||||
now. The answer adds ``prompt_count`` (the distinct linked positives)
|
||||
and ``skipped`` (matching images without one); none linked changes
|
||||
nothing.
|
||||
"""
|
||||
from .images import (
|
||||
MAX_LISTING_SELECTION,
|
||||
ListingSourceChanged,
|
||||
parse_listing_filter,
|
||||
)
|
||||
|
||||
def refused(message, **extra):
|
||||
return web.json_response(
|
||||
{"success": False, "error": message, **extra}, status=400
|
||||
)
|
||||
|
||||
if set(data) != {"images_filter", "category"}:
|
||||
return refused("Request must contain exactly category, images_filter")
|
||||
category = data["category"]
|
||||
try:
|
||||
params = parse_listing_filter(data["images_filter"])
|
||||
validate_category(category)
|
||||
selection = await self.resolve_listing_filter(params)
|
||||
except ListingSourceChanged:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "source-changed",
|
||||
"error": "The source changed. Refresh the image collection.",
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return refused(str(exc))
|
||||
if selection.truncated:
|
||||
return refused(
|
||||
f"Narrow the view to at most {MAX_LISTING_SELECTION:,} images.",
|
||||
code="too-many-images",
|
||||
)
|
||||
category = (category or "").strip()
|
||||
ids = selection.positive_ids
|
||||
updated = (
|
||||
await self._run_in_executor(self.db.bulk_set_category, ids, category)
|
||||
if ids
|
||||
else 0
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"message": f"Set category for {updated} prompts",
|
||||
"updated_count": updated,
|
||||
"prompt_count": len(ids),
|
||||
"skipped": selection.unlinked,
|
||||
}
|
||||
)
|
||||
|
||||
def _all_prompts_for_export(self):
|
||||
"""Every prompt, fetched in EXPORT_PAGE_SIZE pages (blocking)."""
|
||||
prompts, offset = [], 0
|
||||
|
||||
+11
-1
@@ -677,7 +677,14 @@ def _verify_and_read(store, capture, item, root, need_metadata: bool):
|
||||
facts = store.verify(root, relative)
|
||||
captured = None
|
||||
if need_metadata:
|
||||
captured = captured_review(store, capture, item_profile(item["evidence"]))
|
||||
# The file's own name, so an inspection item resolves anchored on its
|
||||
# saver as it was recorded (#390).
|
||||
captured = captured_review(
|
||||
store,
|
||||
capture,
|
||||
item_profile(item["evidence"]),
|
||||
relative.rsplit("/", 1)[-1],
|
||||
)
|
||||
return relative, facts, captured
|
||||
|
||||
|
||||
@@ -723,6 +730,9 @@ def _candidate_still_offered(resolution: dict, choice: str) -> bool:
|
||||
that rewords the top-level reason or reorders candidates leaves a
|
||||
pending item applicable.
|
||||
"""
|
||||
if isinstance(resolution, dict) and resolution.get("anchor") == "saver":
|
||||
# Only the saver anchor resolves the bytes (#390): its pair alone.
|
||||
resolution = {"state": "ambiguous", "candidates": [resolution]}
|
||||
if not isinstance(resolution, dict) or resolution.get("state") != "ambiguous":
|
||||
return False
|
||||
evidence, _count, limited = review_evidence(resolution)
|
||||
|
||||
@@ -88,7 +88,8 @@ class ReviewRecordingMixin:
|
||||
def _record_scanned_review(self, media_file, roots, review=None, scanned=None):
|
||||
"""Record one ambiguous, unlinked scanned file (logged and counted).
|
||||
|
||||
``scanned`` is the scan's ``(metadata, inspection_resolution)``.
|
||||
``scanned`` is the scan's ``(metadata, inspection_resolution)``; the
|
||||
resolution is None when the scan did not run the inspection profile.
|
||||
"""
|
||||
try:
|
||||
outcome = self._record_review_sync(
|
||||
@@ -199,7 +200,7 @@ class ReviewRecordingMixin:
|
||||
key = _memory_key(locator)
|
||||
if state is not None and memory.get(key) == state:
|
||||
return None
|
||||
resolution = review_capture.inspection_choice(canonical)
|
||||
resolution = review_capture.inspection_choice(canonical, media_file.name)
|
||||
if resolution is not None:
|
||||
memory.pop(key, None)
|
||||
return canonical, resolution
|
||||
|
||||
@@ -173,8 +173,8 @@ def _validate_autotag_settings(data):
|
||||
EXISTING_POLICIES,
|
||||
MAX_SUGGESTION_LIMIT,
|
||||
MODEL_TYPES,
|
||||
validate_saved_threshold,
|
||||
validate_template,
|
||||
validate_threshold,
|
||||
)
|
||||
|
||||
updates = {}
|
||||
@@ -198,7 +198,7 @@ def _validate_autotag_settings(data):
|
||||
):
|
||||
if key in data:
|
||||
try:
|
||||
value = validate_threshold(label, data[key])
|
||||
value = validate_saved_threshold(label, data[key])
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{error}.") from None
|
||||
updates[key.upper()] = value
|
||||
|
||||
+23
-11
@@ -88,9 +88,9 @@ from .thumbnail_encode import shutdown_encoders, writer_settings
|
||||
from .thumbnail_jobs import ThumbnailOptions
|
||||
|
||||
try:
|
||||
from ...utils.parallel import map_parallel
|
||||
from ...utils.parallel import map_parallel, note_refusal, refused_at_shutdown
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils.parallel import map_parallel
|
||||
from utils.parallel import map_parallel, note_refusal, refused_at_shutdown
|
||||
|
||||
# The placeholder frame a video without a preview is answered with (JPEG
|
||||
# bytes, never written to disk); written previews follow the settings.
|
||||
@@ -410,7 +410,8 @@ class AutoThumbnailer:
|
||||
|
||||
def enqueue(self, src, dst, is_video):
|
||||
"""Queue *dst*'s preview for the worker; False when already queued,
|
||||
full or closed. Never blocks on a write."""
|
||||
full, closed or its worker is refused at interpreter exit. Never
|
||||
blocks on a write."""
|
||||
key = _dst_key(dst)
|
||||
with self._changed:
|
||||
if self._closed or key in self._queued:
|
||||
@@ -420,30 +421,41 @@ class AutoThumbnailer:
|
||||
self._queued.add(key)
|
||||
self._queue.append((key, Path(src), Path(dst), bool(is_video)))
|
||||
self._busy = True
|
||||
started = True
|
||||
if self._start_worker and not self._worker_running:
|
||||
self._start_worker_locked()
|
||||
started = self._start_worker_locked()
|
||||
if not started:
|
||||
# Refused at interpreter exit: no worker will take it (#375).
|
||||
self._queue.pop()
|
||||
self._queued.discard(key)
|
||||
self._changed.notify_all()
|
||||
return True
|
||||
return started
|
||||
|
||||
def _start_worker_locked(self):
|
||||
"""Start the worker (lock held). A start that fails leaves the file
|
||||
queued and the next enqueue tries again."""
|
||||
"""Start the worker (lock held); False only when the interpreter is
|
||||
exiting (#375: logged once at DEBUG, the caller drops the file).
|
||||
Another start failure leaves the file queued and the next enqueue
|
||||
tries again."""
|
||||
worker = threading.Thread(target=self._run, name=WORKER_NAME, daemon=True)
|
||||
worker.pm_preview_token = _MODULE_TOKEN
|
||||
worker.pm_preview_close = self.close
|
||||
self._worker_running = True
|
||||
try:
|
||||
worker.start()
|
||||
except RuntimeError:
|
||||
except RuntimeError as error:
|
||||
self._worker_running = False
|
||||
self._busy = False
|
||||
if refused_at_shutdown(error):
|
||||
note_refusal(self._logger, "The automatic preview worker")
|
||||
return False
|
||||
self._logger.warning(
|
||||
"The automatic preview worker could not start; the next output "
|
||||
"tries again",
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
return True
|
||||
self._worker = worker
|
||||
return True
|
||||
|
||||
def _run(self):
|
||||
try:
|
||||
@@ -790,8 +802,8 @@ class AutoThumbnailMixin:
|
||||
|
||||
Returns ``queued``, ``fresh`` (its preview is current) or ``skipped``
|
||||
(not a media file under a current root, a link, a preview itself,
|
||||
already queued, or after the app's cleanup). Never writes on the
|
||||
caller's thread.
|
||||
already queued, after the app's cleanup, or when its worker is refused
|
||||
at interpreter exit). Never writes on the caller's thread.
|
||||
"""
|
||||
if self._auto_thumbnails_shut:
|
||||
return "skipped"
|
||||
|
||||
@@ -756,6 +756,24 @@ class AutoTagService:
|
||||
|
||||
self.logger.info("Model unloaded, memory freed")
|
||||
|
||||
def wd14_device(self) -> str:
|
||||
"""``"gpu"`` when WD14 runs (or would run) on CUDA, else ``"cpu"``.
|
||||
|
||||
A loaded WD14 session answers from its own first provider; otherwise
|
||||
ONNX Runtime's available providers say whether CUDA would be offered
|
||||
(a session can still fall back to the processor). Loads no model.
|
||||
"""
|
||||
tagger = self._tagger
|
||||
if tagger is not None and tagger[0] == "wd14":
|
||||
providers = tagger[1][0].get_providers()
|
||||
return "gpu" if providers[:1] == ["CUDAExecutionProvider"] else "cpu"
|
||||
try:
|
||||
import onnxruntime as ort
|
||||
except ImportError:
|
||||
return "cpu"
|
||||
available = ort.get_available_providers()
|
||||
return "gpu" if "CUDAExecutionProvider" in available else "cpu"
|
||||
|
||||
def is_model_loaded(self) -> bool:
|
||||
"""Check if a model is currently loaded."""
|
||||
return self._tagger is not None
|
||||
|
||||
+4
-2
@@ -573,6 +573,7 @@ try:
|
||||
MAX_INSTRUCTION_CHARS,
|
||||
MODEL_TYPES,
|
||||
WD14_MODEL_TYPES,
|
||||
validate_saved_threshold,
|
||||
validate_template,
|
||||
validate_threshold,
|
||||
)
|
||||
@@ -585,6 +586,7 @@ except ImportError:
|
||||
MAX_INSTRUCTION_CHARS,
|
||||
MODEL_TYPES,
|
||||
WD14_MODEL_TYPES,
|
||||
validate_saved_threshold,
|
||||
validate_template,
|
||||
validate_threshold,
|
||||
)
|
||||
@@ -631,12 +633,12 @@ class AutotagConfig:
|
||||
"existing": ("EXISTING", _valid_existing, "skip"),
|
||||
"general_threshold": (
|
||||
"GENERAL_THRESHOLD",
|
||||
lambda v: validate_threshold("value", v),
|
||||
lambda v: validate_saved_threshold("value", v),
|
||||
0.35,
|
||||
),
|
||||
"character_threshold": (
|
||||
"CHARACTER_THRESHOLD",
|
||||
lambda v: validate_threshold("value", v),
|
||||
lambda v: validate_saved_threshold("value", v),
|
||||
0.85,
|
||||
),
|
||||
"template": ("TEMPLATE", validate_template, DEFAULT_TAG_TEMPLATE),
|
||||
|
||||
+13
-10
@@ -12,9 +12,10 @@ last writes to the database:
|
||||
the sidecars (``discover``). In ``alongside`` mode a sidecar beside its
|
||||
model is kept when ``<base>.safetensors`` exists; in ``centralized`` mode
|
||||
every sidecar under the sidecar root is kept when its ``file_path`` names
|
||||
a model inside a LoRA root. ``fingerprint`` covers the kept sidecars (path,
|
||||
size and modification time) and the layout; ``preview`` adds the counts a
|
||||
review shows. Nothing is written and nothing is requested.
|
||||
a model inside a LoRA root. ``fingerprint`` covers every scanned sidecar,
|
||||
kept or skipped (path, size, modification time, kept or not, #362) and the
|
||||
layout; ``preview`` adds the counts a review shows. Nothing is written
|
||||
and nothing is requested.
|
||||
2. **stage**: read each sidecar and build its record. Its LoraManager preview
|
||||
and example images are linked where they are (``local_media``); only a
|
||||
LoRA without any local image downloads its CivitAI example images into the
|
||||
@@ -69,9 +70,10 @@ def discover(lora_manager_path: Optional[str]) -> Discovery:
|
||||
|
||||
|
||||
def fingerprint(discovery: Discovery) -> str:
|
||||
"""What a review saw: the layout, then each kept sidecar's path, size and
|
||||
modification time (a LoraManager save that changes unrelated settings
|
||||
leaves it as it was)."""
|
||||
"""What a review saw: the layout, then every scanned sidecar, kept or
|
||||
skipped, with its path, size, modification time and whether it was kept
|
||||
(#362). A LoraManager save that changes unrelated settings leaves it as
|
||||
it was."""
|
||||
layout = discovery.layout
|
||||
digest = hashlib.sha256()
|
||||
for value in (
|
||||
@@ -87,13 +89,14 @@ def fingerprint(discovery: Discovery) -> str:
|
||||
digest.update(f"{value}\n".encode("utf-8", "surrogatepass"))
|
||||
for name in discovery.unreadable:
|
||||
digest.update(f"unreadable\0{name}\n".encode("utf-8", "surrogatepass"))
|
||||
for sidecar in discovery.sidecars:
|
||||
path = sidecar.metadata_path
|
||||
kept = {str(sidecar.metadata_path) for sidecar in discovery.sidecars}
|
||||
for path in discovery.scanned_files:
|
||||
flag = 1 if str(path) in kept else 0
|
||||
try:
|
||||
stat = path.stat()
|
||||
entry = f"{path}\0{stat.st_size}\0{stat.st_mtime_ns}\n"
|
||||
entry = f"{path}\0{stat.st_size}\0{stat.st_mtime_ns}\0{flag}\n"
|
||||
except OSError:
|
||||
entry = f"{path}\0missing\n"
|
||||
entry = f"{path}\0missing\0{flag}\n"
|
||||
digest.update(entry.encode("utf-8", "surrogatepass"))
|
||||
return "sha256:" + digest.hexdigest()
|
||||
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
"""A per-process cache of LoraManager's layout for the image hot paths (#362).
|
||||
|
||||
Every Library list, prompt detail, inspect, reveal and serve request needs the
|
||||
LoRA allow-list (``ImagesMixin._lora_image_dirs``). Building it means finding
|
||||
the LoraManager install, reading its plugin settings file and its settings
|
||||
file, and resolving every folder. ``LayoutCache`` keeps the last result and
|
||||
reuses it while all of these hold:
|
||||
|
||||
- the key (the configured install path, stripped) and the home folder the
|
||||
folder guard compares against are the same;
|
||||
- the entry is younger than ``MAX_AGE_SECONDS`` (monotonic clock);
|
||||
- the plugin settings file and the settings file have the same signature
|
||||
(``st_mtime_ns``, ``st_size``, or ``MISSING``) as when it was stored;
|
||||
- the install and every stored media root still resolve to themselves (a
|
||||
folder swapped for a symlink reloads, so the folder guard runs again).
|
||||
|
||||
A result is stored only when no watched file is "racy" (modified within
|
||||
``RACY_SECONDS`` of the load, git's rule for racily clean index entries),
|
||||
every watched file could be checked, the settings file was readable, and no
|
||||
``clear()`` ran during the load. What each rule covers: signatures catch a
|
||||
settings change on the next request; the racy rule catches a write during the
|
||||
load, a half-written file and coarse timestamps; the maximum age bounds the
|
||||
inputs no signature sees (``extra_model_paths.yaml``, environment, temporary
|
||||
failures); ``clear()`` makes a finished import's new folders servable at once.
|
||||
|
||||
Hits and loads take no lock: a slow ``os.stat`` on a network share holds up
|
||||
only its own request, and two threads that miss together both load (both
|
||||
loads only read). A short lock guards the compare-and-store and the logging
|
||||
state; no file I/O runs under it. Standard library only, so it is tested
|
||||
without ComfyUI or LoraManager.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
from .lora_manager_layout import UNREADABLE, LoraManagerLayout, _canonical
|
||||
|
||||
# A watched file modified this recently is not trusted: LoraManager writes its
|
||||
# settings in place, and FAT/exFAT store modification times in 2 s steps.
|
||||
RACY_SECONDS = 3
|
||||
# Bounds what no signature sees (unwatched inputs, temporary failures).
|
||||
MAX_AGE_SECONDS = 30
|
||||
|
||||
# The signature of a watched file that does not exist (a valid state).
|
||||
MISSING = object()
|
||||
# ERROR_BAD_NETPATH and ERROR_BAD_NET_NAME, which Windows raises as
|
||||
# FileNotFoundError.
|
||||
_OFFLINE_SHARE_ERRORS = frozenset({53, 67})
|
||||
|
||||
logger = logging.getLogger("prompt_manager.lora_layout_cache")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Entry:
|
||||
key: str
|
||||
home: str
|
||||
install: Optional[str]
|
||||
layout: LoraManagerLayout
|
||||
signatures: Tuple[Tuple[str, object], ...]
|
||||
stored_at: float
|
||||
|
||||
|
||||
class _CannotValidate(Exception):
|
||||
"""A watched file could not be checked (not merely missing)."""
|
||||
|
||||
|
||||
class LayoutCache:
|
||||
"""At most one ``(install, layout)`` for the active LoraManager config."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
resolve: Callable[[str], Optional[str]],
|
||||
load: Callable[[Optional[str]], LoraManagerLayout],
|
||||
*,
|
||||
wall_ns: Callable[[], int] = time.time_ns,
|
||||
monotonic: Callable[[], float] = time.monotonic,
|
||||
stat=os.stat,
|
||||
pathmod=os.path,
|
||||
home: Callable[[], str] = lambda: str(Path.home()),
|
||||
):
|
||||
self._resolve = resolve
|
||||
self._load = load
|
||||
self._wall_ns = wall_ns
|
||||
self._monotonic = monotonic
|
||||
self._stat = stat
|
||||
self._pathmod = pathmod
|
||||
self._home = home
|
||||
self._entry: Optional[_Entry] = None
|
||||
self._generation = 0
|
||||
self._lock = threading.Lock()
|
||||
# What was last logged; survives clear() so a reload of the same
|
||||
# layout stays quiet.
|
||||
self._logged_layout: Optional[LoraManagerLayout] = None
|
||||
self._logged_refused: Tuple[str, ...] = ()
|
||||
|
||||
def get(self, configured: str) -> Tuple[Optional[str], LoraManagerLayout]:
|
||||
"""``(install, layout)`` for the configured path ("" auto-detects)."""
|
||||
key = (configured or "").strip()
|
||||
home = self._home()
|
||||
entry = self._entry
|
||||
if entry is not None and self._valid(entry, key, home):
|
||||
return entry.install, entry.layout
|
||||
with self._lock:
|
||||
if entry is not None and self._entry is entry:
|
||||
self._entry = None
|
||||
generation = self._generation
|
||||
install = self._resolve(key)
|
||||
layout = self._load(install)
|
||||
self._log_refused(layout)
|
||||
try:
|
||||
signatures = tuple(
|
||||
(path, self._signature(path)) for path in self._watched(install, layout)
|
||||
)
|
||||
except _CannotValidate:
|
||||
return install, layout
|
||||
if layout.error == UNREADABLE or self._racy(signatures, self._wall_ns()):
|
||||
return install, layout
|
||||
stored = _Entry(key, home, install, layout, signatures, self._monotonic())
|
||||
changed = False
|
||||
with self._lock:
|
||||
if self._generation == generation:
|
||||
self._entry = stored
|
||||
changed = layout != self._logged_layout
|
||||
if changed:
|
||||
self._logged_layout = layout
|
||||
if changed:
|
||||
logger.info(
|
||||
f"LoraManager layout changed: {layout.source or 'no'} settings, "
|
||||
f"{layout.mode} storage, {len(layout.media_roots)} media folders"
|
||||
)
|
||||
return install, layout
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Drop the entry; a load that started before this stores nothing."""
|
||||
with self._lock:
|
||||
self._entry = None
|
||||
self._generation += 1
|
||||
|
||||
def _log_refused(self, layout: LoraManagerLayout) -> None:
|
||||
"""One warning per default LoRA folder newly refused by the guard."""
|
||||
refused = tuple(layout.refused_defaults)
|
||||
with self._lock:
|
||||
if refused == self._logged_refused:
|
||||
return
|
||||
new = [folder for folder in refused if folder not in self._logged_refused]
|
||||
self._logged_refused = refused
|
||||
for folder in new:
|
||||
logger.warning(
|
||||
f"ComfyUI LoRA folder '{folder}' is neither scanned nor served: "
|
||||
"it is a filesystem root, the home folder or a folder that "
|
||||
"contains it"
|
||||
)
|
||||
|
||||
# ── validation ──────────────────────────────────────────────────
|
||||
|
||||
def _valid(self, entry: _Entry, key: str, home: str) -> bool:
|
||||
if entry.key != key or entry.home != home:
|
||||
return False
|
||||
if self._monotonic() - entry.stored_at >= MAX_AGE_SECONDS:
|
||||
return False
|
||||
try:
|
||||
for path, signature in entry.signatures:
|
||||
if self._signature(path) != signature:
|
||||
return False
|
||||
except _CannotValidate:
|
||||
return False
|
||||
folders = entry.layout.media_roots
|
||||
if entry.install is not None:
|
||||
folders = (entry.install,) + tuple(folders)
|
||||
for folder in folders:
|
||||
try:
|
||||
if _canonical(folder, self._pathmod) != folder:
|
||||
return False
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _watched(self, install, layout) -> Tuple[str, ...]:
|
||||
"""The plugin settings file, then the settings file, each once."""
|
||||
if install is None:
|
||||
return ()
|
||||
paths = [self._pathmod.join(install, "settings.json")]
|
||||
if layout.settings_path is not None:
|
||||
paths.append(str(layout.settings_path))
|
||||
watched, seen = [], set()
|
||||
for path in paths:
|
||||
identity = self._pathmod.normcase(self._pathmod.abspath(path))
|
||||
if identity not in seen:
|
||||
seen.add(identity)
|
||||
watched.append(path)
|
||||
return tuple(watched)
|
||||
|
||||
def _signature(self, path: str):
|
||||
try:
|
||||
result = self._stat(path)
|
||||
except (FileNotFoundError, NotADirectoryError) as error:
|
||||
# Windows reports an offline share as "not found": that is a
|
||||
# temporary failure, not a missing file.
|
||||
if getattr(error, "winerror", None) in _OFFLINE_SHARE_ERRORS:
|
||||
raise _CannotValidate(path) from error
|
||||
return MISSING
|
||||
except (OSError, ValueError) as error:
|
||||
raise _CannotValidate(path) from error
|
||||
return (result.st_mtime_ns, result.st_size)
|
||||
|
||||
@staticmethod
|
||||
def _racy(signatures, now_ns: int) -> bool:
|
||||
threshold = now_ns - RACY_SECONDS * 10**9
|
||||
return any(
|
||||
signature is not MISSING and signature[0] > threshold
|
||||
for _path, signature in signatures
|
||||
)
|
||||
+65
-14
@@ -12,8 +12,10 @@ server share one definition:
|
||||
plugin file, or the user config folder, in LoraManager's order), keeps only
|
||||
the seven keys the import needs, and resolves the LoRA roots, the sidecar
|
||||
root (centralized storage), the example-images root and the active
|
||||
LoraManager library. Every folder from the settings passes the folder
|
||||
guard (exists, not a filesystem root, not the home folder itself).
|
||||
LoraManager library. Every folder from the settings, and each of
|
||||
ComfyUI's default LoRA folders, passes the folder guard: it must exist and
|
||||
must not be a filesystem root, the home folder or a folder that contains
|
||||
it. Default folders the guard refuses are listed in ``refused_defaults``.
|
||||
- ``discover_sidecars`` finds every sidecar whose model file exists inside a
|
||||
LoRA root: beside the models in ``alongside`` mode, under the sidecar root
|
||||
(matched to its model by ``file_path``) in ``centralized`` mode.
|
||||
@@ -25,6 +27,7 @@ it is testable outside ComfyUI.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass, replace
|
||||
@@ -98,7 +101,12 @@ NOT_OBJECT = "the settings file is not a JSON object"
|
||||
TOO_LARGE = "the settings file is larger than 5 MB"
|
||||
NO_PLATFORMDIRS = "platformdirs is not available"
|
||||
NO_SIDECAR_ROOT = "the sidecar folder does not exist"
|
||||
BAD_SIDECAR_ROOT = "the sidecar folder is a filesystem root or the home folder"
|
||||
BAD_SIDECAR_ROOT = (
|
||||
"the sidecar folder is a filesystem root, the home folder or a folder "
|
||||
"that contains it"
|
||||
)
|
||||
|
||||
logger = logging.getLogger("prompt_manager.lora_manager_layout")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -114,6 +122,9 @@ class LoraManagerLayout:
|
||||
example_images_root: Optional[str]
|
||||
library: str
|
||||
media_roots: Tuple[str, ...]
|
||||
# ComfyUI's default LoRA folders that exist but that the folder guard
|
||||
# refused (canonical): the layout cache logs them (#362).
|
||||
refused_defaults: Tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -126,13 +137,20 @@ class Sidecar:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Discovery:
|
||||
"""The sidecars kept, how many metadata files were seen, and those unread."""
|
||||
"""The sidecars kept, every metadata file seen, and those unread."""
|
||||
|
||||
layout: LoraManagerLayout
|
||||
sidecars: List[Sidecar]
|
||||
scanned: int
|
||||
# Every ``*.metadata.json`` seen, kept or skipped: resolved, once each,
|
||||
# sorted by ``str(path)``. The import's fingerprint covers all of them.
|
||||
scanned_files: Tuple[Path, ...]
|
||||
unreadable: List[str]
|
||||
|
||||
@property
|
||||
def scanned(self) -> int:
|
||||
"""How many metadata files were seen."""
|
||||
return len(self.scanned_files)
|
||||
|
||||
|
||||
class _SettingsError(Exception):
|
||||
"""The settings file cannot be used, for the reason it carries."""
|
||||
@@ -315,6 +333,23 @@ def _guarded(paths, home: str) -> Tuple[str, ...]:
|
||||
return tuple(kept)
|
||||
|
||||
|
||||
def _refused(paths, home: str) -> Tuple[str, ...]:
|
||||
"""The folders in ``paths`` that exist but that the guard refuses."""
|
||||
refused = []
|
||||
for path in paths:
|
||||
absolute = _absolute(path)
|
||||
if not absolute or not os.path.isdir(absolute):
|
||||
continue
|
||||
if _guard_folder(absolute, home) is None:
|
||||
try:
|
||||
canonical = _canonical(absolute)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if canonical not in refused:
|
||||
refused.append(canonical)
|
||||
return tuple(refused)
|
||||
|
||||
|
||||
def _example_images_root(
|
||||
settings: Mapping, library: str, libraries: int, home: str
|
||||
) -> Optional[str]:
|
||||
@@ -351,7 +386,9 @@ def load_layout(
|
||||
default LoRA roots, so serving and trigger words keep working.
|
||||
"""
|
||||
home = str(Path.home())
|
||||
defaults = _guarded(default_roots(), home)
|
||||
default_list = list(default_roots())
|
||||
defaults = _guarded(default_list, home)
|
||||
refused_defaults = _refused(default_list, home)
|
||||
fallback = LoraManagerLayout(
|
||||
settings_path=None,
|
||||
source=None,
|
||||
@@ -362,6 +399,7 @@ def load_layout(
|
||||
example_images_root=None,
|
||||
library="default",
|
||||
media_roots=defaults,
|
||||
refused_defaults=refused_defaults,
|
||||
)
|
||||
if not lora_manager_path:
|
||||
return fallback
|
||||
@@ -377,10 +415,14 @@ def load_layout(
|
||||
except _SettingsError as error:
|
||||
return replace(fallback, error=str(error))
|
||||
settings = _settings_subset(raw or {})
|
||||
return _layout_from_settings(settings, settings_path, source, defaults, home)
|
||||
return _layout_from_settings(
|
||||
settings, settings_path, source, defaults, home, refused_defaults
|
||||
)
|
||||
|
||||
|
||||
def _layout_from_settings(settings, settings_path, source, defaults, home):
|
||||
def _layout_from_settings(
|
||||
settings, settings_path, source, defaults, home, refused_defaults=()
|
||||
):
|
||||
library, entry, libraries = _active_library(settings)
|
||||
lora_roots = _media_roots(
|
||||
defaults,
|
||||
@@ -400,6 +442,7 @@ def _layout_from_settings(settings, settings_path, source, defaults, home):
|
||||
example_images_root=example_root,
|
||||
library=library,
|
||||
media_roots=_media_roots(lora_roots, [example_root]),
|
||||
refused_defaults=refused_defaults,
|
||||
)
|
||||
if mode != MODE_CENTRALIZED:
|
||||
return layout
|
||||
@@ -482,13 +525,13 @@ def discover_sidecars(layout: LoraManagerLayout) -> Discovery:
|
||||
cannot see every folder must not replace the collection with part of it.
|
||||
"""
|
||||
if layout.error is not None:
|
||||
return Discovery(layout, [], 0, [])
|
||||
return Discovery(layout, [], (), [])
|
||||
if layout.mode == MODE_CENTRALIZED:
|
||||
folders = [layout.sidecar_root]
|
||||
else:
|
||||
folders = list(layout.lora_roots)
|
||||
seen = set()
|
||||
scanned = 0
|
||||
scanned: List[Path] = []
|
||||
unreadable: List[str] = []
|
||||
sidecars = []
|
||||
for folder in folders:
|
||||
@@ -498,12 +541,13 @@ def discover_sidecars(layout: LoraManagerLayout) -> Discovery:
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
scanned += 1
|
||||
scanned.append(resolved)
|
||||
model = _model_for(resolved, layout, unreadable)
|
||||
if model is not None:
|
||||
sidecars.append(Sidecar(resolved, Path(os.path.realpath(model))))
|
||||
sidecars.sort(key=lambda sidecar: str(sidecar.metadata_path))
|
||||
return Discovery(layout, sidecars, scanned, unreadable)
|
||||
scanned.sort(key=str)
|
||||
return Discovery(layout, sidecars, tuple(scanned), unreadable)
|
||||
|
||||
|
||||
# ── images ──────────────────────────────────────────────────────────
|
||||
@@ -526,8 +570,15 @@ def _example_files(folder: str) -> List[str]:
|
||||
def _preview_candidates(sidecar: Sidecar, metadata: Dict) -> Iterator[str]:
|
||||
"""The paths a preview is looked for at, in LoraManager's order."""
|
||||
url = metadata.get("preview_url")
|
||||
if isinstance(url, str) and url.strip() and os.path.isabs(url.strip()):
|
||||
yield url.strip()
|
||||
if isinstance(url, str) and url.strip():
|
||||
if os.path.isabs(url.strip()):
|
||||
yield url.strip()
|
||||
else:
|
||||
# LoraManager writes absolute paths; it would resolve a relative
|
||||
# one against its working directory, not the sidecar folder.
|
||||
logger.debug(
|
||||
"Relative preview_url skipped in %s", sidecar.metadata_path.name
|
||||
)
|
||||
base = sidecar.metadata_path.name[: -len(METADATA_SUFFIX)]
|
||||
for suffix in PREVIEW_EXTENSIONS:
|
||||
yield str(sidecar.metadata_path.parent / f"{base}{suffix}")
|
||||
|
||||
+40
-6
@@ -30,6 +30,7 @@ except ImportError:
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
from .lora_layout_cache import LayoutCache
|
||||
from .lora_manager_layout import (
|
||||
MODE_ALONGSIDE,
|
||||
METADATA_SUFFIX,
|
||||
@@ -295,10 +296,35 @@ def layout_for_install(install: Optional[str]) -> LoraManagerLayout:
|
||||
|
||||
|
||||
def lora_manager_layout(lora_manager_path: Optional[str]) -> LoraManagerLayout:
|
||||
"""Where LoraManager keeps its files (``py/lora_manager_layout.py``)."""
|
||||
"""Where LoraManager keeps its files (``py/lora_manager_layout.py``).
|
||||
|
||||
A fresh read: the import review, Start, the legacy scan and the trigger
|
||||
word cache use it. The image hot paths use ``cached_lora_manager_layout``.
|
||||
"""
|
||||
return layout_for_install(lora_manager_install(lora_manager_path))
|
||||
|
||||
|
||||
# The image hot paths' layout cache (#362). The lambdas look the functions up
|
||||
# when called, so a test that patches them here is seen by the cache.
|
||||
_layout_cache = LayoutCache(
|
||||
lambda path: lora_manager_install(path),
|
||||
lambda install: layout_for_install(install),
|
||||
)
|
||||
|
||||
|
||||
def cached_lora_manager_layout(
|
||||
configured: Optional[str],
|
||||
) -> Tuple[Optional[str], LoraManagerLayout]:
|
||||
"""``(install, layout)`` for the configured path, from the layout cache
|
||||
while LoraManager's settings files are unchanged (``py/lora_layout_cache.py``)."""
|
||||
return _layout_cache.get(configured or "")
|
||||
|
||||
|
||||
def clear_layout_cache() -> None:
|
||||
"""Drop the cached layout (after an import linked files in new folders)."""
|
||||
_layout_cache.clear()
|
||||
|
||||
|
||||
def _get_extra_lora_paths(comfyui_root: Path) -> List[Path]:
|
||||
"""Parse extra_model_paths.yaml for additional LoRA directories."""
|
||||
results = []
|
||||
@@ -342,7 +368,7 @@ def read_lora_metadata(metadata_path: Path) -> Optional[Dict]:
|
||||
try:
|
||||
with open(metadata_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
except (ValueError, OSError) as e: # JSONDecodeError, UnicodeDecodeError
|
||||
logger.debug(f"Failed to read {metadata_path}: {e}")
|
||||
return None
|
||||
|
||||
@@ -857,11 +883,13 @@ class TriggerWordCache:
|
||||
"""
|
||||
try:
|
||||
new_cache = self._build(lora_manager_path)
|
||||
except Exception as error:
|
||||
except OSError as error:
|
||||
logger.warning(f"Trigger word cache not refreshed: {error}")
|
||||
with self._lock:
|
||||
self._failed_at = time.monotonic()
|
||||
return len(self._cache)
|
||||
return self._failed()
|
||||
except Exception:
|
||||
# Not a disk error: keep the traceback (#362).
|
||||
logger.exception("Trigger word cache not refreshed")
|
||||
return self._failed()
|
||||
|
||||
with self._lock:
|
||||
self._cache = new_cache
|
||||
@@ -873,6 +901,12 @@ class TriggerWordCache:
|
||||
)
|
||||
return len(new_cache)
|
||||
|
||||
def _failed(self) -> int:
|
||||
"""Keep the previous map and note when this load failed."""
|
||||
with self._lock:
|
||||
self._failed_at = time.monotonic()
|
||||
return len(self._cache)
|
||||
|
||||
@staticmethod
|
||||
def _build(lora_manager_path: str) -> Dict[str, List[str]]:
|
||||
layout = lora_manager_layout(lora_manager_path)
|
||||
|
||||
+36
-1
@@ -13,7 +13,23 @@ LLM_MODEL_TYPES = frozenset({"gguf", "hf"})
|
||||
MODEL_TYPES = WD14_MODEL_TYPES | LLM_MODEL_TYPES
|
||||
MAX_INSTRUCTION_CHARS = 4000
|
||||
MAX_INSTRUCTION_BYTES = 16 * 1024
|
||||
# What a batch does with a prompt that already has tags (#328).
|
||||
# Tag output images' review estimate (#379): seconds per image for the WD14
|
||||
# engines, by where ONNX Runtime runs them. The graphics card figure was
|
||||
# measured in PR 1's live check (RTX 4090, CUDA: Auto-tag selected on 100
|
||||
# images in 16.7 s with the model load, 0.167 s/image). The processor figure
|
||||
# is an upper bound from the maintainer's cancelled CPU run (19 min / 697
|
||||
# images, backup, model load and skips included), kept as the fallback. The
|
||||
# language-model engines have no figure.
|
||||
WD14_SECONDS_PER_IMAGE = 1.6
|
||||
WD14_GPU_SECONDS_PER_IMAGE = 0.17
|
||||
|
||||
|
||||
def wd14_seconds_per_image(device: str) -> float:
|
||||
"""The estimate's rate for ``"gpu"`` or ``"cpu"`` (anything else)."""
|
||||
return WD14_GPU_SECONDS_PER_IMAGE if device == "gpu" else WD14_SECONDS_PER_IMAGE
|
||||
|
||||
|
||||
# What a batch does with an image that already has tags (#328, #379).
|
||||
EXISTING_POLICIES = ("skip", "merge", "replace_after_review")
|
||||
|
||||
# The language-model tag template every page starts from until one is saved.
|
||||
@@ -39,6 +55,25 @@ def validate_threshold(name: str, value: Any) -> float:
|
||||
return float(value)
|
||||
|
||||
|
||||
# Every page shows the thresholds in steps of 0.05 (the concept's System rows,
|
||||
# the Tag image dialogs and the legacy sliders), so a saved one must be on that
|
||||
# grid (#369). Per-request thresholds keep their own checks.
|
||||
THRESHOLD_STEP = 0.05
|
||||
_STEPS_PER_UNIT = round(1 / THRESHOLD_STEP)
|
||||
# A computed float can miss the grid by a hair: (0.1 + 0.2) * 20 is 6.000000000000001.
|
||||
_STEP_TOLERANCE = 1e-12 # in steps; refuses 0.35000000001
|
||||
|
||||
|
||||
def validate_saved_threshold(name: str, value: Any) -> float:
|
||||
"""A saved WD14 threshold: in [0, 1] and a multiple of ``THRESHOLD_STEP``,
|
||||
returned exactly on the grid; else ValueError naming ``name``."""
|
||||
value = validate_threshold(name, value)
|
||||
steps = value * _STEPS_PER_UNIT
|
||||
if abs(steps - round(steps)) > _STEP_TOLERANCE:
|
||||
raise ValueError(f"{name} must be a multiple of {THRESHOLD_STEP}")
|
||||
return round(steps) / _STEPS_PER_UNIT
|
||||
|
||||
|
||||
def validate_template(value: Any, name: str = "instructions") -> str:
|
||||
"""Non-empty language-model instructions within the character and byte
|
||||
caps, or ValueError naming ``name``."""
|
||||
|
||||
@@ -8,8 +8,8 @@ coverage>=7.16.2
|
||||
numpy>=2.2.6 # WD14 engine tests
|
||||
|
||||
# Code quality
|
||||
black>=26.10.1
|
||||
flake8>=7.4.1
|
||||
black==26.10.1
|
||||
flake8==7.4.1
|
||||
|
||||
# Security scanning
|
||||
bandit[toml]>=1.9.4
|
||||
|
||||
@@ -676,3 +676,73 @@ export function measureInspector(selector) {
|
||||
primary:node.classList.contains('ca-primary'),minHeight:getComputedStyle(node).minHeight})),
|
||||
};
|
||||
}
|
||||
|
||||
// #249 System S7 Task 20, shared since S12 (#329): the maintenance plan dialog's measurement and its D100 mapping,
|
||||
// moved unchanged from system-maintenance-concept.spec.mjs so the rotated-log review uses the same ones.
|
||||
/** Runs in the page: the open review dialog (plan or duplicates): its title, body and the concept's tokens. */
|
||||
export function measureReview() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const body = dialog.querySelector('#modal-body');
|
||||
const grid = body.querySelector(':scope > dl.meta-grid');
|
||||
const compare = body.querySelector('.compare-grid');
|
||||
const item = compare?.querySelector('.compare-item');
|
||||
const notes = [...body.querySelectorAll(':scope > p.small')];
|
||||
const note = notes.at(-1);
|
||||
const actions = body.querySelector(':scope > .dialog-actions');
|
||||
return {
|
||||
title: text(dialog.querySelector('#modal-title')),
|
||||
tree: [...body.children].map(name),
|
||||
heading: body.querySelector(':scope > h3') && { text: text(body.querySelector(':scope > h3')),
|
||||
...css(body.querySelector(':scope > h3'), 'fontSize', 'fontWeight', 'color', 'marginBottom') },
|
||||
grid: grid && { rows: [...grid.children].map(row => [text(row.querySelector('dt')), text(row.querySelector('dd'))]),
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
dt: css(grid.querySelector('dt'), 'fontSize', 'color', 'textTransform'), dd: css(grid.querySelector('dd'), 'fontSize', 'color') },
|
||||
details: body.querySelector(':scope > details') && { summary: text(body.querySelector(':scope > details > summary')),
|
||||
tree: [...body.querySelector(':scope > details').children].map(name) },
|
||||
backup: body.querySelector(':scope > .notice') && { tree: [...body.querySelector(':scope > .notice').children].map(name),
|
||||
text: [...body.querySelectorAll(':scope > .notice > div:first-child > *')].map(node => [node.localName, text(node)]),
|
||||
buttons: [...body.querySelectorAll(':scope > .notice button')].map(text) },
|
||||
compare: compare && { tree: [...compare.children].map(name), ...css(compare, 'display', 'gridTemplateColumns', 'columnGap'),
|
||||
items: [...compare.children].map(entry => ({ tree: [...entry.children].map(name), label: text(entry.querySelector('label')),
|
||||
checked: entry.querySelector('input[type=radio]').checked, image: entry.querySelector('img')?.className ?? null })),
|
||||
item: css(item, 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius'),
|
||||
image: css(item.querySelector('img'), 'width', 'objectFit'),
|
||||
label: css(item.querySelector('label'), 'fontSize', 'color'),
|
||||
radio: { name: item.querySelector('input[type=radio]').name.startsWith('keep-') ? 'keep-*' : 'other',
|
||||
type: item.querySelector('input').type } },
|
||||
note: note && { text: text(note), ...css(note, 'fontSize', 'color', 'marginTop') },
|
||||
actions: { tree: [...actions.children].map(name), ...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(item => ({ text: text(item), className: item.className,
|
||||
...css(item, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor') })) },
|
||||
};
|
||||
}
|
||||
|
||||
export const BACKUP_BOX = Object.freeze({ tree: ['div', 'div.row-actions'], text: [['h3', 'Back up the database first (recommended)'],
|
||||
['p', 'Downloads a copy of the prompts database before this operation changes any records.']], buttons: ['Back up now'] });
|
||||
|
||||
/**
|
||||
* D100: the server's plan in the concept's dialog: the operation's row title as its `h3` (the concept prints its
|
||||
* `data-plan` name), the four rows with the plan's values, the listed records (`details`), the hold line in the
|
||||
* concept's `p.small`, the D55 backup box before the actions, and "Apply reviewed operation". No sample note.
|
||||
*/
|
||||
export function mapPlanConcept(reference, title, plan) {
|
||||
const backup = plan.backup.policy === 'off' ? 'No safety backup (the policy is off)'
|
||||
: `Create and verify before changes (${plan.backup.destination_label})`;
|
||||
const tree = ['h3', 'dl.meta-grid', ...(plan.details.length ? ['details'] : []), 'p.small', 'div.notice', 'div.dialog-actions'];
|
||||
const [cancel, apply] = reference.actions.buttons;
|
||||
return {
|
||||
...reference,
|
||||
tree,
|
||||
heading: { ...reference.heading, text: title },
|
||||
grid: { ...reference.grid, rows: [['Scope', plan.scope.label],
|
||||
['Protected', plan.protected.reasons.length ? plan.protected.reasons.join(' · ') : 'Nothing is protected'],
|
||||
['Backup', backup], ['Media on disk', plan.media.label]] },
|
||||
details: plan.details.length ? { summary: `Listed records (${plan.details.length})`, tree: ['summary', 'pre'] } : null,
|
||||
backup: BACKUP_BOX,
|
||||
note: { ...reference.note, text: 'Nothing changes until you apply. This review holds for 15 minutes.' },
|
||||
actions: { ...reference.actions, buttons: [cancel, { ...apply, text: 'Apply reviewed operation' }] },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
"""Run a browser-test fixture seeder off the live database, then publish once (#384).
|
||||
|
||||
The seeders make hundreds of small commits. On a slow CI disk the live
|
||||
database's commits queue behind WAL checkpoint syncs and a reset took over
|
||||
30 s. ``ScratchSeeder.run`` instead:
|
||||
|
||||
1. copies the live database into a scratch database (SQLite backup API) whose
|
||||
connections never sync (a crash loses only the copy);
|
||||
2. runs the unchanged seeder against an API proxy whose ``db`` is the scratch
|
||||
database; everything else (files, thumbnails, caches, command state) is
|
||||
the live API's, written exactly as before;
|
||||
3. replaces the rows of every table the seed changed in the live database, in
|
||||
one transaction on the live connection. Tables the seed left alone are not
|
||||
touched, so state another fixture owns stays as it was.
|
||||
|
||||
A commit by any other live connection while the seed ran fails the reset
|
||||
instead of being overwritten (``PRAGMA data_version``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import weakref
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, TypeVar
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
# Set to 1 to run seeders on the live database as before #384 (the equivalence
|
||||
# proof, tests/browser/reset_equivalence.py, compares both ways).
|
||||
SEED_ON_LIVE_ENV = "PM_TEST_SEED_ON_LIVE"
|
||||
SCRATCH_SCHEMA = "reset_seed"
|
||||
# API calls a seed makes that must see the published rows (see _SeedApi).
|
||||
DEFERRED_CALLS = frozenset({"invalidate_gallery_cache"})
|
||||
# Linux's shared-memory tmpfs (present in the Playwright image and on runners).
|
||||
MEMORY_ROOT = Path("/dev/shm")
|
||||
# The largest fixtures' scratch copy is about 8 MB (database plus a WAL that
|
||||
# autocheckpoints near 4 MB). Docker's default /dev/shm is 64 MB, shared with
|
||||
# the browsers and the other workers; with less free than this, the scratch
|
||||
# copy goes on disk instead.
|
||||
MEMORY_FREE_BYTES = 32 * 1024 * 1024
|
||||
# Scratch folders are named <prefix><owner pid>-<random>.
|
||||
SCRATCH_PREFIX = "pm-reset-"
|
||||
|
||||
|
||||
def _quote(name: str) -> str:
|
||||
return '"' + name.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def _tables(conn: sqlite3.Connection, schema: str = "main") -> list[str]:
|
||||
rows = conn.execute(
|
||||
f"SELECT name, sql FROM {schema}.sqlite_master WHERE type = 'table' "
|
||||
"ORDER BY name"
|
||||
).fetchall()
|
||||
for name, sql in rows:
|
||||
if sql and sql.lstrip().upper().startswith("CREATE VIRTUAL"):
|
||||
raise NotImplementedError(f"virtual table {name} cannot be copied")
|
||||
return [name for name, _sql in rows]
|
||||
|
||||
|
||||
def _columns(conn: sqlite3.Connection, table: str, schema: str = "main") -> list[str]:
|
||||
"""Writable columns (hidden and generated ones excluded), plus ``rowid``
|
||||
when the table has a rowid that no INTEGER PRIMARY KEY column carries."""
|
||||
info = conn.execute(f"PRAGMA {schema}.table_xinfo({_quote(table)})").fetchall()
|
||||
columns = [row[1] for row in info if row[6] == 0]
|
||||
keys = [row for row in info if row[5]]
|
||||
sql = conn.execute(
|
||||
f"SELECT sql FROM {schema}.sqlite_master WHERE type = 'table' AND name = ?",
|
||||
(table,),
|
||||
).fetchone()[0]
|
||||
without_rowid = sql is not None and "WITHOUT ROWID" in sql.upper()
|
||||
integer_key = len(keys) == 1 and str(keys[0][2]).upper() == "INTEGER"
|
||||
if not without_rowid and not integer_key and table != "sqlite_sequence":
|
||||
columns.insert(0, "rowid")
|
||||
return columns
|
||||
|
||||
|
||||
def table_digests(conn: sqlite3.Connection, schema: str = "main") -> dict[str, str]:
|
||||
"""A digest of every table's rows (rowids included), by table name."""
|
||||
result = {}
|
||||
for table in _tables(conn, schema):
|
||||
selected = ", ".join(
|
||||
_quote(c) if c != "rowid" else "rowid"
|
||||
for c in _columns(conn, table, schema)
|
||||
)
|
||||
digest = hashlib.sha256()
|
||||
rows = conn.execute(f"SELECT {selected} FROM {schema}.{_quote(table)}")
|
||||
for row in sorted(repr(tuple(row)) for row in rows):
|
||||
digest.update(row.encode("utf-8", "surrogatepass"))
|
||||
digest.update(b"\n")
|
||||
result[table] = digest.hexdigest()
|
||||
return result
|
||||
|
||||
|
||||
def _process_alive(pid: int) -> bool:
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except OSError:
|
||||
return True # it exists but belongs to someone else
|
||||
return True
|
||||
|
||||
|
||||
def remove_orphaned_scratch(root: Path) -> None:
|
||||
"""Remove scratch folders of servers that died without cleaning up.
|
||||
|
||||
A server killed outright (SIGKILL, a crash) never runs ``close``, and
|
||||
tmpfs keeps its folder in memory until reboot. The folder name carries
|
||||
the owner's process id; a folder whose process is gone is removed.
|
||||
"""
|
||||
if os.name != "posix":
|
||||
return # os.kill(pid, 0) would terminate the process on Windows
|
||||
for folder in root.glob(f"{SCRATCH_PREFIX}*-*"):
|
||||
pid = folder.name[len(SCRATCH_PREFIX) :].split("-", 1)[0]
|
||||
if pid.isdigit() and not _process_alive(int(pid)):
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
|
||||
|
||||
class _SeedApi:
|
||||
"""The live API, except that ``db`` is the scratch database.
|
||||
|
||||
Cache drops the seed asks for (``DEFERRED_CALLS``) are recorded and made
|
||||
after the publish: dropped earlier, a request in between would cache the
|
||||
live database's old rows again.
|
||||
"""
|
||||
|
||||
def __init__(self, api: Any, db: Any) -> None:
|
||||
object.__setattr__(self, "_api", api)
|
||||
object.__setattr__(self, "db", db)
|
||||
object.__setattr__(self, "deferred", [])
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
if name in DEFERRED_CALLS:
|
||||
|
||||
def record(*args: Any, **kwargs: Any) -> None:
|
||||
self.deferred.append((name, args, kwargs))
|
||||
|
||||
return record
|
||||
return getattr(self._api, name)
|
||||
|
||||
def __setattr__(self, name: str, value: Any) -> None:
|
||||
setattr(self._api, name, value)
|
||||
|
||||
|
||||
class ScratchSeeder:
|
||||
"""Seed into a scratch copy and publish the changed tables in one commit."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api: Any,
|
||||
directory: Path,
|
||||
*,
|
||||
on_live: bool = False,
|
||||
memory_root: Path | None = MEMORY_ROOT,
|
||||
) -> None:
|
||||
self._api = api
|
||||
self._fallback = Path(directory)
|
||||
self._memory_root = memory_root
|
||||
self._directory: Path | None = None
|
||||
self._on_live = on_live
|
||||
self._scratch = None
|
||||
self._unsynced: "weakref.WeakSet[sqlite3.Connection]" = weakref.WeakSet()
|
||||
|
||||
def run(self, seed: Callable[[Any], T]) -> T:
|
||||
"""Run ``seed(api)`` (blocking; call it in an executor thread)."""
|
||||
if self._on_live:
|
||||
return seed(self._api)
|
||||
live = self._api.db.model.get_connection()
|
||||
if live.in_transaction:
|
||||
raise RuntimeError("the live connection has an open transaction")
|
||||
version = live.execute("PRAGMA data_version").fetchone()[0]
|
||||
scratch = self._scratch_database()
|
||||
target = self._scratch_connection(scratch)
|
||||
with target.using():
|
||||
live.backup(target)
|
||||
before = table_digests(target)
|
||||
target_api = _SeedApi(self._api, scratch)
|
||||
result = seed(target_api)
|
||||
target = self._scratch_connection(scratch)
|
||||
if target.in_transaction:
|
||||
raise RuntimeError("the seed left a scratch transaction open")
|
||||
after = table_digests(target)
|
||||
changed = [table for table in after if after[table] != before.get(table)]
|
||||
if set(before) != set(after):
|
||||
raise RuntimeError("the seed changed the database schema")
|
||||
if live.execute("PRAGMA data_version").fetchone()[0] != version:
|
||||
raise RuntimeError("the live database changed while the seed ran")
|
||||
self._publish(live, scratch.model.db_path, changed, version)
|
||||
for name, args, kwargs in target_api.deferred:
|
||||
getattr(self._api, name)(*args, **kwargs)
|
||||
return result
|
||||
|
||||
def _scratch_database(self):
|
||||
if self._scratch is None:
|
||||
from database.operations import PromptDatabase
|
||||
|
||||
self._directory = self._scratch_directory()
|
||||
self._scratch = PromptDatabase(str(self._directory / "seed.db"))
|
||||
return self._scratch
|
||||
|
||||
def _scratch_connection(self, scratch) -> sqlite3.Connection:
|
||||
"""The calling thread's scratch connection, set never to sync."""
|
||||
conn = scratch.model.get_connection()
|
||||
if conn not in self._unsynced:
|
||||
conn.execute("PRAGMA synchronous = OFF")
|
||||
self._unsynced.add(conn)
|
||||
return conn
|
||||
|
||||
@staticmethod
|
||||
def _publish(
|
||||
live: sqlite3.Connection, path: str, tables: list[str], version: int
|
||||
) -> None:
|
||||
"""Replace ``tables`` with the scratch rows in one live transaction.
|
||||
|
||||
``version`` is the live ``data_version`` read before the seed. It is
|
||||
checked again under the write lock: a writer whose transaction was
|
||||
open during the seed commits before ``BEGIN IMMEDIATE`` returns, and
|
||||
its rows must fail the reset instead of being overwritten.
|
||||
"""
|
||||
if not tables:
|
||||
return
|
||||
foreign_keys = live.execute("PRAGMA foreign_keys").fetchone()[0]
|
||||
# Rows are copied as they are; no cascade may reach a table left alone.
|
||||
live.execute("PRAGMA foreign_keys = OFF")
|
||||
try:
|
||||
live.execute(f"ATTACH DATABASE ? AS {SCRATCH_SCHEMA}", (path,))
|
||||
try:
|
||||
live.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
if live.execute("PRAGMA data_version").fetchone()[0] != version:
|
||||
raise RuntimeError(
|
||||
"the live database changed while the seed ran"
|
||||
)
|
||||
for table in tables:
|
||||
columns = ", ".join(
|
||||
_quote(c) if c != "rowid" else "rowid"
|
||||
for c in _columns(live, table)
|
||||
)
|
||||
name = _quote(table)
|
||||
live.execute(f"DELETE FROM main.{name}")
|
||||
live.execute(
|
||||
f"INSERT INTO main.{name} ({columns}) "
|
||||
f"SELECT {columns} FROM {SCRATCH_SCHEMA}.{name}"
|
||||
)
|
||||
live.commit()
|
||||
except BaseException:
|
||||
live.rollback()
|
||||
raise
|
||||
finally:
|
||||
live.execute(f"DETACH DATABASE {SCRATCH_SCHEMA}")
|
||||
finally:
|
||||
live.execute(f"PRAGMA foreign_keys = {int(foreign_keys)}")
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close every scratch connection, then remove the scratch files."""
|
||||
if self._on_live:
|
||||
return
|
||||
if self._scratch is not None:
|
||||
self._scratch.close_all()
|
||||
self._scratch = None
|
||||
if self._directory is not None:
|
||||
shutil.rmtree(self._directory, ignore_errors=True)
|
||||
self._directory = None
|
||||
|
||||
def _scratch_directory(self) -> Path:
|
||||
"""A folder in memory (tmpfs) where there is one, else ``directory``.
|
||||
|
||||
Even unsynced, scratch writes on the runner's disk wait whenever the
|
||||
kernel is writing back the pages they rewrite.
|
||||
"""
|
||||
root = self._memory_root
|
||||
if root is not None and root.is_dir() and os.access(root, os.W_OK):
|
||||
remove_orphaned_scratch(root)
|
||||
if shutil.disk_usage(root).free >= MEMORY_FREE_BYTES:
|
||||
prefix = f"{SCRATCH_PREFIX}{os.getpid()}-"
|
||||
return Path(tempfile.mkdtemp(prefix=prefix, dir=root))
|
||||
self._fallback.mkdir(parents=True, exist_ok=True)
|
||||
return self._fallback
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Empty a browser-test media folder without waiting for the disk (#384).
|
||||
|
||||
A reset used to ``shutil.rmtree`` the output and input folders the last
|
||||
fixture wrote. Unlinking a file whose pages are still being written back
|
||||
waits for that write, so on a slow CI disk a reset stalled in ``rmtree``.
|
||||
``replace_folder`` renames the old folder into ``<parent>/reset-trash`` (one
|
||||
metadata change) and makes a new empty one; a background thread deletes the
|
||||
trash, and ``close_trash`` stops that thread and removes what is left before
|
||||
the server's temp root goes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import shutil
|
||||
import threading
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
TRASH_NAME = "reset-trash"
|
||||
|
||||
|
||||
class _Trash:
|
||||
def __init__(self, directory: Path) -> None:
|
||||
self.directory = directory
|
||||
self._queue: "queue.SimpleQueue[Path | None]" = queue.SimpleQueue()
|
||||
self._thread = threading.Thread(
|
||||
target=self._work, name="pm-reset-trash", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def _work(self) -> None:
|
||||
while (folder := self._queue.get()) is not None:
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
|
||||
def discard(self, folder: Path) -> None:
|
||||
self.directory.mkdir(exist_ok=True)
|
||||
moved = self.directory / f"{folder.name}-{uuid.uuid4().hex}"
|
||||
folder.rename(moved)
|
||||
self._queue.put(moved)
|
||||
|
||||
def close(self) -> None:
|
||||
self._queue.put(None)
|
||||
self._thread.join()
|
||||
shutil.rmtree(self.directory, ignore_errors=True)
|
||||
|
||||
|
||||
_trashes: dict[Path, _Trash] = {}
|
||||
# Trash folders whose server has cleaned up: a reset still running then
|
||||
# deletes in place instead of starting a thread nobody would stop.
|
||||
_closed: set[Path] = set()
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def replace_folder(folder: Path) -> None:
|
||||
"""Make ``folder`` an empty folder; its old content is deleted later."""
|
||||
folder = Path(folder)
|
||||
if folder.exists():
|
||||
directory = folder.parent / TRASH_NAME
|
||||
with _lock:
|
||||
trash = _trashes.get(directory)
|
||||
if trash is None and directory not in _closed:
|
||||
trash = _trashes[directory] = _Trash(directory)
|
||||
if trash is not None:
|
||||
trash.discard(folder)
|
||||
if folder.exists():
|
||||
shutil.rmtree(folder)
|
||||
folder.mkdir()
|
||||
|
||||
|
||||
def close_trash(parent: Path) -> None:
|
||||
"""Stop deleting under ``parent``'s trash and remove it (blocking)."""
|
||||
directory = Path(parent) / TRASH_NAME
|
||||
with _lock:
|
||||
_closed.add(directory)
|
||||
trash = _trashes.pop(directory, None)
|
||||
if trash is not None:
|
||||
trash.close()
|
||||
@@ -286,7 +286,8 @@ const continuationText=page=>page.locator('.infinite-continuation > p').first().
|
||||
async function searchOf(page,term) {
|
||||
const before=await countText(page);
|
||||
await page.locator('#search').fill(term);
|
||||
await expect.poll(()=>countText(page)).not.toBe(before);
|
||||
// Production's caption reads "Loading images…" until the search's listing answers (#391); the concept's is instant.
|
||||
await expect.poll(async()=>{const text=await countText(page);return text!==before&&text!=='Loading images…';}).toBe(true);
|
||||
await expect.poll(()=>page.locator(CARDS).count()).toBeLessThan(FIXTURE_IMAGES);
|
||||
await settle(page);
|
||||
const result={count:await countText(page),continuation:await continuationText(page),cards:await page.locator(`${CARDS} .image-open`).evaluateAll(nodes=>nodes.map(node=>node.getAttribute('aria-label')))};
|
||||
|
||||
@@ -228,7 +228,8 @@ async function productionSendOf(page,context) {
|
||||
async function tagOf(page) {
|
||||
const before=await page.locator('#result-count').textContent();
|
||||
await panelOf(page).locator('.detail-inner > .tags').last().getByRole('button',{name:'cinematic',exact:true}).click();
|
||||
await expect.poll(()=>page.locator('#result-count').textContent()).not.toBe(before);
|
||||
// Production's caption reads "Loading images…" until the tag's listing answers (#391); the concept's search is instant.
|
||||
await expect.poll(async()=>{const text=await page.locator('#result-count').textContent();return text!==before&&text!=='Loading images…';}).toBe(true);
|
||||
await settle(page);
|
||||
return {count:await page.locator('#result-count').textContent(),
|
||||
cards:await page.locator(`${CARDS} .image-open`).evaluateAll(nodes=>nodes.map(node=>node.getAttribute('aria-label')))};
|
||||
@@ -328,6 +329,25 @@ test('a recipe-linked image without recipe metadata shows its positive prompt me
|
||||
await expect(panelOf(page).locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Prompt history']);
|
||||
});
|
||||
|
||||
// #391: a tag chip replaces the collection; until its listing answers the caption says it is loading, never that the
|
||||
// filter matched nothing ("0 of 0 images" was painted before the response and read by a fast reader).
|
||||
test('a tag chip collection reads as loading, not empty, until its listing answers (#391)',async({page})=>{
|
||||
await page.setViewportSize({width:1600,height:1100});
|
||||
await openProductionPanel(page,'dark');
|
||||
let release,arrived;
|
||||
const held=new Promise(resolve=>{release=resolve;}),reached=new Promise(resolve=>{arrived=resolve;});
|
||||
await page.route(url=>url.pathname==='/prompt_manager/images/output'&&url.searchParams.get('tags')==='cinematic',
|
||||
async route=>{arrived();await held;await route.continue();},{times:1});
|
||||
const count=page.locator('#result-count');
|
||||
await panelOf(page).locator('.detail-inner > .tags').last().getByRole('button',{name:'cinematic',exact:true}).click();
|
||||
await reached;
|
||||
await expect(count).toHaveText('Loading images…');
|
||||
await expect(page.locator(CARDS)).toHaveCount(0);
|
||||
release();
|
||||
await expect(count).toHaveText(countCopy('8 of 8 images · 2 positive prompts'));
|
||||
await expect(page.locator(CARDS)).toHaveCount(8);
|
||||
});
|
||||
|
||||
// Task C3 (D88, #249 PR C): the concept's chip row is the image's own tags (`i.tags`, app.js a402). With those names
|
||||
// saved as the file's current image tags, production draws the row from them (Tags filter buttons in the panel, plain
|
||||
// in the viewer, the linked positive's equal tags not repeated) and still equals the concept, in both sizes.
|
||||
|
||||
@@ -95,8 +95,14 @@ async function watchProgress(page) {
|
||||
await page.evaluate(() => {
|
||||
window.progressSeen = [];
|
||||
const node = document.getElementById('autoTagProgressPercent');
|
||||
new MutationObserver(() => window.progressSeen.push(node.textContent))
|
||||
.observe(node, { childList: true, characterData: true, subtree: true });
|
||||
// Each change from its own record: frames that land in one task share one
|
||||
// callback, and the node's text by then is only the last of them.
|
||||
new MutationObserver(records => {
|
||||
for (const record of records) {
|
||||
if (record.type === 'characterData') window.progressSeen.push(record.target.data);
|
||||
for (const added of record.addedNodes) window.progressSeen.push(added.textContent);
|
||||
}
|
||||
}).observe(node, { childList: true, characterData: true, subtree: true });
|
||||
});
|
||||
}
|
||||
|
||||
@@ -108,25 +114,58 @@ async function runGalleryBatch(page) {
|
||||
await page.locator('#startAutoTagBtn').click();
|
||||
await done;
|
||||
await expect(page.locator('#autoTagProgressModal')).toBeHidden();
|
||||
await expect(page.getByText(/^Auto tagging complete! Applied tags to \d+ prompts\.$/)).toBeVisible();
|
||||
// #379: the batch counts images, never prompts.
|
||||
await expect(page.getByText(/^Auto tagging complete! Tagged \d+ images?\.$/)).toBeVisible();
|
||||
}
|
||||
|
||||
const CLOSE_NOTE = 'Closing this window stops tagging. Tags already added are kept. Prompt Manager\'s Activity page shows the '
|
||||
+ 'time left.';
|
||||
|
||||
async function serverJobs(request) {
|
||||
const response = await request.get('/prompt_manager/jobs');
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).jobs;
|
||||
}
|
||||
|
||||
async function runNativeBatch(request) {
|
||||
const started = await request.post('/prompt_manager/jobs/autotag', { data: { model_type: 'wd14-swinv2', existing: 'skip',
|
||||
options: { general_threshold: 0.35, character_threshold: 0.85 } } });
|
||||
expect(started.status()).toBe(202);
|
||||
const { job } = await started.json();
|
||||
await expect.poll(async () => (await serverJobs(request)).find(item => item.id === job.id)?.state).toBe('finished');
|
||||
}
|
||||
|
||||
test('gallery batch modal counts progress and closes on complete', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await runGalleryBatch(page);
|
||||
const seen = await page.evaluate(() => window.progressSeen);
|
||||
// The fixture's prompts are already tagged, so the batch skips them; its progress frames still move the bar.
|
||||
expect(seen).toContain('95%');
|
||||
// The phase frames (model 5%, listing 8%) then the per-image frames (from 10%) move the bar.
|
||||
expect(seen).toContain('8%');
|
||||
expect(seen.some(value => parseInt(value, 10) >= 10)).toBe(true);
|
||||
await expect(page.locator('#autoTagCloseNote')).toHaveText(CLOSE_NOTE);
|
||||
// The batch ran as the autotag job of the legacy page.
|
||||
expect((await serverJobs(request)).filter(job => job.kind === 'autotag').map(job => [job.title, job.state]))
|
||||
.toEqual([['Tag output images (legacy page)', 'finished']]);
|
||||
expect((await downloads(request)).length).toBe(0);
|
||||
});
|
||||
|
||||
test('admin batch copy names images, not prompts (#379)', async ({ page }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await expect(page.locator('input[name="autoTagMode"][value="skip"] + span')).toHaveText('Skip images already auto-tagged');
|
||||
await expect(page.getByText('"auto-scanned" is not counted as a real tag')).toHaveCount(0);
|
||||
await expect(page.locator('#autoTagCloseNote')).toHaveText(CLOSE_NOTE);
|
||||
await expect(page.locator('#retagConfirmModal h3')).toHaveText('Image Already Auto-tagged');
|
||||
await expect(page.locator('#retagConfirmModal')).toContainText('A tagging model already tagged this image. Generate new tags?');
|
||||
});
|
||||
|
||||
test('admin review opens the first image', async ({ page }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check();
|
||||
await page.locator('#startReviewBtn').click();
|
||||
// The first image's prompt is already tagged: confirm a new pass, then the review opens on that image.
|
||||
await page.locator('#retagConfirmBtn').click();
|
||||
// #379: no image is auto-tagged yet (prompt tags no longer count), so the review opens on the first image.
|
||||
await expect(page.locator('#retagConfirmModal')).toBeHidden();
|
||||
await expect(page.locator('#autoTagReviewModal')).toBeVisible();
|
||||
await expect(page.locator('#reviewImageName')).not.toHaveText(/^(image_name\.png|Unknown|)$/);
|
||||
await expect(page.getByText('Failed to start review mode')).toHaveCount(0);
|
||||
@@ -150,3 +189,70 @@ test('gallery unload works and is refused while the gate is held', async ({ page
|
||||
await unload.click();
|
||||
await expect(page.getByText('WD14-SWINV2 model unloaded successfully')).toBeVisible();
|
||||
});
|
||||
|
||||
/** The admin review lists every row, videos too: Skip any image that is not one until the confirmation shows. */
|
||||
async function untilAutoTagged(page, confirm) {
|
||||
// Each image's skip check answers before its confirmation or its error shows.
|
||||
const failed = page.locator('#reviewTagsVisible .text-pm-error');
|
||||
for (let tries = 0; tries < 5; tries++) {
|
||||
await expect.poll(async () => (await confirm.isVisible()) || (await failed.count()) > 0).toBe(true);
|
||||
if (await confirm.isVisible()) return;
|
||||
const answered = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/single'));
|
||||
await page.locator('#skipReviewBtn').click();
|
||||
await answered;
|
||||
}
|
||||
await expect(confirm).toBeVisible();
|
||||
}
|
||||
|
||||
// #379: the admin review asks before re-tagging an image a model already tagged, and Apply & Next writes that image's
|
||||
// tags (its bytes, never its prompt).
|
||||
test('admin review: an auto-tagged image asks first; Re-tag then Apply & Next tags the image', async ({ page, request }) => {
|
||||
await runNativeBatch(request);
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check();
|
||||
await page.locator('#startReviewBtn').click();
|
||||
const confirm = page.locator('#retagConfirmModal');
|
||||
await untilAutoTagged(page, confirm);
|
||||
await expect(confirm.locator('#retagExistingTags span').first()).toHaveText('sandstone');
|
||||
await page.locator('#retagConfirmBtn').click();
|
||||
await expect(page.locator('#autoTagReviewModal')).toBeVisible();
|
||||
await expect(page.locator('#reviewTagsVisible .tag-chip').first()).toContainText('sandstone');
|
||||
const applied = page.waitForRequest(item => item.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
const answer = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
await page.locator('#applyReviewBtn').click();
|
||||
const body = (await applied).postDataJSON();
|
||||
expect(body.prompt_id).toBeUndefined();
|
||||
expect(body.sha256).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect((await answer).status()).toBe(200);
|
||||
});
|
||||
|
||||
test('admin review: Skip All Tagged passes every auto-tagged image', async ({ page, request }) => {
|
||||
await runNativeBatch(request);
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check();
|
||||
await page.locator('#startReviewBtn').click();
|
||||
const confirm = page.locator('#retagConfirmModal');
|
||||
await untilAutoTagged(page, confirm);
|
||||
await page.locator('#retagSkipAllBtn').click();
|
||||
await expect(page.locator('#retagConfirmModal')).toBeHidden();
|
||||
await expect(page.getByText('Review complete!')).toBeVisible();
|
||||
});
|
||||
|
||||
test('gallery review: Apply & Next tags the image by its path and bytes', async ({ page }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check();
|
||||
const suggested = page.waitForRequest(item => item.url().endsWith('/prompt_manager/autotag/single'));
|
||||
await page.locator('#startReviewBtn').click();
|
||||
expect((await suggested).postDataJSON().existing).toBe('merge');
|
||||
await expect(page.locator('#reviewTagsContainer .tag-chip').first()).toContainText('sandstone');
|
||||
const applied = page.waitForRequest(item => item.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
const answer = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
await page.locator('#applyReviewBtn').click();
|
||||
const body = (await applied).postDataJSON();
|
||||
expect(Object.keys(body).sort()).toEqual(['path', 'root_index', 'sha256', 'tags'].filter(key => key in body).sort());
|
||||
expect(body.prompt_id).toBeUndefined();
|
||||
expect((await answer).status()).toBe(200);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
// System S13 (#330): the legacy admin and gallery pages open native System for their System tools. Each replaced
|
||||
// control is a plain link (`workspace#/system/<id>`, a new tab), and the legacy flows S13 keeps still work.
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
const PAGES = Object.freeze({ admin: '/prompt_manager/admin', gallery: '/prompt_manager/gallery' });
|
||||
const LABELS = Object.freeze({
|
||||
overview: 'Overview',
|
||||
appearance: 'Appearance',
|
||||
sources: 'Sources & scans',
|
||||
integrations: 'Integrations',
|
||||
tagging: 'Auto-tagging',
|
||||
maintenance: 'Maintenance',
|
||||
backups: 'Backup & export',
|
||||
diagnostics: 'Diagnostics & logs',
|
||||
});
|
||||
|
||||
// One row per System link of the design's §2 map: the page, the control's label, how to find it, the section it opens
|
||||
// and what a test must arrange before the control shows.
|
||||
const LINKS = Object.freeze([
|
||||
{ page: 'admin', label: 'Diagnostics', locate: page => page.locator('#diagnosticsBtn'), section: 'diagnostics' },
|
||||
{ page: 'admin', label: 'View Logs', locate: page => page.locator('#logsBtn'), section: 'diagnostics' },
|
||||
]);
|
||||
|
||||
async function post(request, url, data) {
|
||||
await expect(await request.post(url, data === undefined ? {} : { data })).toBeOK();
|
||||
}
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await post(request, '/__test__/reset');
|
||||
await post(request, '/__test__/system/reset');
|
||||
await post(request, '/__test__/tagging/reset');
|
||||
// The 3.2.0 update notice would cover the admin page's toolbar.
|
||||
await page.addInitScript(() => { localStorage.setItem('pm_update_notice_dismissed', '3.2.0'); });
|
||||
page.legacyErrors = [];
|
||||
page.on('pageerror', error => page.legacyErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await post(request, '/__test__/tagging/reset');
|
||||
await post(request, '/__test__/system/reset');
|
||||
expect(page.legacyErrors).toEqual([]);
|
||||
});
|
||||
|
||||
/** Opens the legacy page with the link showing; returns the link. */
|
||||
async function openLink(page, request, link) {
|
||||
if (link.arrange) await link.arrange(page, request);
|
||||
await page.goto(PAGES[link.page]);
|
||||
if (link.reveal) await link.reveal(page);
|
||||
const control = link.locate(page);
|
||||
await expect(control).toBeVisible();
|
||||
return control;
|
||||
}
|
||||
|
||||
for (const link of LINKS) {
|
||||
test(`legacy ${link.page} ${link.label} opens System ${link.section}`, async ({ page, context, request }) => {
|
||||
const control = await openLink(page, request, link);
|
||||
const legacyUrl = page.url();
|
||||
await expect(control).toHaveAttribute('href', `workspace#/system/${link.section}`);
|
||||
await expect(control).toHaveAttribute('target', '_blank');
|
||||
await expect(control).toHaveAttribute('rel', 'noopener');
|
||||
const opened = context.waitForEvent('page');
|
||||
await control.click();
|
||||
const system = await opened;
|
||||
await expect(system).toHaveURL(new RegExp(`/prompt_manager/workspace#/system/${link.section}$`));
|
||||
// System's page heading is "System"; the open section is the pressed button of its section switcher.
|
||||
const main = system.locator('#workspace-main');
|
||||
await expect(main.getByRole('heading', { level: 1, name: 'System', exact: true })).toBeVisible();
|
||||
await expect(main.getByRole('group', { name: 'System sections', exact: true })
|
||||
.getByRole('button', { name: LABELS[link.section], exact: true })).toHaveAttribute('aria-pressed', 'true');
|
||||
expect(page.url()).toBe(legacyUrl);
|
||||
await system.close();
|
||||
});
|
||||
}
|
||||
|
||||
test('a System link sends no request from the legacy page', async ({ page, context, request }) => {
|
||||
for (const link of LINKS) {
|
||||
const control = await openLink(page, request, link);
|
||||
// The page's own start-up reads (list, film strips, settings) finish first; then the click must add none.
|
||||
await page.waitForLoadState('networkidle');
|
||||
const sent = [];
|
||||
const record = item => sent.push(`${item.method()} ${item.url()}`);
|
||||
page.on('request', record);
|
||||
const opened = context.waitForEvent('page');
|
||||
await control.click();
|
||||
const system = await opened;
|
||||
await system.waitForLoadState('domcontentloaded');
|
||||
page.off('request', record);
|
||||
expect(sent, `${link.page} ${link.label}`).toEqual([]);
|
||||
await system.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('kept admin flows still work', async ({ page, context }) => {
|
||||
await page.goto(PAGES.admin);
|
||||
const texts = page.locator('#resultsList .prompt-text');
|
||||
await expect(texts.first()).toBeVisible();
|
||||
|
||||
// K1: sorting reorders the list.
|
||||
for (const [value, order] of [['text_asc', 1], ['text_desc', -1]]) {
|
||||
const listed = page.waitForResponse(item => /\/prompt_manager\/(recent|search)\?/.test(item.url()) && item.url().includes(`sort=${value}`));
|
||||
await page.locator('#sortBy').selectOption(value);
|
||||
await listed;
|
||||
await expect.poll(async () => {
|
||||
const shown = await texts.allTextContents();
|
||||
// The server sorts case-insensitively (SQLite NOCASE), not by locale.
|
||||
const key = text => text.toLowerCase();
|
||||
const sorted = [...shown].sort((a, b) => order * (key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0));
|
||||
return shown.length > 1 && shown.join('\n') === sorted.join('\n');
|
||||
}).toBe(true);
|
||||
}
|
||||
// K3: Add Prompt saves a prompt the list then shows.
|
||||
await page.locator('#addPromptBtn').click();
|
||||
await page.locator('#addPromptText').fill('Kept flow lighthouse');
|
||||
const saved = page.waitForResponse(item => item.request().method() === 'POST' && item.url().endsWith('/prompt_manager/save'));
|
||||
await page.locator('#saveNewPromptBtn').click();
|
||||
expect((await saved).ok()).toBe(true);
|
||||
await expect(page.locator('#addPromptModal')).toBeHidden();
|
||||
await expect(texts.filter({ hasText: 'Kept flow lighthouse' })).toBeVisible();
|
||||
|
||||
// K1: a search narrows the list, and a selection enables the bulk buttons.
|
||||
const all = await texts.allTextContents();
|
||||
const word = all[0].trim().split(/\s+/)[0];
|
||||
const searched = page.waitForResponse(item => item.url().includes('/prompt_manager/search?') && item.url().includes('text='));
|
||||
await page.locator('#searchText').fill(word);
|
||||
await page.locator('#searchBtn').click();
|
||||
await searched;
|
||||
await expect.poll(async () => {
|
||||
const shown = await texts.allTextContents();
|
||||
return shown.length > 0 && shown.length < all.length && shown.every(text => text.toLowerCase().includes(word.toLowerCase()));
|
||||
}).toBe(true);
|
||||
await expect(page.locator('#bulkDeleteBtn')).toBeDisabled();
|
||||
await page.locator('#resultsList .prompt-checkbox').first().check();
|
||||
for (const id of ['#bulkDeleteBtn', '#bulkTagBtn', '#bulkCategoryBtn']) await expect(page.locator(id)).toBeEnabled();
|
||||
|
||||
// K4: Stats reads the statistics again; Export downloads the prompts.
|
||||
const stats = page.waitForResponse(item => item.url().endsWith('/prompt_manager/stats') && item.ok());
|
||||
await page.locator('#statsBtn').click();
|
||||
await stats;
|
||||
await expect(page.getByText('Stats refreshed successfully!')).toBeVisible();
|
||||
const download = page.waitForEvent('download');
|
||||
await page.locator('#exportBtn').click();
|
||||
expect((await download).suggestedFilename()).toMatch(/^prompts-\d{4}-\d{2}-\d{2}\.json$/);
|
||||
|
||||
// K5: Scan Images opens its dialog (with the quick backup) without starting a scan.
|
||||
await page.locator('#scanBtn').click();
|
||||
await expect(page.locator('#scanModal')).toBeVisible();
|
||||
await expect(page.locator('#quickBackupBtn')).toBeVisible();
|
||||
await page.locator('#cancelScan').click();
|
||||
await expect(page.locator('#scanModal')).toBeHidden();
|
||||
|
||||
// K7: Metadata and Gallery open their pages in a new tab.
|
||||
for (const [id, path] of [['#metadataBtn', '/prompt_manager/gallery.html'], ['#galleryBtn', '/prompt_manager/gallery']]) {
|
||||
const opened = context.waitForEvent('page');
|
||||
await page.locator(id).click();
|
||||
const tab = await opened;
|
||||
await tab.waitForLoadState('domcontentloaded');
|
||||
expect(new URL(tab.url()).pathname).toBe(path);
|
||||
await tab.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Prove the scratch-seeded resets leave the state the old resets left (#384).
|
||||
|
||||
Run by hand on Linux (the browser server's platform):
|
||||
|
||||
python -m tests.browser.reset_equivalence
|
||||
|
||||
It starts the browser server three times: twice seeding on the live database
|
||||
as before #384 (``PM_TEST_SEED_ON_LIVE=1``) and once with the scratch seeder.
|
||||
Each server gets the same sequence: every reset kind of ``/__test__/reset``
|
||||
and ``/__test__/concept/reset``, with other fixtures' resets and API writes
|
||||
between them. After every reset it reads every table's rows (by rowid), the
|
||||
output and input file trees (size, SHA-256, mtime) and the response body.
|
||||
|
||||
Values that differ between the two old runs are wall-clock values (created_at,
|
||||
an unpinned concept "now", file mtimes); they are masked. Everything else must
|
||||
be equal between the old runs and the new one. Exit status 0 means equal.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
READY_TIMEOUT_S = 60
|
||||
CLOCK = "<clock>"
|
||||
|
||||
RESETS = [
|
||||
("/__test__/reset", {}),
|
||||
("/__test__/reset", {"extra_images": 2}),
|
||||
("/__test__/reset", {"extra_images": 108}),
|
||||
("/__test__/reset", {"include_video": True}),
|
||||
("/__test__/reset", {"prompt_inspector": True}),
|
||||
("/__test__/reset", {"native_library": True}),
|
||||
("/__test__/reset", {"native_library": True, "extra_prompts": 125}),
|
||||
("/__test__/reset", {"native_library": True, "extra_variants": 41}),
|
||||
("/__test__/reset", {"workspace_visual": True}),
|
||||
("/__test__/reset", {"full_inspector": True}),
|
||||
(
|
||||
"/__test__/reset",
|
||||
{"full_inspector": True, "include_video": True, "extra_images": 2},
|
||||
),
|
||||
("/__test__/reset", {"chronological_history": True}),
|
||||
("/__test__/reset", {"chronological_history": True, "history_runs": 85}),
|
||||
("/__test__/concept/reset", {}),
|
||||
("/__test__/concept/reset", {"extra_images": 100}),
|
||||
("/__test__/concept/reset", {"seedless_images": 2, "short_seedless": True}),
|
||||
("/__test__/concept/reset", {"now": 1_780_000_000}),
|
||||
]
|
||||
# Writes by other fixtures and the API between resets; the resets must keep or
|
||||
# replace them exactly as before.
|
||||
DIRTY = [
|
||||
("/__test__/tagging/reset", {}),
|
||||
("/__test__/system/reset", None),
|
||||
(
|
||||
"/__test__/saved-views/raw",
|
||||
{"page": "images", "name": "dirty", "state": "{}"},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class Server:
|
||||
def __init__(self, on_live: bool) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory(prefix="pm-equivalence-")
|
||||
env = {**os.environ, "TMPDIR": self.tmp.name}
|
||||
env.update(TEMP=self.tmp.name, TMP=self.tmp.name)
|
||||
env.pop("PM_TEST_SEED_ON_LIVE", None)
|
||||
if on_live:
|
||||
env["PM_TEST_SEED_ON_LIVE"] = "1"
|
||||
self.process = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"tests.browser.server",
|
||||
"--port",
|
||||
"0",
|
||||
"--exit-on-stdin-eof",
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
line = self.process.stdout.readline().decode()
|
||||
if not line.startswith("PM_TEST_SERVER_PORT="):
|
||||
raise RuntimeError(f"no port line: {line!r}")
|
||||
self.base = f"http://127.0.0.1:{int(line.split('=', 1)[1])}"
|
||||
deadline = time.monotonic() + READY_TIMEOUT_S
|
||||
while True:
|
||||
try:
|
||||
self.post_json("GET", "/prompt_manager/test", None)
|
||||
break
|
||||
except OSError:
|
||||
if time.monotonic() > deadline or self.process.poll() is not None:
|
||||
raise
|
||||
time.sleep(0.1)
|
||||
(self.root,) = Path(self.tmp.name).glob("pm-browser-*")
|
||||
|
||||
def post_json(self, method: str, path: str, data) -> dict:
|
||||
body = None if data is None else json.dumps(data).encode()
|
||||
request = urllib.request.Request(
|
||||
self.base + path,
|
||||
data=body if method == "POST" else None,
|
||||
method=method,
|
||||
headers={"Content-Type": "application/json"} if body else {},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=120) as response: # nosec B310
|
||||
text = response.read().decode()
|
||||
try:
|
||||
return json.loads(text)
|
||||
except ValueError:
|
||||
return {"text": text[:200]}
|
||||
|
||||
def stop(self) -> None:
|
||||
self.process.stdin.close()
|
||||
self.process.wait(timeout=READY_TIMEOUT_S)
|
||||
self.process.stdout.close()
|
||||
self.tmp.cleanup()
|
||||
|
||||
|
||||
def read_tables(path: Path) -> dict:
|
||||
conn = sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)
|
||||
try:
|
||||
tables = {}
|
||||
names = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
|
||||
)
|
||||
]
|
||||
for name in names:
|
||||
quoted = '"' + name.replace('"', '""') + '"'
|
||||
columns = [row[1] for row in conn.execute(f"PRAGMA table_info({quoted})")]
|
||||
try:
|
||||
cursor = conn.execute(f"SELECT rowid, * FROM {quoted} ORDER BY rowid")
|
||||
columns = ["rowid", *columns]
|
||||
except sqlite3.OperationalError:
|
||||
cursor = conn.execute(f"SELECT * FROM {quoted} ORDER BY 1")
|
||||
rows = [
|
||||
[value.hex() if isinstance(value, bytes) else value for value in row]
|
||||
for row in cursor
|
||||
]
|
||||
tables[name] = {"columns": columns, "rows": rows}
|
||||
return tables
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def read_files(root: Path) -> dict:
|
||||
files = {}
|
||||
for top in ("output", "input"):
|
||||
for path in sorted((root / top).rglob("*")):
|
||||
if path.is_file():
|
||||
stat = path.stat()
|
||||
files[path.relative_to(root).as_posix()] = [
|
||||
stat.st_size,
|
||||
hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
stat.st_mtime_ns,
|
||||
]
|
||||
return files
|
||||
|
||||
|
||||
def flatten(value, prefix="") -> dict:
|
||||
if isinstance(value, dict):
|
||||
out = {}
|
||||
for key, item in value.items():
|
||||
out.update(flatten(item, f"{prefix}/{key}"))
|
||||
return out
|
||||
if isinstance(value, list):
|
||||
out = {f"{prefix}#len": len(value)}
|
||||
for index, item in enumerate(value):
|
||||
out.update(flatten(item, f"{prefix}[{index}]"))
|
||||
return out
|
||||
return {prefix: value}
|
||||
|
||||
|
||||
def snapshot(server: Server, response: dict) -> dict:
|
||||
flat = {}
|
||||
for table, data in read_tables(server.root / "browser.db").items():
|
||||
flat[f"table {table}#rows"] = len(data["rows"])
|
||||
for index, row in enumerate(data["rows"]):
|
||||
for column, value in zip(data["columns"], row):
|
||||
flat[f"table {table}[{index}].{column}"] = value
|
||||
for name, (size, digest, mtime) in read_files(server.root).items():
|
||||
flat[f"file {name}"] = [size, digest]
|
||||
flat[f"file {name}#mtime"] = mtime
|
||||
flat["files#count"] = sum(
|
||||
1 for key in flat if key.startswith("file ") and "#" not in key
|
||||
)
|
||||
for key, value in flatten(response).items():
|
||||
flat[f"response{key}"] = value
|
||||
# Each server has its own temp root and a random storage epoch; name them
|
||||
# instead of masking every column that carries them.
|
||||
epoch = flat.get("table execution_history_metadata[0].storage_epoch")
|
||||
names = [(str(server.root), "<root>")]
|
||||
if isinstance(epoch, str) and epoch:
|
||||
names.append((epoch, "<epoch>"))
|
||||
for key, value in flat.items():
|
||||
if isinstance(value, str):
|
||||
for text, name in names:
|
||||
value = value.replace(text, name)
|
||||
flat[key] = value
|
||||
return flat
|
||||
|
||||
|
||||
def run(on_live: bool) -> list[dict]:
|
||||
server = Server(on_live)
|
||||
try:
|
||||
states = []
|
||||
for index, (path, data) in enumerate(RESETS * 2):
|
||||
dirty_path, dirty_data = DIRTY[index % len(DIRTY)]
|
||||
server.post_json("POST", dirty_path, dirty_data)
|
||||
started = time.monotonic()
|
||||
response = server.post_json("POST", path, data)
|
||||
elapsed = time.monotonic() - started
|
||||
print(f" {'old' if on_live else 'new'} {path} {data}: {elapsed:.2f}s")
|
||||
states.append(snapshot(server, response))
|
||||
return states
|
||||
finally:
|
||||
server.stop()
|
||||
|
||||
|
||||
def column_key(key: str) -> str:
|
||||
"""``table t[3].created_at`` -> ``table t.created_at`` (mask whole columns)."""
|
||||
if key.startswith("table ") and "[" in key:
|
||||
head, _, tail = key.partition("[")
|
||||
return head + tail.partition("]")[2]
|
||||
return key
|
||||
|
||||
|
||||
def main() -> int:
|
||||
old_a, old_b, new = run(True), run(True), run(False)
|
||||
failures = 0
|
||||
masked_all = set()
|
||||
for step, (path, data) in enumerate(RESETS * 2):
|
||||
a, b, n = old_a[step], old_b[step], new[step]
|
||||
masked = {
|
||||
column_key(key)
|
||||
for key in set(a) | set(b)
|
||||
if a.get(key, CLOCK) != b.get(key, CLOCK)
|
||||
}
|
||||
masked_all |= masked
|
||||
keys = set(a) | set(n)
|
||||
diffs = [
|
||||
key
|
||||
for key in sorted(keys)
|
||||
if column_key(key) not in masked
|
||||
and a.get(key, "<missing>") != n.get(key, "<missing>")
|
||||
]
|
||||
tables = len(
|
||||
{k.split("[")[0].split("#")[0] for k in a if k.startswith("table ")}
|
||||
)
|
||||
status = "EQUAL" if not diffs else f"DIFFERENT ({len(diffs)} values)"
|
||||
print(
|
||||
f"{status}: {path} {json.dumps(data)}: {tables} tables, "
|
||||
f"{a['files#count']} files, {len(keys)} values, "
|
||||
f"{len(masked)} masked columns"
|
||||
)
|
||||
for key in diffs[:20]:
|
||||
old, now = a.get(key, "<missing>"), n.get(key, "<missing>")
|
||||
print(f" {key}: old {old!r} new {now!r}")
|
||||
failures += bool(diffs)
|
||||
print("masked (differ between the two old runs):")
|
||||
for key in sorted(masked_all):
|
||||
print(f" {key}")
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+69
-38
@@ -19,8 +19,10 @@ from aiohttp.web_runner import GracefulExit
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
from tests.browser.folder_trash import close_trash, replace_folder
|
||||
from tests.browser.request_hold import register_request_holds
|
||||
from tests.browser import slow_reset_watch
|
||||
from tests.browser.fast_reset import SEED_ON_LIVE_ENV, ScratchSeeder
|
||||
from tests.browser.workspace_activity_fixture import register_activity_commands
|
||||
from tests.browser.workspace_system_fixture import register_system_commands
|
||||
from tests.browser.workspace_integrations_fixture import (
|
||||
@@ -45,6 +47,10 @@ from tests.browser.workspace_library_editor_fixture import (
|
||||
from tests.browser.workspace_vocabulary_fixture import register_vocabulary_commands
|
||||
from tests.browser.workspace_tagging_fixture import register_tagging_commands
|
||||
from tests.browser.workspace_review_fixture import register_review_commands
|
||||
from tests.browser.workspace_diagnostics_fixture import (
|
||||
register_diagnostics_commands,
|
||||
repoint_logs,
|
||||
)
|
||||
|
||||
CANONICAL_ROOT = Path(__file__).resolve().parents[2]
|
||||
CANONICAL_ASSETS = {
|
||||
@@ -199,6 +205,8 @@ def create_app(root: Path) -> web.Application:
|
||||
directory.mkdir()
|
||||
setattr(folder_paths, f"get_{name}_directory", lambda path=directory: str(path))
|
||||
sys.modules["folder_paths"] = folder_paths
|
||||
# S12: the logger writes into this server's root before any record is written.
|
||||
repoint_logs(root)
|
||||
|
||||
from py.config import GalleryConfig, PromptManagerConfig
|
||||
|
||||
@@ -223,8 +231,13 @@ def create_app(root: Path) -> web.Application:
|
||||
register_activity_commands(app, routes, api)
|
||||
register_system_commands(app, routes, api, root)
|
||||
register_integrations_commands(app, routes, api, root)
|
||||
reset_logs = register_diagnostics_commands(app, routes, api, root)
|
||||
request_holds = register_request_holds(app, routes)
|
||||
slow_reset_watch.install(app, api)
|
||||
# Resets seed a scratch copy and publish it in one commit (#384).
|
||||
seeder = ScratchSeeder(
|
||||
api, root / "reset-scratch", on_live=os.environ.get(SEED_ON_LIVE_ENV) == "1"
|
||||
)
|
||||
|
||||
async def canonical_asset(request: web.Request) -> web.Response:
|
||||
filename, content_type = CANONICAL_ASSETS[request.path]
|
||||
@@ -238,6 +251,7 @@ def create_app(root: Path) -> web.Application:
|
||||
routes.get(path)(canonical_asset)
|
||||
|
||||
def seed(
|
||||
target,
|
||||
extra_images: int = 0,
|
||||
include_video: bool = False,
|
||||
prompt_inspector: bool = False,
|
||||
@@ -245,7 +259,7 @@ def create_app(root: Path) -> web.Application:
|
||||
extra_prompts: int = 0,
|
||||
extra_variants: int = 0,
|
||||
) -> None:
|
||||
with api.db.model.get_connection() as conn:
|
||||
with target.db.model.get_connection() as conn:
|
||||
for table in (
|
||||
"saved_views",
|
||||
"image_favorites",
|
||||
@@ -264,8 +278,7 @@ def create_app(root: Path) -> web.Application:
|
||||
):
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for name in ("output", "input"):
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
replace_folder(root / name)
|
||||
for index, (directory, text, tag) in enumerate(
|
||||
[
|
||||
("output", "Cat portrait", "cats"),
|
||||
@@ -308,28 +321,30 @@ def create_app(root: Path) -> web.Application:
|
||||
image.save(image_path, pnginfo=info)
|
||||
stamp = time.time() - index * 10
|
||||
os.utime(image_path, (stamp, stamp))
|
||||
pid = api.db.save_prompt(
|
||||
pid = target.db.save_prompt(
|
||||
text, tags=[tag], prompt_hash=generate_prompt_hash(text)
|
||||
)
|
||||
api.db.link_image_to_prompt(pid, str(image_path))
|
||||
target.db.link_image_to_prompt(pid, str(image_path))
|
||||
if index == 0:
|
||||
video_prompt_id = pid
|
||||
for number in range(extra_images):
|
||||
extra = root / directory / "bulk" / f"{number:04}.png"
|
||||
extra.parent.mkdir(exist_ok=True)
|
||||
shutil.copyfile(image_path, extra)
|
||||
api.db.link_image_to_prompt(pid, str(extra))
|
||||
target.db.link_image_to_prompt(pid, str(extra))
|
||||
if native_library:
|
||||
positive = api.db.get_prompt_by_hash(generate_prompt_hash("Cat portrait"))
|
||||
positive = target.db.get_prompt_by_hash(
|
||||
generate_prompt_hash("Cat portrait")
|
||||
)
|
||||
positive_id = positive["id"]
|
||||
api.db.update_prompt_metadata(
|
||||
target.db.update_prompt_metadata(
|
||||
positive_id, category="Animals", tags=["cats", "studio"]
|
||||
)
|
||||
variants = []
|
||||
for index, negative in enumerate(
|
||||
["blurry, watermark", "low quality", "low contrast", ""]
|
||||
):
|
||||
pair_id = api.db.save_prompt_pair(
|
||||
pair_id = target.db.save_prompt_pair(
|
||||
"Cat portrait",
|
||||
negative,
|
||||
title=f"Portrait variant {index + 1}",
|
||||
@@ -337,28 +352,30 @@ def create_app(root: Path) -> web.Application:
|
||||
tags=["studio"] if index == 0 else ["outdoors"],
|
||||
)
|
||||
variants.append(pair_id)
|
||||
with api.db.model.get_connection() as conn:
|
||||
with target.db.model.get_connection() as conn:
|
||||
images = conn.execute(
|
||||
"SELECT id FROM generated_images WHERE prompt_id=? ORDER BY id",
|
||||
(positive_id,),
|
||||
).fetchall()
|
||||
for index, image in enumerate(images[:2]):
|
||||
api.db.link_image_to_pair(
|
||||
target.db.link_image_to_pair(
|
||||
variants[index], image["id"], {"source": "fixture", "anchor": index}
|
||||
)
|
||||
api.db.link_image_to_pair(
|
||||
target.db.link_image_to_pair(
|
||||
variants[0], images[0]["id"], {"source": "fixture second evidence"}
|
||||
)
|
||||
api.db.record_prompt_pair_use(variants[0], times=3)
|
||||
target.db.record_prompt_pair_use(variants[0], times=3)
|
||||
for number in range(extra_variants):
|
||||
api.db.save_prompt_pair("Cat portrait", f"Negative extra {number:04}")
|
||||
target.db.save_prompt_pair(
|
||||
"Cat portrait", f"Negative extra {number:04}"
|
||||
)
|
||||
legacy = root / "output" / "legacy-unassigned.png"
|
||||
shutil.copyfile(root / "output" / "shared" / "same.png", legacy)
|
||||
api.db.link_image_to_prompt(positive_id, str(legacy))
|
||||
api.db.save_prompt("Dog landscape", category="Animals", tags=["dogs"])
|
||||
api.db.save_prompt_pair("Dual role portrait", "Cat portrait")
|
||||
target.db.link_image_to_prompt(positive_id, str(legacy))
|
||||
target.db.save_prompt("Dog landscape", category="Animals", tags=["dogs"])
|
||||
target.db.save_prompt_pair("Dual role portrait", "Cat portrait")
|
||||
for number in range(extra_prompts):
|
||||
api.db.save_prompt(
|
||||
target.db.save_prompt(
|
||||
f"Library extra {number:04}", category="Bulk", tags=["bulk"]
|
||||
)
|
||||
if prompt_inspector:
|
||||
@@ -379,8 +396,8 @@ def create_app(root: Path) -> web.Application:
|
||||
from py.api.thumbnail_settings import ThumbnailSettings
|
||||
|
||||
# 150 px previews in the default format (#317 thumbnail settings).
|
||||
api._generate_gallery_thumbnails_sync(
|
||||
api._get_all_output_dirs(), ThumbnailSettings("webp", 80, 150)
|
||||
target._generate_gallery_thumbnails_sync(
|
||||
target._get_all_output_dirs(), ThumbnailSettings("webp", 80, 150)
|
||||
)
|
||||
if include_video:
|
||||
# Seed after thumbnail generation to exercise a real unthumbnailed video.
|
||||
@@ -388,8 +405,8 @@ def create_app(root: Path) -> web.Application:
|
||||
shutil.copyfile(
|
||||
Path(__file__).resolve().parents[1] / "fixtures" / "tiny.webm", video
|
||||
)
|
||||
api.db.link_image_to_prompt(video_prompt_id, str(video))
|
||||
api.invalidate_gallery_cache()
|
||||
target.db.link_image_to_prompt(video_prompt_id, str(video))
|
||||
target.invalidate_gallery_cache()
|
||||
|
||||
history_fixture = None
|
||||
|
||||
@@ -415,6 +432,7 @@ def create_app(root: Path) -> web.Application:
|
||||
# No preview queued by the last test may land in the new fixture.
|
||||
await api._run_in_executor(api.close_auto_thumbnails)
|
||||
library_editor_commands.reset()
|
||||
await api._run_in_executor(reset_logs)
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
if not isinstance(data, dict):
|
||||
raise web.HTTPBadRequest()
|
||||
@@ -434,7 +452,10 @@ def create_app(root: Path) -> web.Application:
|
||||
raise web.HTTPBadRequest()
|
||||
if chronological_history:
|
||||
manifest = await api._run_in_executor(
|
||||
lambda: seed_workspace_history(api, root, history_runs=history_runs)
|
||||
seeder.run,
|
||||
lambda target: seed_workspace_history(
|
||||
target, root, history_runs=history_runs
|
||||
),
|
||||
)
|
||||
history_fixture = manifest
|
||||
return web.json_response({"success": True, **manifest})
|
||||
@@ -492,25 +513,31 @@ def create_app(root: Path) -> web.Application:
|
||||
raise web.HTTPBadRequest()
|
||||
if workspace_visual:
|
||||
history_fixture = None
|
||||
await api._run_in_executor(seed_workspace_visual, api, root)
|
||||
await api._run_in_executor(
|
||||
seeder.run, lambda target: seed_workspace_visual(target, root)
|
||||
)
|
||||
return web.json_response({"success": True})
|
||||
if full_inspector:
|
||||
history_fixture = None
|
||||
await api._run_in_executor(
|
||||
lambda: seed_workspace_inspector(
|
||||
api, root, extra_images=extra, include_video=include_video
|
||||
)
|
||||
seeder.run,
|
||||
lambda target: seed_workspace_inspector(
|
||||
target, root, extra_images=extra, include_video=include_video
|
||||
),
|
||||
)
|
||||
return web.json_response({"success": True})
|
||||
history_fixture = None
|
||||
await api._run_in_executor(
|
||||
seed,
|
||||
extra,
|
||||
include_video,
|
||||
prompt_inspector,
|
||||
native_library,
|
||||
extra_prompts,
|
||||
extra_variants,
|
||||
seeder.run,
|
||||
lambda target: seed(
|
||||
target,
|
||||
extra,
|
||||
include_video,
|
||||
prompt_inspector,
|
||||
native_library,
|
||||
extra_prompts,
|
||||
extra_variants,
|
||||
),
|
||||
)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@@ -550,14 +577,15 @@ def create_app(root: Path) -> web.Application:
|
||||
reveal_stub.reset()
|
||||
history_fixture = None
|
||||
manifest = await api._run_in_executor(
|
||||
lambda: seed_workspace_concept(
|
||||
api,
|
||||
seeder.run,
|
||||
lambda target: seed_workspace_concept(
|
||||
target,
|
||||
root,
|
||||
extra_images=data.get("extra_images", 0),
|
||||
seedless_images=data.get("seedless_images", 0),
|
||||
short_seedless=data.get("short_seedless", False),
|
||||
now=data.get("now"),
|
||||
)
|
||||
),
|
||||
)
|
||||
return web.json_response({"success": True, **manifest})
|
||||
|
||||
@@ -625,9 +653,12 @@ def create_app(root: Path) -> web.Application:
|
||||
async def cleanup(application: web.Application) -> None:
|
||||
# A scan of an ambiguous fixture file captures it for Needs review.
|
||||
api.close_annotation_captures()
|
||||
seeder.close()
|
||||
api.db.close_all()
|
||||
# Old media folders still waiting for deletion go before the temp root.
|
||||
close_trash(root)
|
||||
|
||||
seed()
|
||||
seed(api)
|
||||
# Open Needs review change streams end first, as on the real host.
|
||||
app.on_shutdown.append(api.close_review_events)
|
||||
app.on_cleanup.append(cleanup)
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate, measureReview, mapPlanConcept }
|
||||
from './concept-inventory.mjs';
|
||||
|
||||
// #249 System S12 Task 7 (#329): System › Diagnostics & logs (app.js renderSystem 694–695) and its log cleanup review
|
||||
// (the concept's `maintenancePlan("rotated logs")`, 953–958) against the immutable concept, on the concept-shaped
|
||||
// fixture with the logger writing into this server's own root and seeded with the concept's sample (four lines, a
|
||||
// 1.4 MB active log, three rotated copies, console logging off). Each concept measurement is mapped only by the recorded
|
||||
// deviations (D109, D110, and D100 for the review), each commented where it applies; after that the difference must be
|
||||
// empty, and a removed term, a renamed button, a reordered option, a primary Refresh, a squeezed select or a swapped
|
||||
// token must be noticed.
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(await request.post('/__test__/logs/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/logs/seed', { data: { mode: 'concept' } })).toBeOK();
|
||||
page.conceptErrors = [];
|
||||
page.on('pageerror', error => page.conceptErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/logs/reset');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.conceptErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const REPORT_URL = '/prompt_manager/system/diagnostics';
|
||||
const PREVIEW_URL = '/prompt_manager/maintenance/preview';
|
||||
const LINE_FORMAT = /^\d\d:\d\d:\d\d [A-Z]+ +\S/;
|
||||
|
||||
/** Runs in the page: Diagnostics & logs under `mainSelector`: every surface's text, controls, geometry and tokens. */
|
||||
function measureDiagnostics([mainSelector, lineFormat]) {
|
||||
const format = new RegExp(lineFormat);
|
||||
const layout = document.querySelector(`${mainSelector} .settings-layout`);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const tokens = (node, ...names) => Object.fromEntries(names.map(key => [key, getComputedStyle(node).getPropertyValue(key).trim()]));
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const button = node => ({ text: text(node), className: node.className });
|
||||
const control = row => {
|
||||
const select = row.querySelector(':scope > select');
|
||||
if (select) return { kind: 'select', id: select.id, options: [...select.options].map(option => option.textContent),
|
||||
selected: select.selectedOptions[0]?.textContent ?? null, disabled: select.disabled };
|
||||
const toggle = row.querySelector(':scope > * input[type="checkbox"]');
|
||||
return toggle ? { kind: 'switch', id: toggle.id, wrapper: name(toggle.parentElement), checked: toggle.checked,
|
||||
disabled: toggle.disabled } : null;
|
||||
};
|
||||
const operations = parent => [...parent.querySelectorAll(':scope > .operation')].map(row => ({ tree: [...row.children].map(name),
|
||||
title: text(row.querySelector('h3')), description: text(row.querySelector('p')),
|
||||
actions: [...row.querySelectorAll('.row-actions > button')].map(button) }));
|
||||
const toolbarItem = node => ({ name: name(node), text: node.localName === 'select' || node.localName === 'input' ? null : text(node),
|
||||
for: node.localName === 'label' ? node.htmlFor : null, id: node.id || null,
|
||||
options: node.localName === 'select' ? [...node.options].map(option => option.textContent) : null,
|
||||
// The chosen option, so a wrong default level is noticed.
|
||||
selected: node.localName === 'select' ? node.selectedOptions[0]?.textContent ?? null : null,
|
||||
input: node.localName === 'input' ? [node.type, node.min, node.max, node.value] : null,
|
||||
className: node.localName === 'button' ? node.className : null });
|
||||
const surfaces = [...layout.querySelectorAll(':scope > section.surface')];
|
||||
const pre = layout.querySelector('pre');
|
||||
const lines = (pre?.textContent ?? '').split('\n').filter(Boolean);
|
||||
const details = layout.querySelector('details');
|
||||
const firstRow = layout.querySelector('.setting-row');
|
||||
const toolbar = layout.querySelector('.toolbar');
|
||||
return {
|
||||
layout: [...layout.children].map(name),
|
||||
surfaces: surfaces.map(surface => {
|
||||
const heading = surface.querySelector(':scope > .panel-heading');
|
||||
const grid = surface.querySelector(':scope > .meta-grid');
|
||||
return {
|
||||
tree: [...surface.children].map(name),
|
||||
heading: text(surface.querySelector(':scope > h2, :scope > .panel-heading > h2')),
|
||||
panelHeading: heading && [...heading.children].map(node => [name(node), text(node)]),
|
||||
grid: grid && [...grid.children].map(entry => [name(entry), text(entry.querySelector('dt')), text(entry.querySelector('dd'))]),
|
||||
buttons: [...surface.querySelectorAll(':scope > button')].map(button),
|
||||
toolbar: surface.querySelector(':scope > .toolbar') && [...surface.querySelector(':scope > .toolbar').children].map(toolbarItem),
|
||||
pre: surface.querySelector(':scope > pre') && { tabIndex: pre.tabIndex, tree: [...pre.children].map(name), lines,
|
||||
format: lines.every(line => format.test(line)) },
|
||||
notes: [...surface.querySelectorAll(':scope > p')].map(node => [name(node), text(node), node.getAttribute('role'),
|
||||
node.getAttribute('aria-label')]),
|
||||
rows: [...surface.querySelectorAll(':scope > .setting-row')].map(row => ({ tree: [...row.children].map(name),
|
||||
text: [...row.firstElementChild.children].map(node => [name(node), text(node), node.localName === 'label' ? node.htmlFor : null]),
|
||||
control: control(row) })),
|
||||
details: surface.querySelector(':scope > details') && {
|
||||
summary: text(details.querySelector(':scope > summary')), tree: [...details.children].map(name),
|
||||
notes: [...details.querySelectorAll(':scope > p')].map(node => [name(node), text(node)]),
|
||||
buttons: [...details.querySelectorAll(':scope > button')].map(button),
|
||||
files: [...details.querySelectorAll(':scope > .operation-list > .operation')].map(row => ({
|
||||
title: text(row.querySelector('h3')), detail: text(row.querySelector('p')), actions: [...row.querySelectorAll('.row-actions > button')].map(item =>
|
||||
[text(item), item.getAttribute('aria-label')]) })),
|
||||
operations: operations(details) },
|
||||
operations: operations(surface),
|
||||
};
|
||||
}),
|
||||
tokens: tokens(surfaces[0], '--line', '--panel', '--accent'),
|
||||
surface: css(surfaces[1], 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius',
|
||||
'marginTop'),
|
||||
grid: css(layout.querySelector('.meta-grid'), 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
toolbar: css(toolbar, 'display', 'flexWrap', 'alignItems', 'columnGap', 'rowGap', 'marginBottom'),
|
||||
pre: css(pre, 'fontFamily', 'fontSize', 'backgroundColor', 'borderTopColor', 'borderTopLeftRadius', 'paddingTop', 'overflowX'),
|
||||
row: css(firstRow, 'display', 'alignItems', 'justifyContent', 'columnGap', 'paddingTop', 'paddingBottom', 'borderTopWidth',
|
||||
'borderTopColor'),
|
||||
label: css(firstRow.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
description: css(firstRow.querySelector('p'), 'fontSize', 'color', 'marginTop'),
|
||||
select: css(firstRow.querySelector('select'), 'fontSize', 'minWidth', 'maxWidth', 'backgroundColor', 'color', 'borderTopColor'),
|
||||
summary: css(details.querySelector('summary'), 'fontSize', 'fontWeight', 'color', 'cursor'),
|
||||
small: css(details.querySelector(':scope > p.small'), 'fontSize', 'color'),
|
||||
// Whether each select is as wide as the browser lays out a copy holding only its chosen option, i.e. it shows
|
||||
// that option whole (the browser's own select sizing, arrow included).
|
||||
selectsFit: [...layout.querySelectorAll('select')].map(select => {
|
||||
const copy = select.cloneNode(false);
|
||||
copy.removeAttribute('id');
|
||||
copy.append(new Option(select.selectedOptions[0]?.textContent ?? ''));
|
||||
Object.assign(copy.style, { position: 'absolute', visibility: 'hidden', width: 'max-content', minWidth: '0',
|
||||
maxWidth: 'none', flex: 'none' });
|
||||
select.parentElement.append(copy);
|
||||
const needed = copy.getBoundingClientRect().width;
|
||||
copy.remove();
|
||||
return select.getBoundingClientRect().width + 0.5 >= needed;
|
||||
}),
|
||||
pageScroll: document.documentElement.scrollWidth > document.documentElement.clientWidth,
|
||||
};
|
||||
}
|
||||
|
||||
const plural = (count, one, many = `${one}s`) => `${count.toLocaleString('en-US')} ${count === 1 ? one : many}`;
|
||||
/** The server's size label (1 KB = 1024 bytes), as the section prints it. */
|
||||
function sizeText(bytes) {
|
||||
if (bytes < 1024) return plural(bytes, 'byte');
|
||||
if (bytes < 2 ** 20) return `${(bytes / 2 ** 10).toFixed(1)} KB`;
|
||||
if (bytes < 2 ** 30) return `${(bytes / 2 ** 20).toFixed(1)} MB`;
|
||||
return `${(bytes / 2 ** 30).toFixed(1)} GB`;
|
||||
}
|
||||
|
||||
const KINDS = Object.freeze({ active: 'Active log', rotated: 'Rotated copy', other: 'Other file' });
|
||||
const RENDERER_NONE = 'No ComfyUI window is open in this browser, so its renderer is not known.';
|
||||
const COPY = Object.freeze({
|
||||
testLink: 'Write a placeholder image record to check that linking works.',
|
||||
runtimeLevel: 'How much the server writes to the log. Applied until the server restarts.',
|
||||
console: 'Print Prompt Manager’s warnings and errors in the ComfyUI console (everything at DEBUG). Applied until the server restarts.',
|
||||
clear: 'Empty the active log and delete its rotated copies.',
|
||||
retention: 'Rotates at 10 MB · keeps 5 rotated copies · file logging on · memory buffer 1,000 entries',
|
||||
});
|
||||
|
||||
/**
|
||||
* The concept's Diagnostics & logs with the recorded deviations applied, for the server's `report` and the seeded
|
||||
* `files` (`/logs/files`). The concept's own sample values the fixture reproduces stay as they are: the four log lines,
|
||||
* the Viewer.js line, "prompt_manager.log · 1.4 MB · rotated copies: 3", INFO and console logging off.
|
||||
*/
|
||||
function mapDiagnosticsConcept(reference, { report, files, stats }) {
|
||||
const [diagnostics, logs] = reference.surfaces;
|
||||
const [levelLabel, level, refresh, download] = logs.toolbar;
|
||||
const [runtime, consoleRow] = logs.rows;
|
||||
const database = report.checks.database;
|
||||
const { configured, watching, unavailable } = report.sources;
|
||||
const sources = configured === 0 ? `ComfyUI output folder · ${watching ? 'watching' : 'manual scan'}`
|
||||
: unavailable > 0 ? `${plural(configured, 'source')} · ${unavailable} not available`
|
||||
: `${plural(configured, 'source')} · ${watching} watching`;
|
||||
const errors = (stats.level_counts.ERROR ?? 0) + (stats.level_counts.CRITICAL ?? 0);
|
||||
return {
|
||||
...reference,
|
||||
surfaces: [
|
||||
{ ...diagnostics,
|
||||
// D110: the Test image link operation follows the concept's Image-link diagnostics.
|
||||
tree: [...diagnostics.tree, 'div.operation'],
|
||||
// D110: live values. Database and Sources from `GET /system/diagnostics`, Host renderers is D106's
|
||||
// renderer line (no ComfyUI window in this browser); the pinned Viewer.js line is the concept's own.
|
||||
grid: diagnostics.grid.map(([entry, term, value]) => [entry, term, {
|
||||
Database: `Readable · ${sizeText(report.database_info.database_size_bytes)} · ${plural(database.image_count, 'image')}`,
|
||||
Sources: sources,
|
||||
'Host renderers': RENDERER_NONE }[term] ?? value]),
|
||||
// D110: the buttons drop the concept's "sample".
|
||||
buttons: diagnostics.buttons.map(item => ({ ...item, text: item.text.replace('Download sample diagnostic report',
|
||||
'Download diagnostic report') })),
|
||||
operations: [
|
||||
// D110: "Inspect" opens the read-only Inspect a file page.
|
||||
...diagnostics.operations.map(row => ({ ...row, actions: row.actions.map(item => ({ ...item, text: 'Inspect' })) })),
|
||||
// D110: the legacy synthetic link (S12-11), enabled: the fixture has a most recent prompt.
|
||||
{ ...diagnostics.operations[0], title: 'Test image link', description: COPY.testLink,
|
||||
actions: [{ ...diagnostics.operations[0].actions[0], text: 'Create test link…' }] },
|
||||
] },
|
||||
{ ...logs,
|
||||
// D109: the statistics line follows the viewer.
|
||||
tree: [logs.tree[0], logs.tree[1], logs.tree[2], 'p.small', ...logs.tree.slice(3)],
|
||||
toolbar: [
|
||||
levelLabel,
|
||||
// D109: all six minimum levels (the concept has three).
|
||||
{ ...level, options: ['All levels', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] },
|
||||
// D109: the added Lines control, 1–500, default 100.
|
||||
{ name: 'label.small', text: 'Lines', for: 'log-limit', id: null, options: null, selected: null, input: null,
|
||||
className: null },
|
||||
{ name: 'input', text: null, for: null, id: 'log-limit', options: null, selected: null,
|
||||
input: ['number', '1', '500', '100'], className: null },
|
||||
refresh,
|
||||
// D109: the active log, not a sample.
|
||||
{ ...download, text: 'Download log' },
|
||||
],
|
||||
// D109: "Log statistics" (errors are ERROR + CRITICAL).
|
||||
notes: [['p.small', `${plural(stats.buffer_count, 'entry', 'entries')} in memory · ${plural(errors, 'error')} · `
|
||||
+ `${plural(stats.level_counts.WARNING ?? 0, 'warning')} · ${plural(stats.log_files_count, 'file')} · `
|
||||
+ `${sizeText(stats.total_log_size)} on disk`, 'status', 'Log statistics']],
|
||||
rows: [
|
||||
// D109: CRITICAL after the concept's levels; the value lasts until the server restarts.
|
||||
{ ...runtime, text: [runtime.text[0], ['p', COPY.runtimeLevel, null]],
|
||||
control: { ...runtime.control, options: [...runtime.control.options, 'CRITICAL'] } },
|
||||
// D109: what the console setting does and that it lasts until the server restarts (off, as the
|
||||
// concept's sample, by the fixture's seed).
|
||||
{ ...consoleRow, text: [consoleRow.text[0], ['p', COPY.console, null]] },
|
||||
],
|
||||
details: { ...logs.details,
|
||||
// D109: the retention line, the files and Clear all logs follow the concept's line and review.
|
||||
tree: [...logs.details.tree, 'p.small', 'div.operation-list', 'div.operation'],
|
||||
notes: [...logs.details.notes, ['p.small', COPY.retention]],
|
||||
files: files.map(row => ({ title: row.filename, detail: `${sizeText(row.size)} · ${row.modified.slice(0, 16)
|
||||
.replace('T', ' ')} · ${KINDS[row.kind]}`, actions: [['Download', `Download ${row.filename}`]] })),
|
||||
operations: [{ tree: ['div', 'div.row-actions'], title: 'Clear all logs', description: COPY.clear,
|
||||
actions: [{ text: 'Clear all logs…', className: 'danger' }] }] } },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptDiagnostics(page, theme) {
|
||||
await openConcept(page, 'system', theme);
|
||||
await page.locator('#main .section-nav').getByRole('button', { name: 'Diagnostics & logs', exact: true }).click();
|
||||
await expect(page.locator('#main .settings-layout > section.surface')).toHaveCount(2);
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureDiagnostics, ['#main', LINE_FORMAT.source]);
|
||||
}
|
||||
|
||||
/** Production's section once every read answered, with the server's report, files and statistics it shows. */
|
||||
async function productionDiagnostics(page, request, theme) {
|
||||
await openProduction(page, 'system/diagnostics', theme);
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(main.locator('.settings-layout > section.surface')).toHaveCount(2);
|
||||
await expect(main.locator('.meta-grid dd')).not.toContainText(['Loading…']);
|
||||
await expect(main.locator('.meta-grid dd').last()).toHaveText(RENDERER_NONE);
|
||||
await expect(main.locator('#log-lines')).not.toHaveText('Loading…');
|
||||
await expect(main.locator('#log-stats')).not.toHaveText('Loading…');
|
||||
await expect(main.locator('details > p.small')).not.toContainText(['Loading…']);
|
||||
await expect(main.locator('#runtime-level')).toBeEnabled();
|
||||
await expect(main.locator('.operation').filter({ hasText: 'Create test link…' }).locator('button')).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await page.mouse.move(0, 0);
|
||||
const production = await page.evaluate(measureDiagnostics, ['#workspace-main', LINE_FORMAT.source]);
|
||||
const read = async url => (await request.get(url)).json();
|
||||
const report = await read(REPORT_URL);
|
||||
const { files } = await read('/prompt_manager/logs/files');
|
||||
const { stats } = await read('/prompt_manager/logs/stats');
|
||||
return { production, report, files, stats };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`diagnostics and logs match the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const concept = await conceptDiagnostics(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-diagnostics-${theme}-${width}.png`), fullPage: true });
|
||||
const { production, ...live } = await productionDiagnostics(page, request, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-diagnostics-${theme}-${width}.png`), fullPage: true });
|
||||
expect(diffInventory(mapDiagnosticsConcept(concept, live), production)).toEqual([]);
|
||||
expect(production.surfaces[1].pre.format).toBe(true);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
}
|
||||
|
||||
// Below the concept's smallest size: on a 320px phone the Level and Runtime log level selects still show their whole
|
||||
// option, without page scroll.
|
||||
for (const theme of THEMES) test(`diagnostics selects fit at 320 wide ${theme}`, async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 320, height: 700 });
|
||||
const { production } = await productionDiagnostics(page, request, theme);
|
||||
expect(production.selectsFit).toEqual([true, true]);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
|
||||
/**
|
||||
* D100 with D109's rule for this plan: the server's `rotated_logs` plan in the concept's dialog, "Rotated log copies"
|
||||
* as its `h3`, the plan's grid values (its Backup row says none is needed), "Listed files (N)", the hold line, no D55
|
||||
* backup box (nothing in the library changes) and "Apply reviewed operation".
|
||||
*/
|
||||
function mapCleanupConcept(reference, plan) {
|
||||
const mapped = mapPlanConcept(reference, 'Rotated log copies', plan);
|
||||
return {
|
||||
...mapped,
|
||||
// D109: no backup box for a plan that changes no library record.
|
||||
tree: mapped.tree.filter(item => item !== 'div.notice'),
|
||||
grid: { ...mapped.grid, rows: mapped.grid.rows.map(([term, value]) =>
|
||||
[term, term === 'Backup' ? 'Not needed: no library record changes' : value]) },
|
||||
backup: null,
|
||||
// D109: the plan lists files, not records.
|
||||
details: { ...mapped.details, summary: `Listed files (${plan.details.length})` },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptCleanup(page, theme) {
|
||||
await conceptDiagnostics(page, theme);
|
||||
await page.locator('#main details > summary').click();
|
||||
await page.locator('#main [data-plan="rotated logs"]').click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review maintenance plan');
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureReview);
|
||||
}
|
||||
|
||||
async function productionCleanup(page, request, theme) {
|
||||
await productionDiagnostics(page, request, theme);
|
||||
await page.locator('#workspace-main details > summary').click();
|
||||
const answer = page.waitForResponse(response => response.url().endsWith(PREVIEW_URL));
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Review log cleanup', exact: true }).click();
|
||||
const plan = await (await answer).json();
|
||||
await expect(page.locator('dialog#modal[open] #modal-body > h3')).toHaveText('Rotated log copies');
|
||||
await page.mouse.move(0, 0);
|
||||
return { plan, production: await page.evaluate(measureReview) };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`log cleanup review matches the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const concept = await conceptCleanup(page, theme);
|
||||
expect(concept.heading.text).toBe('rotated logs');
|
||||
expect(concept.tree).toEqual(['p.demo-note', 'h3', 'dl.meta-grid', 'p.small', 'div.dialog-actions']);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-log-cleanup-${theme}-${width}.png`) });
|
||||
const { plan, production } = await productionCleanup(page, request, theme);
|
||||
expect(plan.operation).toBe('rotated_logs');
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-log-cleanup-${theme}-${width}.png`) });
|
||||
expect(diffInventory(mapCleanupConcept(concept, plan), production)).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
// Every path where production differs from the unmapped concept: D110's live values (Viewer.js is the concept's own),
|
||||
// renamed buttons and added operation, and D109's toolbar, statistics line, runtime rows and log files.
|
||||
const DEVIATED_PATHS = [
|
||||
'surfaces.0.buttons.0', 'surfaces.0.grid.0', 'surfaces.0.grid.1', 'surfaces.0.grid.3', 'surfaces.0.operations.0',
|
||||
'surfaces.0.operations.1', 'surfaces.0.operations.length', 'surfaces.0.tree.4', 'surfaces.0.tree.length',
|
||||
'surfaces.1.details.files', 'surfaces.1.details.notes', 'surfaces.1.details.operations', 'surfaces.1.details.tree',
|
||||
'surfaces.1.notes.0', 'surfaces.1.notes.length', 'surfaces.1.rows.0', 'surfaces.1.rows.1', 'surfaces.1.toolbar.1',
|
||||
'surfaces.1.toolbar.2', 'surfaces.1.toolbar.3', 'surfaces.1.toolbar.4', 'surfaces.1.toolbar.5', 'surfaces.1.toolbar.length',
|
||||
'surfaces.1.tree.3', 'surfaces.1.tree.5', 'surfaces.1.tree.6', 'surfaces.1.tree.length'].sort();
|
||||
|
||||
test('diagnostics comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const concept = await conceptDiagnostics(page, 'dark');
|
||||
const { production, ...live } = await productionDiagnostics(page, request, 'dark');
|
||||
const expected = mapDiagnosticsConcept(concept, live);
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Unmapped, the section differs only where D109 and D110 are recorded.
|
||||
expect(diffInventory(concept, production).map(item => item.path.split('.').slice(0, 4).join('.'))
|
||||
.filter((path, index, all) => all.indexOf(path) === index).sort()).toEqual(DEVIATED_PATHS);
|
||||
const measure = () => page.evaluate(measureDiagnostics, ['#workspace-main', LINE_FORMAT.source]);
|
||||
const paths = async () => diffInventory(expected, await measure()).map(item => item.path);
|
||||
// A removed term.
|
||||
const undoTerm = await mutate(page, { remove: ['#workspace-main .meta-grid > div:nth-child(2)'] });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.0.grid.length']));
|
||||
await undoTerm();
|
||||
// A renamed button.
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main section.surface > button').textContent = 'Download report'; });
|
||||
expect(await paths()).toEqual(['surfaces.0.buttons.0.text']);
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main section.surface > button').textContent = 'Download diagnostic report'; });
|
||||
// A reordered level option.
|
||||
await page.evaluate(() => { const select = document.querySelector('#workspace-main #log-level'); select.append(select.options[1]); });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.1.toolbar.1.options.1']));
|
||||
await page.evaluate(() => { const select = document.querySelector('#workspace-main #log-level'); select.insertBefore(select.options[5], select.options[1]); });
|
||||
// A wrong default level.
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main #log-level').value = 'INFO'; });
|
||||
expect(await paths()).toEqual(['surfaces.1.toolbar.1.selected']);
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main #log-level').value = ''; });
|
||||
// A primary Refresh.
|
||||
await page.evaluate(() => { document.querySelectorAll('#workspace-main .toolbar > button')[0].classList.add('ca-primary'); });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.1.toolbar.4.className']));
|
||||
await page.evaluate(() => { document.querySelectorAll('#workspace-main .toolbar > button')[0].classList.remove('ca-primary'); });
|
||||
// A select squeezed until its chosen option is cut.
|
||||
const undoSqueeze = await mutate(page, { css: '#workspace-main #runtime-level { max-width: 30px !important; min-width: 0 !important; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['selectsFit.1']));
|
||||
await undoSqueeze();
|
||||
// A swapped token.
|
||||
const undoToken = await mutate(page, { css: '#workspace-main .settings-layout { --line: rgb(255, 0, 0); }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['tokens.--line', 'row.borderTopColor']));
|
||||
await undoToken();
|
||||
expect(diffInventory(expected, await measure())).toEqual([]);
|
||||
|
||||
// The log cleanup review: a changed grid term, a backup box that came back and a grid token are noticed.
|
||||
const conceptReview = await conceptCleanup(page, 'dark');
|
||||
const { plan } = await productionCleanup(page, request, 'dark');
|
||||
const expectedReview = mapCleanupConcept(conceptReview, plan);
|
||||
const reviewPaths = async () => diffInventory(expectedReview, await page.evaluate(measureReview)).map(item => item.path);
|
||||
expect(await reviewPaths()).toEqual([]);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[2].textContent = 'Safety'; });
|
||||
expect(await reviewPaths()).toEqual(['grid.rows.2.0']);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[2].textContent = 'Backup'; });
|
||||
const undoGrid = await mutate(page, { css: 'dialog#modal dl.meta-grid { column-gap: 3px; }' });
|
||||
expect(await reviewPaths()).toEqual(expect.arrayContaining(['grid.columnGap']));
|
||||
await undoGrid();
|
||||
expect(await reviewPaths()).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate, measureReview, mapPlanConcept }
|
||||
from './concept-inventory.mjs';
|
||||
|
||||
// #249 System S7 Task 20: System › Maintenance (app.js renderSystem 688–690), its plan dialog (`maintenancePlan`,
|
||||
// 953–958) for each operation and its duplicates dialog (`duplicates`, 1457–1461) against the immutable concept, on
|
||||
@@ -103,47 +104,6 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
});
|
||||
}
|
||||
|
||||
/** Runs in the page: the open review dialog (plan or duplicates): its title, body and the concept's tokens. */
|
||||
function measureReview() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const body = dialog.querySelector('#modal-body');
|
||||
const grid = body.querySelector(':scope > dl.meta-grid');
|
||||
const compare = body.querySelector('.compare-grid');
|
||||
const item = compare?.querySelector('.compare-item');
|
||||
const notes = [...body.querySelectorAll(':scope > p.small')];
|
||||
const note = notes.at(-1);
|
||||
const actions = body.querySelector(':scope > .dialog-actions');
|
||||
return {
|
||||
title: text(dialog.querySelector('#modal-title')),
|
||||
tree: [...body.children].map(name),
|
||||
heading: body.querySelector(':scope > h3') && { text: text(body.querySelector(':scope > h3')),
|
||||
...css(body.querySelector(':scope > h3'), 'fontSize', 'fontWeight', 'color', 'marginBottom') },
|
||||
grid: grid && { rows: [...grid.children].map(row => [text(row.querySelector('dt')), text(row.querySelector('dd'))]),
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
dt: css(grid.querySelector('dt'), 'fontSize', 'color', 'textTransform'), dd: css(grid.querySelector('dd'), 'fontSize', 'color') },
|
||||
details: body.querySelector(':scope > details') && { summary: text(body.querySelector(':scope > details > summary')),
|
||||
tree: [...body.querySelector(':scope > details').children].map(name) },
|
||||
backup: body.querySelector(':scope > .notice') && { tree: [...body.querySelector(':scope > .notice').children].map(name),
|
||||
text: [...body.querySelectorAll(':scope > .notice > div:first-child > *')].map(node => [node.localName, text(node)]),
|
||||
buttons: [...body.querySelectorAll(':scope > .notice button')].map(text) },
|
||||
compare: compare && { tree: [...compare.children].map(name), ...css(compare, 'display', 'gridTemplateColumns', 'columnGap'),
|
||||
items: [...compare.children].map(entry => ({ tree: [...entry.children].map(name), label: text(entry.querySelector('label')),
|
||||
checked: entry.querySelector('input[type=radio]').checked, image: entry.querySelector('img')?.className ?? null })),
|
||||
item: css(item, 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius'),
|
||||
image: css(item.querySelector('img'), 'width', 'objectFit'),
|
||||
label: css(item.querySelector('label'), 'fontSize', 'color'),
|
||||
radio: { name: item.querySelector('input[type=radio]').name.startsWith('keep-') ? 'keep-*' : 'other',
|
||||
type: item.querySelector('input').type } },
|
||||
note: note && { text: text(note), ...css(note, 'fontSize', 'color', 'marginTop') },
|
||||
actions: { tree: [...actions.children].map(name), ...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(item => ({ text: text(item), className: item.className,
|
||||
...css(item, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor') })) },
|
||||
};
|
||||
}
|
||||
|
||||
// The concept's plan names (`data-plan`) and production's operation, row and button for each.
|
||||
const PLANS = Object.freeze([
|
||||
['duplicate prompts', 'duplicate_prompts', 'Duplicate prompt text'],
|
||||
@@ -151,32 +111,6 @@ const PLANS = Object.freeze([
|
||||
['unused prompt components', 'unused_components', 'Unused prompt components'],
|
||||
['missing-file links', 'missing_links', 'Missing-file links'],
|
||||
]);
|
||||
const BACKUP_BOX = Object.freeze({ tree: ['div', 'div.row-actions'], text: [['h3', 'Back up the database first (recommended)'],
|
||||
['p', 'Downloads a copy of the prompts database before this operation changes any records.']], buttons: ['Back up now'] });
|
||||
|
||||
/**
|
||||
* D100: the server's plan in the concept's dialog: the operation's row title as its `h3` (the concept prints its
|
||||
* `data-plan` name), the four rows with the plan's values, the listed records (`details`), the hold line in the
|
||||
* concept's `p.small`, the D55 backup box before the actions, and "Apply reviewed operation". No sample note.
|
||||
*/
|
||||
function mapPlanConcept(reference, title, plan) {
|
||||
const backup = plan.backup.policy === 'off' ? 'No safety backup (the policy is off)'
|
||||
: `Create and verify before changes (${plan.backup.destination_label})`;
|
||||
const tree = ['h3', 'dl.meta-grid', ...(plan.details.length ? ['details'] : []), 'p.small', 'div.notice', 'div.dialog-actions'];
|
||||
const [cancel, apply] = reference.actions.buttons;
|
||||
return {
|
||||
...reference,
|
||||
tree,
|
||||
heading: { ...reference.heading, text: title },
|
||||
grid: { ...reference.grid, rows: [['Scope', plan.scope.label],
|
||||
['Protected', plan.protected.reasons.length ? plan.protected.reasons.join(' · ') : 'Nothing is protected'],
|
||||
['Backup', backup], ['Media on disk', plan.media.label]] },
|
||||
details: plan.details.length ? { summary: `Listed records (${plan.details.length})`, tree: ['summary', 'pre'] } : null,
|
||||
backup: BACKUP_BOX,
|
||||
note: { ...reference.note, text: 'Nothing changes until you apply. This review holds for 15 minutes.' },
|
||||
actions: { ...reference.actions, buttons: [cancel, { ...apply, text: 'Apply reviewed operation' }] },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptPlan(page, theme, name) {
|
||||
await conceptMaintenance(page, theme);
|
||||
|
||||
@@ -33,7 +33,7 @@ test.afterEach(async ({ page, request, browserName }) => {
|
||||
const PREVIEW_URL = '/prompt_manager/autotag/batch/preview';
|
||||
const MODEL_OPTIONS = ['WD14 SwinV2 · ~400 MB', 'WD14 ViT · ~400 MB', 'GGUF JoyCaption · ~4.5 GB', 'HuggingFace JoyCaption · ~16 GB'];
|
||||
const SUGGESTIONS = 'From 1 to 100: the most suggestions Generate shows for one image.';
|
||||
const BATCH = 'Add tags to each prompt from its linked output images with the model above. Review the scope first.';
|
||||
const BATCH = 'Tag each output image from its own pixels with the model above. Prompts are not changed. Review the scope first.';
|
||||
|
||||
/** Runs in the page: Auto-tagging's surface under `mainSelector`: rows, field, actions, operation, geometry and tokens. */
|
||||
function measureTagging(mainSelector) {
|
||||
@@ -127,9 +127,9 @@ function mapTaggingConcept(reference, { template, phone }) {
|
||||
actions: { ...reference.actions, buttons: [preview, ['Download again', ''], ['Unload', '']] },
|
||||
// D134: the prompt-tag batch, reviewed first.
|
||||
operations: [{ tree: ['div', 'div.row-actions'], title: 'Tag output images', description: BATCH, actions: [['Review', '']] }],
|
||||
// D132: the engine select shows its whole chosen option at every width (at 390 the concept's 120px cap cuts it;
|
||||
// `#tag-model` alone may grow there). Existing tags keeps the concept's own fit, cut or not.
|
||||
selectsFit: reference.selectsFit.map((fit, index) => (index === 0 ? true : fit)),
|
||||
// D132 (#398): every select shows its whole chosen option at every width (at 390 the concept's 120px cap cuts
|
||||
// them; production lets each grow past it).
|
||||
selectsFit: reference.selectsFit.map(() => true),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -168,16 +168,28 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
});
|
||||
}
|
||||
|
||||
// D132 below the concept's smallest size: on a 320px phone the engine select still shows its whole option.
|
||||
for (const theme of THEMES) test(`auto-tagging engine select fits at 320 wide ${theme}`, async ({ page }) => {
|
||||
await page.setViewportSize({ width: 320, height: 700 });
|
||||
const production = await productionTagging(page, theme);
|
||||
for (const model of ['hf', 'gguf', 'wd14-swinv2']) {
|
||||
await page.locator('#workspace-main #tag-model').selectOption(model);
|
||||
expect((await page.evaluate(measureTagging, '#workspace-main')).selectsFit[0], model).toBe(true);
|
||||
}
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
// D132 (#398) at phone width and below the concept's smallest size: every option of every select, once chosen, shows
|
||||
// whole, with no page scroll.
|
||||
for (const theme of THEMES) for (const width of [390, 320]) {
|
||||
test(`auto-tagging selects fit every option at ${width} wide ${theme}`, async ({ page }, testInfo) => {
|
||||
await page.setViewportSize({ width, height: 844 });
|
||||
await productionTagging(page, theme);
|
||||
await page.locator('#workspace-main #tag-existing').selectOption('replace_after_review');
|
||||
await page.locator('#workspace-main section.surface').screenshot({ path: testInfo.outputPath(`production-tagging-selects-${theme}-${width}.png`) });
|
||||
const selects = page.locator('#workspace-main section.surface > .setting-row > select');
|
||||
await expect(selects).toHaveCount(2);
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
const select = selects.nth(index);
|
||||
const values = await select.evaluate(node => [...node.options].map(option => option.value));
|
||||
for (const value of values) {
|
||||
await select.selectOption(value);
|
||||
const measured = await page.evaluate(measureTagging, '#workspace-main');
|
||||
expect(measured.selectsFit, `${await select.getAttribute('id')}=${value}`).toEqual([true, true]);
|
||||
expect(measured.pageScroll, value).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('auto-tagging comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
@@ -222,6 +234,9 @@ test('auto-tagging comparison detects mutations', async ({ page, request }) => {
|
||||
const undoSqueeze = await mutate(page, { css: '#workspace-main #tag-model { max-width: 60px !important; min-width: 0 !important; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['selectsFit.0']));
|
||||
await undoSqueeze();
|
||||
const undoSqueezeExisting = await mutate(page, { css: '#workspace-main #tag-existing { max-width: 60px !important; min-width: 0 !important; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['selectsFit.1']));
|
||||
await undoSqueezeExisting();
|
||||
// A row that wraps where the concept's does not.
|
||||
const undoWrap = await mutate(page, { css: '#workspace-main .setting-row:has(> #tag-existing) { flex-wrap: wrap; }' });
|
||||
expect(await paths()).toEqual(['rows.1.flexWrap']);
|
||||
@@ -233,14 +248,14 @@ test('auto-tagging comparison detects mutations', async ({ page, request }) => {
|
||||
expect(diffInventory(expected, await measure())).toEqual([]);
|
||||
});
|
||||
|
||||
// D132 at phone width, unmapped: only the engine row wraps; Existing tags keeps the concept's cut select.
|
||||
// D132 at phone width: only the engine row wraps; Existing tags grows beside its text instead (#398).
|
||||
test('auto-tagging at 390 wraps only the engine row', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
const concept = await conceptTagging(page, 'dark');
|
||||
const production = await productionTagging(page, 'dark');
|
||||
expect(diffInventory(mapTaggingConcept(concept, await savedTemplate(request, page)), production)).toEqual([]);
|
||||
expect(production.rows.map(row => row.flexWrap)).toEqual(['wrap', 'nowrap', 'nowrap', 'nowrap', 'nowrap']);
|
||||
expect(production.selectsFit[1]).toBe(concept.selectsFit[1]);
|
||||
expect(production.selectsFit).toEqual([true, true]);
|
||||
});
|
||||
|
||||
/** Runs in the page: the open review dialog: its title, body and the concept's tokens. */
|
||||
@@ -272,11 +287,19 @@ function measureReview() {
|
||||
}
|
||||
|
||||
const number = value => value.toLocaleString('en-US');
|
||||
/** DEC11 (#379): the Time row the preview's estimate reads as (WD14 on the test server's processor). */
|
||||
function timeRow(plan) {
|
||||
const minutes = Math.max(1, Math.ceil(plan.estimate_seconds / 60));
|
||||
const text = minutes < 90 ? `about ${minutes} min`
|
||||
: `about ${Math.floor(minutes / 60)} h${minutes % 60 ? ` ${minutes % 60} min` : ''}`;
|
||||
return `${text} on a processor; a graphics card can be faster.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* D134: the server's batch preview in the concept's review: "Review tag output images" with "Tag output images" as its
|
||||
* `h3`, the model, policy, counts and settings in the concept's `dl.meta-grid` (Backup in the concept's wording), the
|
||||
* upper-bound sentence in the concept's `p.small`, the D55 backup box and Cancel / "Start tagging".
|
||||
* D134 (#379): the server's batch preview in the concept's review: "Review tag output images" with "Tag output images"
|
||||
* as its `h3`, the model, policy, image counts, time estimate and settings in the concept's `dl.meta-grid` (Backup in
|
||||
* the concept's wording), the estimate sentence in the concept's `p.small`, the D55 backup box and Cancel / "Start
|
||||
* tagging".
|
||||
*/
|
||||
function mapBatchConcept(reference, plan) {
|
||||
const [cancel, start] = reference.actions.buttons;
|
||||
@@ -286,13 +309,14 @@ function mapBatchConcept(reference, plan) {
|
||||
tree: ['h3', 'dl.meta-grid', 'p.small', 'div.notice', 'div.dialog-actions'],
|
||||
heading: { ...reference.heading, text: 'Tag output images' },
|
||||
grid: { ...reference.grid, rows: [['Model', 'WD14 SwinV2 · ~400 MB'], ['Existing tags', 'Merge with existing'],
|
||||
['Images linked', number(plan.images)], ['Prompts', number(plan.prompts)], ['Already tagged', number(plan.already_tagged)],
|
||||
['Prompts to tag', `up to ${number(plan.to_tag)}`], ['Settings', 'General 0.35 · Character 0.85'],
|
||||
['Images', number(plan.images)], ['Already auto-tagged', number(plan.already_tagged)],
|
||||
['Images to tag', `about ${number(plan.to_tag)}`], ['Time', timeRow(plan)],
|
||||
['Settings', `General 0.35 · Character 0.85 · up to ${number(plan.suggestion_limit)} tags per image`],
|
||||
['Backup', 'Create and verify before changes']] },
|
||||
backup: { tree: ['div', 'div.row-actions'], text: [['h3', 'Back up the database first (recommended)'],
|
||||
['p', 'Downloads a copy of the prompts database before this operation changes any records.']], buttons: ['Back up now'] },
|
||||
note: { ...reference.note, text: 'Nothing is written until you start. Missing files and files outside the output folders '
|
||||
+ 'are counted, so the prompts to tag are an upper bound; Activity reports what was tagged.' },
|
||||
note: { ...reference.note, text: 'Nothing is written until you start. The number to tag is an estimate: images changed '
|
||||
+ 'since they were tagged are tagged again, and missing files are skipped. Activity reports what was tagged.' },
|
||||
actions: { ...reference.actions, buttons: [cancel, { ...start, text: 'Start tagging' }] },
|
||||
};
|
||||
}
|
||||
@@ -308,8 +332,8 @@ async function conceptReview(page, theme) {
|
||||
|
||||
async function productionReview(page, theme) {
|
||||
await productionTagging(page, theme);
|
||||
// Every concept-fixture prompt with an image is already tagged, so under the saved Skip there is nothing to start
|
||||
// (the Cancel-only review). Merge reviews a batch with work: the full review the concept's shell is compared with.
|
||||
// Merge reviews every listed still image (#379): a batch with work, the full review the concept's shell is compared
|
||||
// with.
|
||||
await page.locator('#workspace-main #tag-existing').selectOption('merge');
|
||||
const answer = page.waitForResponse(response => response.url().endsWith(PREVIEW_URL));
|
||||
await page.locator('#workspace-main .operation').getByRole('button', { name: 'Review', exact: true }).click();
|
||||
@@ -326,8 +350,9 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
expect(concept.tree).toEqual(['p.demo-note', 'h3', 'dl.meta-grid', 'p.small', 'div.dialog-actions']);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-plan-review-${theme}-${width}.png`) });
|
||||
const { plan, production } = await productionReview(page, theme);
|
||||
// Under Merge every prompt with a linked image is tagged again: the batch has work to review.
|
||||
// Under Merge every listed still image is tagged again: the batch has work to review.
|
||||
expect(plan.to_tag).toBeGreaterThan(0);
|
||||
expect(plan.estimate_device).toBe('cpu');
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-tagging-review-${theme}-${width}.png`) });
|
||||
expect(diffInventory(mapBatchConcept(concept, plan), production)).toEqual([]);
|
||||
});
|
||||
@@ -372,9 +397,17 @@ test('tag output images review comparison detects mutations', async ({ page }) =
|
||||
const expected = mapBatchConcept(concept, plan);
|
||||
const paths = async () => diffInventory(expected, await page.evaluate(measureReview)).map(item => item.path);
|
||||
expect(await paths()).toEqual([]);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[5].textContent = 'To tag'; });
|
||||
expect(await paths()).toEqual(['grid.rows.5.0']);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[5].textContent = 'Prompts to tag'; });
|
||||
// Every label of the grid is compared (#379's rows included): a changed one is named by its row.
|
||||
for (const [index, [term]] of expected.grid.rows.entries()) {
|
||||
await page.evaluate(([at, label]) => {
|
||||
document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[at].textContent = label;
|
||||
}, [index, `${term}!`]);
|
||||
expect(await paths()).toEqual([`grid.rows.${index}.0`]);
|
||||
await page.evaluate(([at, label]) => {
|
||||
document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[at].textContent = label;
|
||||
}, [index, term]);
|
||||
}
|
||||
expect(await paths()).toEqual([]);
|
||||
const undoGrid = await mutate(page, { css: 'dialog#modal dl.meta-grid { column-gap: 3px; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['grid.columnGap']));
|
||||
await undoGrid();
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
// A restore the open workspace page has not noticed yet (#413).
|
||||
//
|
||||
// Specs that restore through the API while a page is open check what the page does with a restore it did not see:
|
||||
// its next write is refused as "Prompt storage was restored since this dialog opened". But the page's Activity store
|
||||
// reads the job list every 15 s (activity-store.mjs POLL_MS), and a read the server answers while the restore job runs
|
||||
// makes the page follow it: when it finishes, the page is shown again from the restored data (S9, index.mjs
|
||||
// onRestoreFinished), which closes any open dialog. On a slow runner a restore can take longer than the time left to
|
||||
// the next read (CI run 40646112668: the restore took 16 s), so whether the page saw it depended on timing.
|
||||
//
|
||||
// `armUnseenRestores(page)`, before the page loads, passes every job-list read of the page through one route, so it
|
||||
// knows which reads the server has not answered yet. `unseenRestore(page, restoring)` then closes the gate, waits
|
||||
// for those reads' answers (sent before the restore began, they cannot list it), runs `restoring()`, and only after
|
||||
// it has answered lets the held reads go on: they read the restore as already ended, as a tab that read its job list
|
||||
// just before and just after the restore does. A restore that is never seen running never reloads the page.
|
||||
const JOBS = '**/prompt_manager/jobs';
|
||||
const gates = new WeakMap();
|
||||
|
||||
/** Route `page`'s job-list reads through the restore gate. Call before the page's first navigation. */
|
||||
export async function armUnseenRestores(page) {
|
||||
const gate = { closed: null, inFlight: 0, idle: [] };
|
||||
gates.set(page, gate);
|
||||
await page.route(JOBS, async route => {
|
||||
if (gate.closed) {
|
||||
await gate.closed;
|
||||
return route.fallback();
|
||||
}
|
||||
gate.inFlight += 1;
|
||||
try {
|
||||
const response = await route.fetch();
|
||||
await route.fulfill({ response }).catch(() => {});
|
||||
} catch {
|
||||
await route.abort().catch(() => {});
|
||||
} finally {
|
||||
gate.inFlight -= 1;
|
||||
if (gate.inFlight === 0) for (const resolve of gate.idle.splice(0)) resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Run `restoring()` (an API restore) unseen by `page`'s Activity store; resolves with what it resolves with. */
|
||||
export async function unseenRestore(page, restoring) {
|
||||
const gate = gates.get(page);
|
||||
if (!gate) throw new Error('armUnseenRestores(page) must run before the page loads');
|
||||
let open;
|
||||
gate.closed = new Promise(resolve => { open = resolve; });
|
||||
try {
|
||||
// Reads already on their way were sent before the restore: their answers cannot list it.
|
||||
if (gate.inFlight > 0) await new Promise(resolve => gate.idle.push(resolve));
|
||||
return await restoring();
|
||||
} finally {
|
||||
gate.closed = null;
|
||||
open();
|
||||
}
|
||||
}
|
||||
@@ -42,8 +42,13 @@ test('actual gallery form saves a prompt that the admin page can edit and reload
|
||||
await page.locator(`[data-action="edit"][data-prompt-id="${id}"]`).click();
|
||||
const editor = page.locator(`.prompt-text[data-id="${id}"]`);
|
||||
await editor.fill('Browser edited portrait');
|
||||
// The page shows the edit at once and saves it after: reload only once the save answered (a reload that wins the
|
||||
// race abandons the PUT, as seen on WebKit in CI).
|
||||
const savedEdit = page.waitForResponse(item => item.request().method() === 'PUT'
|
||||
&& item.url().endsWith(`/prompt_manager/prompts/${id}`));
|
||||
await editor.press('Enter');
|
||||
await expect(page.locator('.prompt-text').filter({ hasText: 'Browser edited portrait' })).toBeVisible();
|
||||
expect((await savedEdit).ok()).toBe(true);
|
||||
await page.reload();
|
||||
await expect(page.locator('.prompt-text').filter({ hasText: 'Browser edited portrait' })).toBeVisible();
|
||||
const saved = await request.post('/prompt_manager/save', { data: { text: 'Browser saved landscape' } });
|
||||
@@ -127,7 +132,8 @@ test('the gallery cache clear keeps preferences and other tools caches', async (
|
||||
}))).toEqual({ theme: 'light', other: 'keep', prefixed: 'keep', notice: '3.2.0', caches: ['other-tool-gallery-thumbnails'] });
|
||||
});
|
||||
|
||||
test('actual gallery review generates and persists tags for the selected root', async ({ page, request }) => {
|
||||
// #379: the reviewed tags are that image's own tags; no prompt changes.
|
||||
test('actual gallery review generates and persists tags for the selected root\'s image', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2);
|
||||
await page.locator('#autoTagBtn').click();
|
||||
@@ -135,13 +141,15 @@ test('actual gallery review generates and persists tags for the selected root',
|
||||
await page.locator('#startReviewBtn').click();
|
||||
const generation = await generated;
|
||||
expect(generation.request().postDataJSON().root_index).toBe(0);
|
||||
await expect(page.locator('#reviewTagsContainer')).toContainText('generated');
|
||||
await expect(page.locator('#reviewTagsContainer')).toContainText('sandstone');
|
||||
const result = page.waitForResponse(response => response.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
await page.locator('#applyReviewBtn').click();
|
||||
expect((await (await result).json()).success).toBe(true);
|
||||
const data = await (await request.get('/prompt_manager/recent?limit=20')).json();
|
||||
expect(data.results.find(prompt => prompt.text === 'Cat portrait').tags).toContain('generated');
|
||||
expect(data.results.find(prompt => prompt.text === 'Dog landscape').tags).not.toContain('generated');
|
||||
expect(data.results.find(prompt => prompt.text === 'Cat portrait').tags).toEqual(['cats']);
|
||||
expect(data.results.find(prompt => prompt.text === 'Dog landscape').tags).toEqual(['dogs']);
|
||||
const tagged = await (await request.get('/prompt_manager/images/output?mode=workspace&tags=sandstone')).json();
|
||||
expect(tagged.images.map(image => `${image.root_index}:${image.path}`)).toEqual(['0:shared/same.png']);
|
||||
});
|
||||
|
||||
test('actual Tags page reloads the selected filter and opens its image viewer', async ({ page }) => {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
|
||||
import {readFile,writeFile} from 'node:fs/promises';
|
||||
import {createHash} from 'node:crypto';
|
||||
|
||||
@@ -16,13 +17,13 @@ const marker=(page,id,anchor='10')=>recorded(page).locator(`button[data-run-id="
|
||||
const exact=async locator=>locator.evaluate(node=>node.textContent);
|
||||
|
||||
test.beforeEach(async({page,request,browserName})=>{
|
||||
await startCoverage(page,browserName);page.historyErrors=[];page.on('pageerror',error=>page.historyErrors.push(error.message));
|
||||
await startCoverage(page,browserName);await armUnseenRestores(page);page.historyErrors=[];page.on('pageerror',error=>page.historyErrors.push(error.message));
|
||||
const reset=await request.post('/__test__/reset',{data:{chronological_history:true}});await expect(reset).toBeOK();page.fixture=await reset.json();
|
||||
});
|
||||
test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.historyErrors).toEqual([]);});
|
||||
|
||||
const command=async(request,data)=>{const response=await request.post('/__test__/history/commands',{data});await expect(response).toBeOK();return response.json();};
|
||||
const restoreSnapshot=async(request,bytes)=>{const response=await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'history.db',mimeType:'application/octet-stream',buffer:bytes}}});await expect(response).toBeOK();expect((await response.json()).success).toBe(true);};
|
||||
const restoreSnapshot=async(page,request,bytes)=>{const response=await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'history.db',mimeType:'application/octet-stream',buffer:bytes}}}));await expect(response).toBeOK();expect((await response.json()).success).toBe(true);};
|
||||
const historyRead=async(request,id)=>{const response=await request.get(`/prompt_manager/library/${id}/history`);await expect(response).toBeOK();return response.json();};
|
||||
const frozenContexts=contexts=>contexts.map(({navigation,positive_prompt_id,negative_prompt_id,pair_id,...frozen})=>frozen);
|
||||
const immutableReceipts=items=>items.map(({current_candidate,...receipt})=>receipt);
|
||||
@@ -62,7 +63,7 @@ for(const boundary of ['window','detail','candidate','inspection','media']) test
|
||||
await command(request,{action:'arm',boundary});const pending=request.get(target);
|
||||
try {
|
||||
await expect.poll(async()=> (await(await request.get('/__test__/history/checkpoint')).json()).entered).toBe(true);
|
||||
await restoreSnapshot(request,bytes);
|
||||
await restoreSnapshot(page,request,bytes);
|
||||
} finally {await command(request,{action:'release'});}
|
||||
const retired=await pending;expect(retired.status()).toBe(409);expect((await retired.json()).code).toBe('history-changed');
|
||||
for(const url of [`/prompt_manager/library/${f.detail_id}/history?limit=1&cursor=${encodeURIComponent(firstWindow.next_cursor)}`,`${base}?epoch=${epoch}`,`${base}/outputs?epoch=${epoch}`,`${base}/outputs?epoch=${epoch}&cursor=${encodeURIComponent(outputs.next_cursor)}`,media.url,media.thumbnail_url,media.inspection_url]) {
|
||||
@@ -130,7 +131,7 @@ test('archive delete merge and prune preserve frozen evidence without repairing
|
||||
|
||||
test('actual restore with reused numeric owner rejects stale history and offers explicit saved recovery',async({page,request})=>{
|
||||
const f=page.fixture;await open(page,f.detail_id);const original=await exact(saved(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'));
|
||||
const source=await request.get('/__test__/history/reused-snapshot');await expect(source).toBeOK();await restoreSnapshot(request,await source.body());
|
||||
const source=await request.get('/__test__/history/reused-snapshot');await expect(source).toBeOK();await restoreSnapshot(page,request,await source.body());
|
||||
await marker(page,f.runs.shared).click();await expect(dialog(page)).toContainText('Recorded evidence changed');await page.keyboard.press('Escape');
|
||||
await expect(page.locator('.library-parent-status')).toContainText('changed');expect(await exact(saved(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'))).toBe(original);
|
||||
await expect(recorded(page).getByRole('button',{name:'Reload recorded history',exact:true})).toBeVisible();
|
||||
@@ -142,12 +143,12 @@ for(const outcome of ['success','error'])test(`delayed actual old browser ${outc
|
||||
const f=page.fixture;await open(page,f.detail_id);const snapshot=await request.get('/__test__/history/snapshot');const bytes=await snapshot.body();
|
||||
let entered,release;const started=new Promise(resolve=>{entered=resolve;}),held=new Promise(resolve=>{release=resolve;});
|
||||
await page.route(new RegExp(`/history/runs/${f.runs.shared}\\?`),async route=>{
|
||||
if(outcome==='error')await restoreSnapshot(request,bytes);
|
||||
if(outcome==='error')await restoreSnapshot(page,request,bytes);
|
||||
const response=await route.fetch();expect(response.status()).toBe(outcome==='success'?200:409);entered();await held;
|
||||
await route.fulfill({response}).catch(()=>{});
|
||||
});
|
||||
try {
|
||||
await marker(page,f.runs.shared).click();await started;if(outcome==='success')await restoreSnapshot(request,bytes);
|
||||
await marker(page,f.runs.shared).click();await started;if(outcome==='success')await restoreSnapshot(page,request,bytes);
|
||||
await page.evaluate(()=>{location.hash='#/overview';});await expect(page.locator('#workspace-main h1')).toHaveText('Overview');release();
|
||||
await page.unroute(new RegExp(`/history/runs/${f.runs.shared}\\?`));await open(page,f.detail_id);
|
||||
await expect(recorded(page).locator('button[data-run-id]')).toHaveCount(0);
|
||||
@@ -258,7 +259,7 @@ test('inspection first discovers a real restore and immediately retires every su
|
||||
if(changed)window.recordedRecoveryStates.push({receipts:run.querySelectorAll('[data-receipt]').length,active:[...run.querySelectorAll('[data-send-role],[data-copy-role],[data-view-receipt]')].filter(node=>!node.disabled&&node.checkVisibility()).length});
|
||||
});observer.observe(run,{subtree:true,childList:true,attributes:true,characterData:true});
|
||||
});
|
||||
await restoreSnapshot(request,await snapshot.body());
|
||||
await restoreSnapshot(page,request,await snapshot.body());
|
||||
} finally {await command(request,{action:'release'});}
|
||||
const response=await inspectionRead;expect(response.status()).toBe(409);expect((await response.json()).code).toBe('history-changed');
|
||||
await expect(dialog(page).locator('.recorded-run-status')).toContainText('Recorded evidence changed');
|
||||
|
||||
@@ -2189,3 +2189,248 @@ test('scan for metadata opens the review dialog from Import', async ({ page, req
|
||||
await expect.poll(async () => (await (await request.get('/prompt_manager/jobs')).json()).jobs
|
||||
.map(job => [job.kind, job.state])).toEqual([['backup', 'finished']]);
|
||||
});
|
||||
|
||||
// #392 (D135): file-manager selection beyond the concept's checkbox.
|
||||
const selectedCount = page => bulkbar(page).locator('.bulkbar > strong');
|
||||
const cardAt = (page, index) => page.locator(`article[data-image-key][data-index="${index}"]`);
|
||||
const boxAt = (page, index) => cardAt(page, index).getByRole('checkbox', { name: /^Select / });
|
||||
const checkedIndexes = page => page.locator('article[data-image-key]').evaluateAll(nodes => nodes
|
||||
.filter(node => node.querySelector('.image-overlays input[type="checkbox"]').checked)
|
||||
.map(node => Number(node.dataset.index)).sort((a, b) => a - b));
|
||||
const range = (from, to) => Array.from({ length: to - from + 1 }, (_, index) => from + index);
|
||||
async function bulkReset(page, request) {
|
||||
await page.addInitScript(() => { window.IntersectionObserver = undefined; });
|
||||
await expect(await request.post('/__test__/reset', { data: { extra_images: 200 } })).toBeOK();
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(40);
|
||||
}
|
||||
/** A touch press on `locator` the way a finger makes one: pointerdown, (held), pointerup. */
|
||||
const touchDown = locator => locator.evaluate(node => {
|
||||
const { left, top } = node.getBoundingClientRect();
|
||||
node.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true, isPrimary: true, pointerType: 'touch', clientX: left + 5, clientY: top + 5 }));
|
||||
});
|
||||
const touchUpAndClick = locator => locator.evaluate(node => {
|
||||
node.dispatchEvent(new PointerEvent('pointerup', { bubbles: true, isPrimary: true, pointerType: 'touch' }));
|
||||
// A tap's click counts one press (`detail` 1); a keyboard click (and `HTMLElement.click()`) counts none.
|
||||
node.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, detail: 1 }));
|
||||
});
|
||||
async function touchTap(locator) {
|
||||
await touchDown(locator);
|
||||
await touchUpAndClick(locator);
|
||||
}
|
||||
|
||||
test('Shift+click selects a range across an appended batch in Grid, List and Masonry', async ({ page, request }) => {
|
||||
await bulkReset(page, request);
|
||||
await page.getByRole('button', { name: 'Load more images', exact: true }).click();
|
||||
await expect(caption(page)).toContainText('80 of 202 images');
|
||||
for (const mode of ['Grid', 'List', 'Masonry']) {
|
||||
await page.getByRole('group', { name: 'Image layout', exact: true }).getByRole('button', { name: mode, exact: true }).click();
|
||||
// The anchor is the last image toggled on its own (List hides the concept's checkbox: the card carries it).
|
||||
await cardAt(page, 3).locator('.media-info h3').click({ modifiers: ['ControlOrMeta'] });
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
// The range runs into the second batch (index 40 on), and backwards from the same anchor.
|
||||
await cardAt(page, 45).locator('.media-info h3').click({ modifiers: ['Shift'] });
|
||||
await expect(selectedCount(page)).toHaveText('43 selected');
|
||||
expect(await checkedIndexes(page)).toEqual(range(3, 45));
|
||||
await cardAt(page, 0).locator('.media-info h3').click({ modifiers: ['Shift'] });
|
||||
await expect(selectedCount(page)).toHaveText('46 selected');
|
||||
expect(await checkedIndexes(page)).toEqual(range(0, 45));
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
if (mode !== 'List') {
|
||||
// A Shift+click on a checkbox gives its range the box's new state: here, unchecked.
|
||||
await boxAt(page, 3).uncheck();
|
||||
await boxAt(page, 10).click({ modifiers: ['Shift'] });
|
||||
await expect(boxAt(page, 10)).not.toBeChecked();
|
||||
await expect(selectedCount(page)).toHaveText('38 selected');
|
||||
expect(await checkedIndexes(page)).toEqual([0, 1, 2, ...range(11, 45)]);
|
||||
}
|
||||
await bulkbar(page).getByRole('button', { name: 'Clear', exact: true }).click();
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
}
|
||||
});
|
||||
|
||||
test('Ctrl/Cmd+click toggles a card without opening it, and a plain click still opens the viewer', async ({ page }) => {
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
const first = cards(page).first();
|
||||
const inspectorBefore = await details(page).innerText();
|
||||
await first.getByRole('button', { name: 'Open same.png', exact: true }).click({ modifiers: ['ControlOrMeta'] });
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
await expect(checkbox(first, 'same.png')).toBeChecked();
|
||||
await cards(page).last().getByRole('button', { name: 'Inspect same.png', exact: true }).click({ modifiers: ['ControlOrMeta'] });
|
||||
await expect(selectedCount(page)).toHaveText('2 selected');
|
||||
await star(first, 'same.png').click({ modifiers: ['ControlOrMeta'] });
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
await expect(star(first, 'same.png')).toHaveAttribute('aria-pressed', 'false');
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
expect(await details(page).innerText()).toBe(inspectorBefore);
|
||||
// A plain click keeps its meaning: the image opens the viewer.
|
||||
await first.getByRole('button', { name: 'Open same.png', exact: true }).click();
|
||||
await expect(page.locator('#viewer-dialog[open]')).toHaveCount(1);
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
});
|
||||
|
||||
test('Ctrl/Cmd+A, Select all N matching and Add tags tag every matching image on the server', async ({ page, request }) => {
|
||||
await bulkReset(page, request);
|
||||
// Ctrl/Cmd+A in the search box stays the field's own.
|
||||
await search(page).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await expect(selectedCount(page)).toHaveText('40 selected');
|
||||
const offer = bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true });
|
||||
await expect(offer).toBeVisible();
|
||||
await expect(bulkbar(page).locator('.bulkbar > *').nth(1)).toHaveText('Select all 202 matching');
|
||||
await offer.click();
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
await expect(offer).toHaveCount(0);
|
||||
await expect(bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true })).toHaveAttribute('aria-disabled', 'true');
|
||||
// A later batch arrives checked.
|
||||
await page.getByRole('button', { name: 'Load more images', exact: true }).click();
|
||||
await expect(caption(page)).toContainText('80 of 202 images');
|
||||
await expect(boxAt(page, 70)).toBeChecked();
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// Unchecking one image leaves "all matching": the other loaded images stay selected.
|
||||
await boxAt(page, 0).uncheck();
|
||||
await expect(selectedCount(page)).toHaveText('79 selected');
|
||||
await expect(bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true })).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await cardAt(page, 1).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }).click();
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// Add tags sends the listing filter, not 202 identities, and the server tags every match.
|
||||
await bulkbar(page).getByRole('button', { name: 'Add tags', exact: true }).click();
|
||||
const dialog = page.getByRole('dialog', { name: 'Tag selected images', exact: true });
|
||||
await dialog.getByRole('textbox', { name: 'Tags, separated by commas', exact: true }).fill('picked392');
|
||||
const started = page.waitForRequest(item => item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/jobs/image-tags');
|
||||
await dialog.getByRole('button', { name: 'Apply', exact: true }).click();
|
||||
expect((await started).postDataJSON()).toEqual({ mode: 'add', filter: { sort: 'newest' }, tags: ['picked392'] });
|
||||
await expect.poll(async () => (await (await request.get('/prompt_manager/images/output?mode=workspace&limit=1&tags=picked392')).json()).total,
|
||||
{ timeout: 20000 }).toBe(202);
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// Escape in the grid clears the selection.
|
||||
await cardAt(page, 2).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
await expect(boxAt(page, 2)).not.toBeChecked();
|
||||
});
|
||||
|
||||
test('Shift+arrow keys extend the selection from the focused card, and Escape in the bar clears it', async ({ page, request }) => {
|
||||
await bulkReset(page, request);
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('Shift+ArrowRight');
|
||||
await page.keyboard.press('Shift+ArrowRight');
|
||||
await expect(cardAt(page, 2).getByRole('button', { name: /^Open / })).toBeFocused();
|
||||
await expect(selectedCount(page)).toHaveText('3 selected');
|
||||
expect(await checkedIndexes(page)).toEqual([0, 1, 2]);
|
||||
// A plain arrow moves without selecting; Shift+End reaches the last loaded card from the same anchor.
|
||||
await page.keyboard.press('ArrowRight');
|
||||
await expect(selectedCount(page)).toHaveText('3 selected');
|
||||
await page.keyboard.press('Shift+End');
|
||||
await expect(selectedCount(page)).toHaveText('40 selected');
|
||||
await bulkbar(page).getByRole('button', { name: 'Clear', exact: true }).focus();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
await expect(page.locator('#workspace-main h1')).toBeFocused();
|
||||
});
|
||||
|
||||
test('a touch long-press starts selection mode, taps then toggle, and an empty selection ends it', async ({ page }, testInfo) => {
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
const [first, second] = [cards(page).first(), cards(page).last()];
|
||||
const open = card => card.getByRole('button', { name: 'Open same.png', exact: true });
|
||||
await touchDown(open(first));
|
||||
// The context menu of a held press belongs to the long press.
|
||||
expect(await open(first).evaluate(node => !node.dispatchEvent(new MouseEvent('contextmenu', { bubbles: true, cancelable: true })))).toBe(true);
|
||||
// Held: the long press selects the card once its delay passed.
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
await touchUpAndClick(open(first));
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
await expect(checkbox(first, 'same.png')).toBeChecked();
|
||||
// Taps now toggle instead of opening.
|
||||
if (testInfo.project.name === 'touch') await open(second).tap();
|
||||
else await touchTap(open(second));
|
||||
await expect(selectedCount(page)).toHaveText('2 selected');
|
||||
// A keyboard Enter keeps its meaning in touch selection mode: the image opens.
|
||||
await open(first).focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(page.locator('#viewer-dialog[open]')).toHaveCount(1);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(page.locator('#viewer-dialog[open]')).toHaveCount(0);
|
||||
await expect(selectedCount(page)).toHaveText('2 selected');
|
||||
await touchTap(open(first));
|
||||
await touchTap(open(second));
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
// The selection is empty, so touch selection mode ended: a tap opens the image again.
|
||||
await touchTap(open(first));
|
||||
await expect(page.locator('#viewer-dialog[open]')).toHaveCount(1);
|
||||
});
|
||||
|
||||
test('all matching sets the category through the listing filter and survives a return visit as the loaded images', async ({ page, request }) => {
|
||||
await bulkReset(page, request);
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }).click();
|
||||
// The offer left with its press: focus moves to the bar's first action.
|
||||
await expect(bulkbar(page).getByRole('button', { name: 'Add tags', exact: true })).toBeFocused();
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// Export metadata waits for loaded images: a press does nothing.
|
||||
const exportButton = bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true });
|
||||
await expect(exportButton).toHaveAttribute('title', 'Export metadata reads each image in the browser, so it works on loaded images only.');
|
||||
// No match linked (a stubbed answer): the nothing-linked sentence, and the selection stays.
|
||||
await page.route('**/prompt_manager/bulk/category', route => route.fulfill({ json: { success: true, updated_count: 0, prompt_count: 0, skipped: 202 } }), { times: 1 });
|
||||
const categorize = page.getByRole('dialog', { name: 'Categorize selected images', exact: true });
|
||||
await bulkbar(page).getByRole('button', { name: 'Set category', exact: true }).click();
|
||||
await categorize.getByRole('textbox', { name: 'Category', exact: true }).fill('Bulk392');
|
||||
await categorize.getByRole('button', { name: 'Apply', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText("None of the selected images is linked to a saved prompt, so there is no prompt category to set.");
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// The real write: one request with the filter, every matching image's positive changes.
|
||||
await bulkbar(page).getByRole('button', { name: 'Set category', exact: true }).click();
|
||||
await categorize.getByRole('textbox', { name: 'Category', exact: true }).fill('Bulk392');
|
||||
const posted = page.waitForRequest(item => item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/bulk/category');
|
||||
await categorize.getByRole('button', { name: 'Apply', exact: true }).click();
|
||||
expect((await posted).postDataJSON()).toEqual({ images_filter: { sort: 'newest' }, category: 'Bulk392' });
|
||||
await expect(toast(page)).toHaveText(/^Category set on \d+ prompts?\.$/);
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
const filtered = await (await request.get('/prompt_manager/images/output?mode=workspace&limit=1&category=Bulk392')).json();
|
||||
expect(filtered.total).toBe(202);
|
||||
// A return visit keeps "all matching" as the loaded images, and the offer shows again.
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }).click();
|
||||
await page.evaluate(() => { window.location.hash = '/overview'; });
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Overview');
|
||||
await page.evaluate(() => { window.location.hash = '/images'; });
|
||||
await expect(selectedCount(page)).toHaveText('40 selected');
|
||||
await expect(bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true })).toBeVisible();
|
||||
// Escape in the bar with nothing selected does nothing; with a selection it clears.
|
||||
await bulkbar(page).getByRole('button', { name: 'Clear', exact: true }).focus();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
});
|
||||
|
||||
test('all matching over 10,000 images says so before any dialog or request', async ({ page, request }) => {
|
||||
await page.route(url => url.pathname === '/prompt_manager/images/output' && url.searchParams.get('limit') !== '1', async route => {
|
||||
const response = await route.fetch();
|
||||
const body = await response.json();
|
||||
await route.fulfill({ response, json: { ...body, total: 20002 } });
|
||||
});
|
||||
await bulkReset(page, request);
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await bulkbar(page).getByRole('button', { name: 'Select all 20,002 matching', exact: true }).click();
|
||||
await expect(selectedCount(page)).toHaveText('All 20,002 matching selected');
|
||||
let posted = 0;
|
||||
page.on('request', item => { if (item.method() === 'POST') posted += 1; });
|
||||
for (const action of ['Set category', 'Add tags', 'Auto-tag selected']) {
|
||||
await bulkbar(page).getByRole('button', { name: action, exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Narrow the view to at most 10,000 images.');
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
}
|
||||
expect(posted).toBe(0);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {inventoryOf,mapConcept,diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
@@ -13,7 +14,7 @@ const RATINGS=[['5','5 / 5'],['4','4 / 5'],['3','3 / 5'],['2','2 / 5'],['1','1 /
|
||||
const editorDialog=page=>page.locator('dialog.library-editor');
|
||||
|
||||
test.beforeEach(async({page,request,browserName})=>{
|
||||
await startCoverage(page,browserName);
|
||||
await startCoverage(page,browserName);await armUnseenRestores(page);
|
||||
const reset=await request.post('/__test__/library-editor/reset',{data:{}});await expect(reset).toBeOK();page.fixture=await reset.json();
|
||||
page.editorErrors=[];page.on('pageerror',error=>page.editorErrors.push(error.message));
|
||||
page.writes=[];page.on('request',item=>{if(item.method()!=='GET')page.writes.push({method:item.method(),url:new URL(item.url()).pathname,body:item.postDataJSON()});});
|
||||
@@ -260,7 +261,8 @@ test('recovery states render inline and never replay a write',async({page,reques
|
||||
page.writes.length=0;
|
||||
dialog=await openEditor(page,{mode:'new'});await ready(dialog);
|
||||
const snapshot=await request.get('/__test__/library-editor/snapshot');await expect(snapshot).toBeOK();
|
||||
const restore=await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:await snapshot.body()}}});
|
||||
const buffer=await snapshot.body();
|
||||
const restore=await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer}}}));
|
||||
await expect(restore).toBeOK();page.writes.length=0;
|
||||
await field(dialog,'Title').fill('After restore');await field(dialog,'Positive prompt').fill('Restored storage prompt');await button(dialog,'Save').click();
|
||||
await expect(note(dialog)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was saved.');
|
||||
@@ -588,7 +590,7 @@ test('saved refresh failure retries only reads',async({page})=>{
|
||||
|
||||
test('restored storage after a confirmed save or before an archive offers an explicit Library reload',async({page,request})=>{
|
||||
const f=page.fixture;const snapshot=await(await request.get('/__test__/library-editor/snapshot')).body();
|
||||
const restore=()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}});
|
||||
const restore=()=>unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}}));
|
||||
await openHistory(page,f.positive_id);
|
||||
await page.route(`**/prompt_manager/library/${f.positive_id}`,async route=>{
|
||||
if(route.request().method()!=='PATCH')return route.continue();
|
||||
@@ -615,7 +617,7 @@ test('restored storage after a confirmed save or before an archive offers an exp
|
||||
const DISCARDS='Leaving this dialog discards your entries.';
|
||||
test('restored storage before an Edit or Add negative save offers only the Library reload exit',async({page,request})=>{
|
||||
const f=page.fixture;const snapshot=await(await request.get('/__test__/library-editor/snapshot')).body();
|
||||
const restore=()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}});
|
||||
const restore=()=>unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}}));
|
||||
await openHistory(page,f.positive_id);
|
||||
await page.getByRole('button',{name:'Edit metadata',exact:true}).click();let dialog=editorDialog(page);await ready(dialog);
|
||||
await expect(await restore()).toBeOK();
|
||||
|
||||
@@ -47,6 +47,17 @@ async function serverJson(request, url) {
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* The restore job's own end (#361): its progress stream closes when the job takes its final state, however long the
|
||||
* disk-bound publish takes (bounded by the test, not by a 5 s expect), and the job finished. The toast the page shows
|
||||
* for it is then the page's reaction alone, checked with the default timeout.
|
||||
*/
|
||||
async function restoreFinished(request, id) {
|
||||
await expect(await request.get(`/prompt_manager/jobs/${id}/events`)).toBeOK();
|
||||
const { job } = await serverJson(request, `/prompt_manager/jobs/${id}`);
|
||||
expect([job.kind, job.state]).toEqual(['restore', 'finished']);
|
||||
}
|
||||
|
||||
async function openBackups(page) {
|
||||
// A fresh page each time: the drafts live for the page session (Q10), and a hash-only goto would not reload.
|
||||
if (page.url().endsWith('#/system/backups')) await page.reload();
|
||||
@@ -216,6 +227,7 @@ test('Restore starts the job, opens Activity; views reload after it finishes', a
|
||||
const { job } = await (await applied).json();
|
||||
expect([job.kind, job.title, job.cancellable]).toEqual(['restore', 'Database restore', false]);
|
||||
await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`);
|
||||
await restoreFinished(request, job.id);
|
||||
await expect(toast(page)).toHaveText(/^Database restore finished · Restored \d+ prompts?, /);
|
||||
// The library is the backup's again, and the page says so without a reload: the sidebar counts were read again.
|
||||
expect(await libraryTotal(request)).toBe(restoredTotal);
|
||||
@@ -244,7 +256,8 @@ test('a finished restore shows the open page again from the restored data', asyn
|
||||
const card = page.locator('#workspace-main article.prompt-card').filter({ hasText: 'S9 prompt the restore removes' });
|
||||
await expect(card).toHaveCount(1);
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await applied;
|
||||
const { job } = await (await applied).json();
|
||||
await restoreFinished(request, job.id);
|
||||
await expect(toast(page)).toHaveText(/^Database restore finished · /);
|
||||
// The shelf was read again from the restored library, on the same route.
|
||||
await expect(card).toHaveCount(0);
|
||||
@@ -422,6 +435,7 @@ test('a refused restore reopens its review: Restore retries it, Discard releases
|
||||
await dialog(page).getByRole('button', { name: 'Restore', exact: true }).click();
|
||||
const { job } = await (await applied).json();
|
||||
await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`);
|
||||
await restoreFinished(request, job.id);
|
||||
await expect(toast(page)).toHaveText(/^Database restore finished · /);
|
||||
expect(await stagesOf(request)).toBe(0);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,611 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
|
||||
// System › Diagnostics & logs (#249 System S12, #329) on the concept-shaped fixture, with the logger writing into this
|
||||
// server's own root (workspace_diagnostics_fixture.py): the meta-grid's live values, Run checks, the report, Inspect and
|
||||
// the test link (D110); the log viewer, statistics, downloads and runtime drafts (D109); and the reviewed rotated-log
|
||||
// cleanup (D100) and Clear all logs, which delete only inside that temporary log folder.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const REPORT_URL = '/prompt_manager/system/diagnostics';
|
||||
const LOGS_URL = '/prompt_manager/logs';
|
||||
const CONFIG_URL = '/prompt_manager/logs/config';
|
||||
const PREVIEW_URL = '/prompt_manager/maintenance/preview';
|
||||
const APPLY_URL = '/prompt_manager/maintenance/apply';
|
||||
const TEST_LINK_URL = '/prompt_manager/diagnostics/test-link';
|
||||
const CHANGED = 'The log files changed since the review; nothing was deleted. Review again.';
|
||||
const KEY = 'sk-s12-browser-SECRET-0123456789';
|
||||
const LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'];
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/integrations/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(await request.post('/__test__/logs/reset')).toBeOK();
|
||||
page.systemErrors = [];
|
||||
page.on('pageerror', error => page.systemErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/release');
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/logs/reset');
|
||||
await request.post('/__test__/integrations/reset');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.systemErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
const layoutOf = page => page.locator('#workspace-main .settings-layout');
|
||||
const dialog = page => page.locator('dialog#modal[open]');
|
||||
const surfaceOf = (page, title) => layoutOf(page).locator(':scope > section.surface').filter({
|
||||
has: page.getByRole('heading', { level: 2, name: title, exact: true }) });
|
||||
const cell = (page, term) => surfaceOf(page, 'Diagnostics').locator('.meta-grid > div').filter({
|
||||
has: page.locator(`dt:text-is("${term}")`) }).locator('dd');
|
||||
const lines = page => page.locator('#log-lines > code');
|
||||
const operation = (page, title) => layoutOf(page).locator('.operation').filter({ has: page.locator(`h3:text-is("${title}")`) });
|
||||
const settingOf = (page, label) => layoutOf(page).locator('.setting-row').filter({ has: page.locator(`label:text-is("${label}")`) });
|
||||
const status = page => page.locator('#settings-status');
|
||||
const save = page => page.getByRole('button', { name: 'Save preferences', exact: true });
|
||||
|
||||
async function serverJson(request, url) {
|
||||
const response = await request.get(url);
|
||||
await expect(response).toBeOK();
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function seedLogs(request, data = {}) {
|
||||
await expect(await request.post('/__test__/logs/seed', { data })).toBeOK();
|
||||
}
|
||||
|
||||
const names = async request => (await serverJson(request, '/__test__/logs/state')).entries.map(entry => entry.name);
|
||||
|
||||
async function openDiagnostics(page) {
|
||||
if (page.url().endsWith('#/system/diagnostics')) await page.reload();
|
||||
else await page.goto(`${workspace}#/system/diagnostics`);
|
||||
await expect(layoutOf(page).locator(':scope > section.surface')).toHaveCount(2);
|
||||
await expect(lines(page)).not.toHaveText('Loading…');
|
||||
await expect(cell(page, 'Database')).not.toHaveText('Loading…');
|
||||
// The renderer line settles first: its text can move the rows below.
|
||||
await expect(cell(page, 'Host renderers')).not.toHaveText('Loading…');
|
||||
await expect(page.locator('#log-stats')).not.toHaveText('Loading…');
|
||||
return layoutOf(page);
|
||||
}
|
||||
|
||||
async function openFiles(page) {
|
||||
const details = surfaceOf(page, 'Logs').locator('details');
|
||||
if (!(await details.evaluate(node => node.open))) await details.locator('summary').click();
|
||||
await expect(details.locator(':scope > p.small').first()).not.toHaveText('Loading…');
|
||||
return details;
|
||||
}
|
||||
|
||||
/** The levels of the viewer's lines, in order. */
|
||||
const shownLevels = async page => (await lines(page).textContent()).split('\n').filter(Boolean)
|
||||
.map(line => line.split(/ +/)[1]);
|
||||
|
||||
async function ended(request, id) {
|
||||
await expect.poll(async () => (await serverJson(request, `/prompt_manager/jobs/${id}`)).job.state).not.toBe('running');
|
||||
return (await serverJson(request, `/prompt_manager/jobs/${id}`)).job;
|
||||
}
|
||||
|
||||
/** A second page of this origin that serves a fake ComfyUI window on the send bridge. */
|
||||
async function fakeComfyWindow(context, renderer) {
|
||||
const host = await context.newPage();
|
||||
await host.goto(`${workspace}#/overview`);
|
||||
await host.evaluate(async drawn => {
|
||||
const { createBridgeHost } = await import('/prompt_manager/js/shared/toolkit/prompt-bridge.js');
|
||||
window.fakeHost = createBridgeHost({ namespace: '/', origin: location.origin, label: 'PromptManager — ComfyUI',
|
||||
adapter: { listTargets: () => [{ id: '7', label: 'Prompt Manager', type: 'PromptManager', roles: ['positive'],
|
||||
renderer: drawn }], preview() { throw new Error('unused'); }, apply() { throw new Error('unused'); } } });
|
||||
}, renderer);
|
||||
return host;
|
||||
}
|
||||
|
||||
async function readBlob(download) {
|
||||
const chunks = await (await download.createReadStream()).toArray();
|
||||
return Buffer.concat(chunks).toString('utf8');
|
||||
}
|
||||
|
||||
// ── Task 4: the Diagnostics surface ─────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test('meta-grid shows real database and source status, the pinned Viewer.js version and the host renderers',
|
||||
async ({ page, request, context }) => {
|
||||
const mutations = [];
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) mutations.push(item.url()); });
|
||||
await openDiagnostics(page);
|
||||
const report = await serverJson(request, REPORT_URL);
|
||||
const surface = surfaceOf(page, 'Diagnostics');
|
||||
await expect(surface.locator(':scope > .panel-heading > h2')).toHaveText('Diagnostics');
|
||||
await expect(surface.locator(':scope > .panel-heading > button')).toHaveText('Run checks');
|
||||
await expect(surface.locator('.meta-grid dt')).toHaveText(['Database', 'Sources', 'Viewer.js', 'Host renderers']);
|
||||
const images = report.checks.database.image_count;
|
||||
await expect(cell(page, 'Database')).toHaveText(new RegExp(`^Readable · [\\d.]+ (bytes|KB|MB|GB) · ${images.toLocaleString('en-US')} images?$`));
|
||||
// The size is the database's real size (`database_size_bytes`), never a missing key read as 0.
|
||||
expect(report.database_info.database_size_bytes).toBeGreaterThan(0);
|
||||
await expect(cell(page, 'Database')).not.toContainText('· 0 bytes ·');
|
||||
const { configured, watching } = report.sources;
|
||||
await expect(cell(page, 'Sources')).toHaveText(configured
|
||||
? `${configured} source${configured === 1 ? '' : 's'} · ${watching} watching`
|
||||
: `ComfyUI output folder · ${watching ? 'watching' : 'manual scan'}`);
|
||||
await expect(cell(page, 'Viewer.js')).toHaveText('1.15.2 · pinned vendor');
|
||||
await expect(cell(page, 'Host renderers')).toHaveText('No ComfyUI window is open in this browser, so its renderer is not known.');
|
||||
// A ComfyUI window with a Classic node: the line Integrations shows (D106).
|
||||
const host = await fakeComfyWindow(context, 'classic');
|
||||
await openDiagnostics(page);
|
||||
await expect(cell(page, 'Host renderers')).toHaveText('Detected: Classic (1 ComfyUI window).');
|
||||
await host.close();
|
||||
// A refused read names the running job (D57's naming).
|
||||
await page.route(`**${REPORT_URL}`, route => route.fulfill({ status: 409, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, code: 'busy', error: 'busy', running: { id: 'a'.repeat(32), title: 'Restore' } }) }));
|
||||
await openDiagnostics(page);
|
||||
await expect(cell(page, 'Database')).toHaveText('Busy: Restore is running.');
|
||||
await page.unroute(`**${REPORT_URL}`);
|
||||
await expect(await request.post('/__test__/system/fail', { data: { path: REPORT_URL, times: 1 } })).toBeOK();
|
||||
await openDiagnostics(page);
|
||||
await expect(cell(page, 'Sources')).toHaveText('Could not be read.');
|
||||
expect(mutations).toEqual([]);
|
||||
});
|
||||
|
||||
test('Run checks opens the shared Health report', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
await openDiagnostics(page);
|
||||
const run = surfaceOf(page, 'Diagnostics').getByRole('button', { name: 'Run checks', exact: true });
|
||||
await run.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Health report');
|
||||
await expect(run).toHaveText('Run checks');
|
||||
await dialog(page).getByRole('button', { name: 'Close', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(run).toBeFocused();
|
||||
// The same report Overview reads: Overview's metric names it.
|
||||
expect((await serverJson(request, '/prompt_manager/system/health')).report).toBeTruthy();
|
||||
expect((await serverJson(request, '/__test__/system/digest')).rows).toEqual(before.rows);
|
||||
});
|
||||
|
||||
test('report download goes through the api client and holds no secret or absolute path', async ({ page, request }) => {
|
||||
const { revision } = await serverJson(request, '/prompt_manager/settings');
|
||||
await expect(await request.post('/prompt_manager/lora/enable', { data: { enabled: false, trigger_words_enabled: false, path: '',
|
||||
civitai_api_key: KEY, expected_revision: revision } })).toBeOK();
|
||||
await openDiagnostics(page);
|
||||
const reads = [];
|
||||
page.on('request', item => { if (item.url().endsWith(REPORT_URL)) reads.push(item.url()); });
|
||||
const download = page.waitForEvent('download');
|
||||
await surfaceOf(page, 'Diagnostics').getByRole('button', { name: 'Download diagnostic report', exact: true }).click();
|
||||
const file = await download;
|
||||
expect(file.suggestedFilename()).toMatch(/^prompt-manager-diagnostics-\d{8}-\d{4}\.json$/);
|
||||
await expect(toast(page)).toHaveText('Diagnostic report downloaded.');
|
||||
expect(reads).toHaveLength(1);
|
||||
const text = await readBlob(file);
|
||||
const report = JSON.parse(text);
|
||||
expect(report.host_renderers).toBe('No ComfyUI window is open in this browser, so its renderer is not known.');
|
||||
expect(report.config.integrations.has_civitai_api_key).toBe(true);
|
||||
expect(report.redacted).toBe(0);
|
||||
expect(report.viewerjs).toBe('1.15.2');
|
||||
expect(Object.keys(report.checks)).toEqual(['database', 'dependencies', 'comfyui_output', 'image_monitor']);
|
||||
expect(text).not.toContain(KEY);
|
||||
expect(text).not.toContain('pm-browser-');
|
||||
const strings = [];
|
||||
const walk = value => {
|
||||
if (typeof value === 'string') strings.push(value);
|
||||
else if (value && typeof value === 'object') Object.values(value).forEach(walk);
|
||||
};
|
||||
walk(report);
|
||||
expect(strings.filter(value => /^(\/|[A-Za-z]:[\\/]|\\\\)/.test(value))).toEqual([]);
|
||||
// A refused report names the job and saves nothing.
|
||||
await page.route(`**${REPORT_URL}`, route => route.fulfill({ status: 409, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, code: 'busy', error: 'busy', running: { id: 'a'.repeat(32), title: 'Restore' } }) }));
|
||||
let saved = false;
|
||||
page.on('download', () => { saved = true; });
|
||||
await surfaceOf(page, 'Diagnostics').getByRole('button', { name: 'Download diagnostic report', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Busy: Restore is running. Try again when it ends.');
|
||||
await page.unroute(`**${REPORT_URL}`);
|
||||
await page.route(`**${REPORT_URL}`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'The diagnostic report could not be built.' }) }));
|
||||
await surfaceOf(page, 'Diagnostics').getByRole('button', { name: 'Download diagnostic report', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The diagnostic report could not be downloaded. Nothing was saved to this device.');
|
||||
expect(saved).toBe(false);
|
||||
});
|
||||
|
||||
test('Inspect opens the Inspect a file page and writes nothing', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
const mutations = [];
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) mutations.push(item.url()); });
|
||||
await openDiagnostics(page);
|
||||
const row = operation(page, 'Image-link diagnostics');
|
||||
await expect(row.locator('p')).toHaveText('Check a file-to-prompt association without changing it.');
|
||||
await row.getByRole('button', { name: 'Inspect', exact: true }).click();
|
||||
await expect(page).toHaveURL(`${workspace}#/inspect`);
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Inspect a file');
|
||||
expect(mutations).toEqual([]);
|
||||
expect(await serverJson(request, '/__test__/system/digest')).toEqual(before);
|
||||
});
|
||||
|
||||
test('Create test link confirms first and names the prompt', async ({ page, request }) => {
|
||||
const [prompt] = (await serverJson(request, '/prompt_manager/recent?limit=1')).results;
|
||||
const posts = [];
|
||||
page.on('request', item => { if (item.url().endsWith(TEST_LINK_URL)) posts.push(item.postDataJSON()); });
|
||||
await openDiagnostics(page);
|
||||
const row = operation(page, 'Test image link');
|
||||
await expect(row.locator('p')).toHaveText('Write a placeholder image record to check that linking works.');
|
||||
const create = row.getByRole('button', { name: 'Create test link…', exact: true });
|
||||
await expect(create).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await create.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Create test link');
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText(new RegExp('^Writes a placeholder image record '
|
||||
+ `\\(prompt_manager_test_link\\.png\\) linked to prompt #${prompt.id} “.+”\\. Maintenance › Missing-file links removes it later\\.$`));
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(posts).toEqual([]);
|
||||
await create.click();
|
||||
await dialog(page).getByRole('button', { name: 'Create', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText(/^Test link created \(image record \d+\)\.$/);
|
||||
expect(posts).toEqual([{ prompt_id: prompt.id }]);
|
||||
// A failed write says nothing was changed.
|
||||
await page.route(`**${TEST_LINK_URL}`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'Test link failed' }) }));
|
||||
await create.click();
|
||||
await dialog(page).getByRole('button', { name: 'Create', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The test link could not be created. Nothing was changed.');
|
||||
// With no prompt to link, the button is unavailable and says why.
|
||||
await page.route('**/prompt_manager/recent?limit=1', route => route.fulfill({ status: 200, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: true, results: [] }) }));
|
||||
await openDiagnostics(page);
|
||||
await expect(operation(page, 'Test image link').locator('p'))
|
||||
.toHaveText('Write a placeholder image record to check that linking works. Add a prompt first.');
|
||||
await expect(operation(page, 'Test image link').getByRole('button')).toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
|
||||
// ── Task 5: the log viewer, statistics and runtime settings ─────────────────────────────────────────────────────────
|
||||
|
||||
test('level filter is minimum-level over all six choices', async ({ page, request }) => {
|
||||
await seedLogs(request);
|
||||
const urls = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === LOGS_URL) urls.push(new URL(item.url()).search); });
|
||||
await openDiagnostics(page);
|
||||
const level = page.locator('#log-level');
|
||||
await expect(level.locator('option')).toHaveText(['All levels', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']);
|
||||
await expect(page.locator('label[for="log-level"]')).toHaveText('Level');
|
||||
// All levels: the concept's four lines first, newest first, in its format.
|
||||
await expect(lines(page)).toContainText('09:42:18 INFO Scan completed: 12 enriched, 0 duplicate links\n'
|
||||
+ '09:42:18 WARNING 3 images have ambiguous prompt metadata\n09:39:04 INFO Database backup verified\n'
|
||||
+ '09:38:52 INFO Source monitor started: output');
|
||||
for (const line of (await lines(page).textContent()).split('\n')) expect(line).toMatch(/^\d\d:\d\d:\d\d [A-Z]+ +\S/);
|
||||
expect(urls.at(-1)).toBe('?limit=100');
|
||||
for (const minimum of LEVELS) {
|
||||
await level.selectOption(minimum);
|
||||
await expect.poll(() => urls.at(-1)).toBe(`?limit=100&level=${minimum}`);
|
||||
await expect(lines(page)).toContainText('CRITICAL The database could not be opened');
|
||||
const shown = await shownLevels(page);
|
||||
expect(shown.every(name => LEVELS.indexOf(name) >= LEVELS.indexOf(minimum)), `${minimum}: ${shown}`).toBe(true);
|
||||
if (minimum === 'DEBUG') await expect(lines(page)).toContainText('DEBUG Thumbnail cache lookup');
|
||||
if (minimum === 'WARNING') {
|
||||
await expect(lines(page)).not.toContainText('Database backup verified');
|
||||
await expect(lines(page)).toContainText('ERROR A source folder could not be read');
|
||||
}
|
||||
}
|
||||
await level.selectOption('');
|
||||
await expect.poll(() => urls.at(-1)).toBe('?limit=100');
|
||||
});
|
||||
|
||||
test('Lines accepts 1 to 500 and sends nothing when invalid', async ({ page, request }) => {
|
||||
await seedLogs(request, { filler: 30 });
|
||||
const urls = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === LOGS_URL) urls.push(new URL(item.url()).search); });
|
||||
await openDiagnostics(page);
|
||||
const limit = page.locator('#log-limit');
|
||||
await expect(page.locator('label[for="log-limit"]')).toHaveText('Lines');
|
||||
await expect(limit).toHaveValue('100');
|
||||
expect(await limit.evaluate(node => [node.type, node.min, node.max, node.title]))
|
||||
.toEqual(['number', '1', '500', 'Lines to show: a whole number from 1 to 500.']);
|
||||
await limit.fill('20');
|
||||
await limit.dispatchEvent('change');
|
||||
await expect.poll(() => urls.at(-1)).toBe('?limit=20');
|
||||
await expect.poll(async () => (await lines(page).textContent()).split('\n').length).toBe(20);
|
||||
const sent = urls.length;
|
||||
for (const value of ['0', '501', '2.5', '']) {
|
||||
await limit.fill(value);
|
||||
await limit.dispatchEvent('change');
|
||||
await expect(limit).toHaveAttribute('aria-invalid', 'true');
|
||||
}
|
||||
// A Level change while Lines is invalid sends nothing either.
|
||||
await page.locator('#log-level').selectOption('WARNING');
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
expect(urls.length).toBe(sent);
|
||||
await limit.fill('500');
|
||||
await limit.dispatchEvent('change');
|
||||
await expect(limit).not.toHaveAttribute('aria-invalid');
|
||||
await expect.poll(() => urls.at(-1)).toBe('?limit=500&level=WARNING');
|
||||
});
|
||||
|
||||
test('logs refresh handles empty, error and stale answers', async ({ page, request }) => {
|
||||
await seedLogs(request);
|
||||
await openDiagnostics(page);
|
||||
const level = page.locator('#log-level');
|
||||
const refresh = surfaceOf(page, 'Logs').getByRole('button', { name: 'Refresh', exact: true });
|
||||
// Empty: nothing at this level once the buffer is cleared.
|
||||
await expect(await request.post('/__test__/logs/reset')).toBeOK();
|
||||
await level.selectOption('CRITICAL');
|
||||
await expect(lines(page)).toHaveText('No log entries at this level.');
|
||||
// Error.
|
||||
await page.route(`**${LOGS_URL}?*`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'boom', logs: [] }) }));
|
||||
await refresh.click();
|
||||
await expect(lines(page)).toHaveText('The log could not be read.');
|
||||
await page.unroute(`**${LOGS_URL}?*`);
|
||||
// Stale: an older answer that arrives after a newer request never paints.
|
||||
await seedLogs(request);
|
||||
await expect(await request.post('/__test__/hold', { data: { path: LOGS_URL } })).toBeOK();
|
||||
await level.selectOption('DEBUG');
|
||||
await expect.poll(async () => (await serverJson(request, '/__test__/hold')).held[LOGS_URL] ?? 0).toBe(1);
|
||||
await level.selectOption('ERROR');
|
||||
await expect.poll(async () => (await serverJson(request, '/__test__/hold')).held[LOGS_URL] ?? 0).toBe(2);
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await expect(lines(page)).toContainText('CRITICAL The database could not be opened');
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
expect((await shownLevels(page)).every(name => ['ERROR', 'CRITICAL'].includes(name))).toBe(true);
|
||||
});
|
||||
|
||||
test('log statistics show entries, errors, warnings, files and size', async ({ page, request }) => {
|
||||
await seedLogs(request, { rotated: [2048, 1024], other: true, link: true, active: 512 });
|
||||
await openDiagnostics(page);
|
||||
const stats = page.getByRole('status', { name: 'Log statistics', exact: true });
|
||||
await expect(stats).toHaveAttribute('id', 'log-stats');
|
||||
const text = await stats.textContent();
|
||||
const match = /^([\d,]+) entr(?:y|ies) in memory · (\d+) errors? · (\d+) warnings? · (\d+) files? · (.+) on disk$/.exec(text);
|
||||
expect(match, text).not.toBeNull();
|
||||
const [, entries, errors, warnings, files, size] = match;
|
||||
expect(Number(entries.replace(/,/g, ''))).toBeGreaterThanOrEqual(7);
|
||||
expect(Number(errors)).toBeGreaterThanOrEqual(2);
|
||||
expect(Number(warnings)).toBeGreaterThanOrEqual(1);
|
||||
// The active log, two rotated copies and the .bak; the link is not counted (Rule R).
|
||||
expect(Number(files)).toBe(4);
|
||||
expect(size).toMatch(/^[\d.]+ (bytes|KB|MB)$/);
|
||||
});
|
||||
|
||||
test('Download log saves the active log', async ({ page, request }) => {
|
||||
await seedLogs(request, { active: 300 });
|
||||
await openDiagnostics(page);
|
||||
const download = page.waitForEvent('download');
|
||||
await surfaceOf(page, 'Logs').getByRole('button', { name: 'Download log', exact: true }).click();
|
||||
const file = await download;
|
||||
expect(file.suggestedFilename()).toBe('prompt_manager.log');
|
||||
expect((await readBlob(file)).length).toBeGreaterThanOrEqual(300);
|
||||
await page.route('**/prompt_manager/logs/download/**', route => route.fulfill({ status: 404, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'Log file not found' }) }));
|
||||
await surfaceOf(page, 'Logs').getByRole('button', { name: 'Download log', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The log could not be downloaded.');
|
||||
});
|
||||
|
||||
test('runtime level (including CRITICAL) and console logging save and say they apply until the server restarts',
|
||||
async ({ page, request }) => {
|
||||
const posts = [];
|
||||
page.on('request', item => { if (item.method() === 'POST' && item.url().endsWith(CONFIG_URL)) posts.push(item.postDataJSON()); });
|
||||
await openDiagnostics(page);
|
||||
const level = settingOf(page, 'Runtime log level');
|
||||
await expect(level.locator('p')).toHaveText('How much the server writes to the log. Applied until the server restarts.');
|
||||
await expect(level.locator('option')).toHaveText(['INFO', 'WARNING', 'DEBUG', 'ERROR', 'CRITICAL']);
|
||||
await expect(level.locator('select')).toHaveValue('INFO');
|
||||
const consoleRow = settingOf(page, 'Console logging');
|
||||
await expect(consoleRow.locator('p')).toHaveText('Print Prompt Manager’s warnings and errors in the ComfyUI console '
|
||||
+ '(everything at DEBUG). Applied until the server restarts.');
|
||||
const toggle = consoleRow.locator('input[type="checkbox"]');
|
||||
await expect(toggle).toBeChecked();
|
||||
await level.locator('select').selectOption('CRITICAL');
|
||||
await toggle.uncheck();
|
||||
await expect(status(page)).toHaveText('Unsaved changes.');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Applied until the server restarts.');
|
||||
expect(posts).toEqual([{ level: 'CRITICAL', console_logging: false }]);
|
||||
const { config } = await serverJson(request, CONFIG_URL);
|
||||
expect([config.level, config.console_logging]).toEqual(['CRITICAL', false]);
|
||||
// Read again on a fresh page: the server's values.
|
||||
await openDiagnostics(page);
|
||||
await expect(settingOf(page, 'Runtime log level').locator('select')).toHaveValue('CRITICAL');
|
||||
await expect(settingOf(page, 'Console logging').locator('input[type="checkbox"]')).not.toBeChecked();
|
||||
// Only the changed key is sent.
|
||||
await settingOf(page, 'Runtime log level').locator('select').selectOption('DEBUG');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Applied until the server restarts.');
|
||||
expect(posts.at(-1)).toEqual({ level: 'DEBUG' });
|
||||
});
|
||||
|
||||
test('a refused runtime value is marked invalid', async ({ page }) => {
|
||||
await page.route(`**${CONFIG_URL}`, route => (route.request().method() === 'POST'
|
||||
? route.fulfill({ status: 400, contentType: 'application/json', body: JSON.stringify({ success: false, error: 'Invalid log level' }) })
|
||||
: route.continue()));
|
||||
await openDiagnostics(page);
|
||||
const select = settingOf(page, 'Runtime log level').locator('select');
|
||||
await select.selectOption('WARNING');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Some changes were not saved. Review the highlighted settings.');
|
||||
await expect(select).toHaveAttribute('aria-invalid', 'true');
|
||||
await expect(settingOf(page, 'Console logging').locator('input[type="checkbox"]')).not.toHaveAttribute('aria-invalid');
|
||||
// The draft stays: an edit clears the mark.
|
||||
await expect(select).toHaveValue('WARNING');
|
||||
await select.selectOption('ERROR');
|
||||
await expect(select).not.toHaveAttribute('aria-invalid');
|
||||
});
|
||||
|
||||
test('runtime values that cannot be read stay disabled and say so', async ({ page }) => {
|
||||
await page.route(`**${CONFIG_URL}`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'boom' }) }));
|
||||
await openDiagnostics(page);
|
||||
for (const label of ['Runtime log level', 'Console logging']) {
|
||||
const row = settingOf(page, label);
|
||||
await expect(row.locator('p.small')).toHaveText('The current value could not be read.');
|
||||
await expect(row.locator('select, input[type="checkbox"]')).toBeDisabled();
|
||||
}
|
||||
});
|
||||
|
||||
// ── Task 6: log files, the reviewed cleanup and Clear all logs ─────────────────────────────────────────────────────
|
||||
|
||||
const LOG_FOLDER = { rotated: [3000, 2048, 1024], other: true, link: true, active: 100 };
|
||||
const LINKS = process.platform === 'win32' ? 0 : 1;
|
||||
|
||||
test('Review log cleanup lists only rotated copies and keeps the rest', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
const before = await names(request);
|
||||
const previews = [];
|
||||
page.on('request', item => { if (item.url().endsWith(PREVIEW_URL)) previews.push(item.postDataJSON()); });
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/^prompt_manager\.log · [\d.]+ (bytes|KB) · rotated copies: 3$/);
|
||||
await details.getByRole('button', { name: 'Review log cleanup', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review maintenance plan');
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Rotated log copies');
|
||||
expect(previews).toEqual([{ operation: 'rotated_logs', options: {} }]);
|
||||
const grid = await dialog(page).locator('dl.meta-grid > div').evaluateAll(items => items.map(item =>
|
||||
[item.querySelector('dt').textContent, item.querySelector('dd').textContent]));
|
||||
expect(grid.map(([term]) => term)).toEqual(['Scope', 'Protected', 'Backup', 'Media on disk']);
|
||||
expect(grid[0][1]).toMatch(/^3 rotated log copies · 5\.9 KB$/);
|
||||
expect(grid[1][1]).toBe(['Kept: the active log', ...(LINKS ? ['Kept: 1 link'] : []), 'Kept: 1 other file'].join(' · '));
|
||||
expect(grid[2][1]).toBe('Not needed: no library record changes');
|
||||
expect(grid[3][1]).toBe('Deletes only the listed log files. No image, video or library record changes.');
|
||||
await expect(dialog(page).locator('details > summary')).toHaveText('Listed files (3)');
|
||||
await expect(dialog(page).locator('details pre')).toHaveText(/^prompt_manager\.log\.1 · 2\.9 KB · .+\nprompt_manager\.log\.2 · 2\.0 KB · .+\nprompt_manager\.log\.3 · 1\.0 KB · .+$/);
|
||||
// No D55 backup box: nothing in the library changes.
|
||||
await expect(dialog(page).locator('.notice')).toHaveCount(0);
|
||||
await expect(dialog(page).getByRole('button', { name: 'Back up now' })).toHaveCount(0);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(await names(request)).toEqual(before);
|
||||
});
|
||||
|
||||
test('Apply deletes exactly the listed files and Activity lists them', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await details.getByRole('button', { name: 'Review log cleanup', exact: true }).click();
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
const job = await ended(request, page.url().split('/').pop());
|
||||
expect([job.kind, job.state, job.title, job.counters.changed]).toEqual(['maintenance', 'finished', 'Rotated log copies', 3]);
|
||||
await expect(toast(page)).toHaveText(/Rotated log copies (started|finished · 3 changed)/);
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(main.locator('h1')).toHaveText('Rotated log copies');
|
||||
for (const name of ['prompt_manager.log.1', 'prompt_manager.log.2', 'prompt_manager.log.3']) await expect(main).toContainText(name);
|
||||
// Only those three are gone: the active log, the user's .bak and the link are kept.
|
||||
expect(await names(request)).toEqual(['prompt_manager.log', ...(LINKS ? ['prompt_manager.log.4'] : []), 'prompt_manager.log.bak']);
|
||||
});
|
||||
|
||||
test('a rollover after the review deletes nothing and Activity offers Review again', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await details.getByRole('button', { name: 'Review log cleanup', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Rotated log copies');
|
||||
await expect(await request.post('/__test__/logs/rotate')).toBeOK();
|
||||
const rolled = await serverJson(request, '/__test__/logs/state');
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
const job = await ended(request, page.url().split('/').pop());
|
||||
expect([job.state, job.scope.failure]).toEqual(['failed', 'plan-changed']);
|
||||
await expect(page.locator('#workspace-main')).toContainText(CHANGED);
|
||||
expect((await serverJson(request, '/__test__/logs/state')).entries.map(entry => entry.name))
|
||||
.toEqual(rolled.entries.map(entry => entry.name));
|
||||
const previews = [];
|
||||
page.on('request', item => { if (item.url().endsWith(PREVIEW_URL)) previews.push(item.postDataJSON()); });
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Review again', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Rotated log copies');
|
||||
expect(previews).toEqual([{ operation: 'rotated_logs', options: {} }]);
|
||||
});
|
||||
|
||||
test('Clear all logs states both counts, confirms first and reads everything again', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
const truncates = [];
|
||||
page.on('request', item => { if (item.url().endsWith('/prompt_manager/logs/truncate')) truncates.push(item.postDataJSON()); });
|
||||
await openDiagnostics(page);
|
||||
await expect(lines(page)).toContainText('Database backup verified');
|
||||
const details = await openFiles(page);
|
||||
const clearRow = operation(page, 'Clear all logs');
|
||||
await expect(clearRow.locator('p')).toHaveText('Empty the active log and delete its rotated copies.');
|
||||
const clear = clearRow.getByRole('button', { name: 'Clear all logs…', exact: true });
|
||||
await expect(clear).toHaveClass(/\bdanger\b/);
|
||||
await clear.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Clear all logs');
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText('Empties prompt_manager.log, deletes 3 rotated copies (5.9 KB) '
|
||||
+ 'and clears the lines shown here. The other files in the log folder are kept. This cannot be undone.');
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(truncates).toEqual([]);
|
||||
await clear.click();
|
||||
await dialog(page).getByRole('button', { name: 'Clear', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Logs cleared · 3 rotated copies deleted.');
|
||||
expect(truncates).toHaveLength(1);
|
||||
// #389: Clear sends back exactly the files the dialog was built from, each with its identity string.
|
||||
expect(truncates[0].expected.map(row => row.name).sort()).toEqual(['prompt_manager.log', 'prompt_manager.log.1',
|
||||
'prompt_manager.log.2', 'prompt_manager.log.3']);
|
||||
for (const row of truncates[0].expected) expect(row.identity).toMatch(/^\d+:\d+:\d+$/);
|
||||
// Read again: the viewer, the file line and the list.
|
||||
await expect(lines(page)).not.toContainText('Database backup verified');
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/rotated copies: 0$/);
|
||||
const kept = ['prompt_manager.log', 'prompt_manager.log.bak', ...(LINKS ? ['prompt_manager.log.4'] : [])];
|
||||
// The server lists newest first: compare as a set.
|
||||
await expect.poll(async () => (await details.locator('.operation-list h3').allTextContents()).sort()).toEqual([...kept].sort());
|
||||
expect(await names(request)).toEqual(['prompt_manager.log', ...(LINKS ? ['prompt_manager.log.4'] : []), 'prompt_manager.log.bak']);
|
||||
});
|
||||
|
||||
test('Clear all logs reads the files again before it states what it deletes', async ({ page, request }) => {
|
||||
await seedLogs(request, { rotated: [1024], active: 100 });
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/rotated copies: 1$/);
|
||||
// The log rolls over while the page stays open: Clear names what it deletes now, not the count shown.
|
||||
await seedLogs(request, { records: false, rotated: [1024, 1024, 1024] });
|
||||
await operation(page, 'Clear all logs').getByRole('button', { name: 'Clear all logs…', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText(/^Empties prompt_manager\.log, deletes 3 rotated copies \(3\.0 KB\) /);
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/rotated copies: 3$/);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
// A list that cannot be read gives the general sentence, never an old count.
|
||||
await page.route('**/prompt_manager/logs/files', route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'boom', files: [] }) }));
|
||||
await operation(page, 'Clear all logs').getByRole('button', { name: 'Clear all logs…', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText(/^Empties prompt_manager\.log, deletes its rotated copies and clears/);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
expect(await names(request)).toEqual(['prompt_manager.log', 'prompt_manager.log.1', 'prompt_manager.log.2', 'prompt_manager.log.3']);
|
||||
});
|
||||
|
||||
test('a rollover while Clear all logs is open deletes nothing and says why (#389)', async ({ page, request }) => {
|
||||
await seedLogs(request, { rotated: [1024, 1024], active: 100 });
|
||||
await openDiagnostics(page);
|
||||
await openFiles(page);
|
||||
await operation(page, 'Clear all logs').getByRole('button', { name: 'Clear all logs…', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText(/^Empties prompt_manager\.log, deletes 2 rotated copies /);
|
||||
// The log rolls over after the confirmation was built: the copies it listed are not the ones there now.
|
||||
await expect(await request.post('/__test__/logs/rotate')).toBeOK();
|
||||
const rolled = await names(request);
|
||||
const answer = page.waitForResponse(response => response.url().endsWith('/prompt_manager/logs/truncate'));
|
||||
await dialog(page).getByRole('button', { name: 'Clear', exact: true }).click();
|
||||
expect((await answer).status()).toBe(409);
|
||||
await expect(toast(page)).toHaveText('The log files changed since you confirmed; nothing was cleared. '
|
||||
+ 'Read the list again and clear again.');
|
||||
expect(await names(request)).toEqual(rolled);
|
||||
});
|
||||
|
||||
test('each listed file downloads by its returned name and a link has no Download', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await expect(details.locator(':scope > p.small').nth(1))
|
||||
.toHaveText('Rotates at 10 MB · keeps 5 rotated copies · file logging on · memory buffer 1,000 entries');
|
||||
const list = details.locator('.operation-list');
|
||||
expect((await list.locator('h3').allTextContents()).sort()).toEqual(['prompt_manager.log', 'prompt_manager.log.1',
|
||||
'prompt_manager.log.2', 'prompt_manager.log.3', ...(LINKS ? ['prompt_manager.log.4'] : []), 'prompt_manager.log.bak'].sort());
|
||||
const item = name => list.locator('.operation').filter({ has: page.locator(`h3:text-is("${name}")`) });
|
||||
await expect(item('prompt_manager.log.1').locator('p')).toHaveText(/^2\.9 KB · \d{4}-\d\d-\d\d \d\d:\d\d · Rotated copy$/);
|
||||
await expect(item('prompt_manager.log.bak').locator('p')).toHaveText(/^0 bytes · .+ · Other file$/);
|
||||
if (LINKS) {
|
||||
await expect(item('prompt_manager.log.4').locator('p')).toHaveText(/ · Link, not downloaded$/);
|
||||
await expect(item('prompt_manager.log.4').getByRole('button')).toHaveCount(0);
|
||||
}
|
||||
for (const name of ['prompt_manager.log.1', 'prompt_manager.log.bak']) {
|
||||
const download = page.waitForEvent('download');
|
||||
await item(name).getByRole('button', { name: `Download ${name}`, exact: true }).click();
|
||||
const file = await download;
|
||||
expect(file.suggestedFilename()).toBe(name);
|
||||
if (name === 'prompt_manager.log.1') expect((await readBlob(file)).length).toBe(3000);
|
||||
}
|
||||
});
|
||||
@@ -429,7 +429,9 @@ test('Rebuild all is explicit and the plan line says every preview is regenerate
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
expect(seen.starts).toEqual([{ mode: 'rebuild' }]);
|
||||
const id = page.url().split('/').pop();
|
||||
await expect.poll(async () => (await serverJson(request, `/prompt_manager/jobs/${id}`)).job.state).toBe('finished');
|
||||
// Rebuild regenerates all 24 previews with real image work: on a busy CI
|
||||
// runner that took over 5 s (12 of 24 done at the default poll budget).
|
||||
await expect.poll(async () => (await serverJson(request, `/prompt_manager/jobs/${id}`)).job.state, { timeout: 30000 }).toBe('finished');
|
||||
});
|
||||
|
||||
test('a plan that cannot be read says so; Start still starts', async ({ page }) => {
|
||||
|
||||
@@ -188,6 +188,35 @@ test('a refused value shows the server\'s message', async ({ page, request }) =>
|
||||
await expect(actionOf(page, 'Preview tagging review')).not.toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
|
||||
test('a value off the 0.05 step is refused and never reaches the Tag image dialog (#369)', async ({ page, request }) => {
|
||||
const before = (await serverJson(request, SETTINGS_URL)).settings;
|
||||
const STEP = 'General-tag confidence must be a multiple of 0.05.';
|
||||
await selectThenOpen(page, FIRST);
|
||||
await page.locator('#threshold-general').fill('0.37');
|
||||
await page.locator('#threshold-general').press('Tab');
|
||||
// Preview tagging review explains instead of opening a dialog whose Generate the browser would block.
|
||||
await expect(actionOf(page, 'Preview tagging review')).toHaveAttribute('aria-disabled', 'true');
|
||||
await actionOf(page, 'Preview tagging review').focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(toast(page)).toHaveText(STEP);
|
||||
await expect(tagDialog(page)).toHaveCount(0);
|
||||
// The server refuses the save with the same sentence and writes nothing.
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText(STEP);
|
||||
expect((await serverJson(request, SETTINGS_URL)).settings.autotag_general_threshold).toBe(before.autotag_general_threshold);
|
||||
// A value on the step opens the dialog, and its Generate passes the browser's own check and sends.
|
||||
await page.locator('#threshold-general').fill('0.4');
|
||||
await page.locator('#threshold-general').press('Tab');
|
||||
await expect(actionOf(page, 'Preview tagging review')).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await actionOf(page, 'Preview tagging review').click();
|
||||
await expect(tagDialog(page).locator('#image-general-threshold')).toHaveValue('0.4');
|
||||
expect(await tagDialog(page).locator('#image-general-threshold').evaluate(input => input.checkValidity())).toBe(true);
|
||||
const suggested = page.waitForResponse(item => item.url().endsWith('/image-annotations/suggest'));
|
||||
await tagDialog(page).getByRole('button', { name: 'Generate suggestions', exact: true }).click();
|
||||
const answer = await suggested;
|
||||
expect(answer.request().postDataJSON().options.general_threshold).toBe(0.4);
|
||||
});
|
||||
|
||||
test('Download starts the job and Activity shows it without Cancel', async ({ page, request }) => {
|
||||
await unavailable(request, ['wd14-vit']);
|
||||
await fakeDownload(request, 'hold');
|
||||
@@ -286,22 +315,28 @@ test('Preview tagging review without a selection toasts the D115 sentence', asyn
|
||||
await expect(tagDialog(page)).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('Tag output images reviews the scope and starts a batch job', async ({ page, request }) => {
|
||||
test('Tag output images reviews the images, tags each one and never a prompt (#379)', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/__test__/tagging/snapshot');
|
||||
await openTagging(page);
|
||||
// The fixture's one prompt is already tagged: under Merge it is tagged again with what its images give.
|
||||
await page.locator('#tag-existing').selectOption('merge');
|
||||
// The saved policy is Skip: no image is auto-tagged yet, so every listed still image is to tag.
|
||||
const previewed = page.waitForResponse(item => new URL(item.url()).pathname === PREVIEW_URL);
|
||||
await reviewButton(page).click();
|
||||
const answer = await previewed;
|
||||
expect(answer.request().postDataJSON()).toEqual({ model_type: 'wd14-swinv2', existing: 'merge' });
|
||||
expect(answer.request().postDataJSON()).toEqual({ model_type: 'wd14-swinv2', existing: 'skip' });
|
||||
const plan = await answer.json();
|
||||
// Four still images (two linked, a copy, an unlinked one); the video is not a target.
|
||||
expect([plan.images, plan.already_tagged, plan.to_tag]).toEqual([4, 0, 4]);
|
||||
expect(plan.prompts).toBeUndefined();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review tag output images');
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Tag output images');
|
||||
expect(await gridOf(page)).toEqual([['Model', 'WD14 SwinV2 · ~400 MB'], ['Existing tags', 'Merge with existing'],
|
||||
['Images linked', String(plan.images)], ['Prompts', String(plan.prompts)], ['Already tagged', String(plan.already_tagged)],
|
||||
['Prompts to tag', `up to ${plan.to_tag}`], ['Settings', 'General 0.35 · Character 0.85'],
|
||||
expect(await gridOf(page)).toEqual([['Model', 'WD14 SwinV2 · ~400 MB'], ['Existing tags', 'Skip already tagged'],
|
||||
['Images', '4'], ['Already auto-tagged', '0'], ['Images to tag', 'about 4'],
|
||||
['Time', 'about 1 min on a processor; a graphics card can be faster.'],
|
||||
['Settings', `General 0.35 · Character 0.85 · up to ${plan.suggestion_limit} tags per image`],
|
||||
['Backup', 'Create and verify before changes']]);
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Nothing is written until you start. The number to '
|
||||
+ 'tag is an estimate: images changed since they were tagged are tagged again, and missing files are skipped. '
|
||||
+ 'Activity reports what was tagged.');
|
||||
await expect(dialog(page).locator('#modal-body > .notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
// The review wrote nothing.
|
||||
expect(await serverJson(request, '/__test__/tagging/snapshot')).toEqual(before);
|
||||
@@ -309,15 +344,23 @@ test('Tag output images reviews the scope and starts a batch job', async ({ page
|
||||
const started = page.waitForResponse(item => new URL(item.url()).pathname === '/prompt_manager/jobs/autotag');
|
||||
await dialog(page).getByRole('button', { name: 'Start tagging', exact: true }).click();
|
||||
const start = await started;
|
||||
expect(start.request().postDataJSON()).toEqual({ model_type: 'wd14-swinv2', existing: 'merge',
|
||||
expect(start.request().postDataJSON()).toEqual({ model_type: 'wd14-swinv2', existing: 'skip',
|
||||
options: { general_threshold: 0.35, character_threshold: 0.85 } });
|
||||
const { job } = await start.json();
|
||||
await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`);
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Tag output images');
|
||||
const finished = await ended(request, job.id);
|
||||
expect(finished.state).toBe('finished');
|
||||
const after = await serverJson(request, '/__test__/tagging/snapshot');
|
||||
expect(after.prompt_tags.length).toBeGreaterThan(before.prompt_tags.length);
|
||||
expect(finished.status_line).toBe('Tagged 4 images');
|
||||
// Each image got its own tags; no prompt, pair or link row changed.
|
||||
expect(await serverJson(request, '/__test__/tagging/snapshot')).toEqual(before);
|
||||
|
||||
// Under Skip every image is now auto-tagged: nothing to start.
|
||||
await openTagging(page);
|
||||
await reviewButton(page).click();
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Every output image is already auto-tagged.');
|
||||
expect((await gridOf(page)).slice(2, 5)).toEqual([['Images', '4'], ['Already auto-tagged', '4'], ['Images to tag', 'about 0']]);
|
||||
await expect(dialog(page).locator('.dialog-actions button')).toHaveText(['Cancel']);
|
||||
});
|
||||
|
||||
test('Replace after review shows only Cancel with the reason', async ({ page, request }) => {
|
||||
@@ -326,18 +369,13 @@ test('Replace after review shows only Cancel with the reason', async ({ page, re
|
||||
await page.locator('#tag-existing').selectOption('replace_after_review');
|
||||
await reviewButton(page).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review tag output images');
|
||||
expect((await gridOf(page)).map(([term]) => term)).toEqual(['Model', 'Existing tags', 'Images linked', 'Prompts', 'Already tagged',
|
||||
'Prompts to tag', 'Settings', 'Backup']);
|
||||
expect((await gridOf(page)).map(([term]) => term)).toEqual(['Model', 'Existing tags', 'Images', 'Already auto-tagged',
|
||||
'Images to tag', 'Time', 'Settings', 'Backup']);
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Replacing tags needs a review of each image. '
|
||||
+ 'Use Preview tagging review, or choose Skip or Merge for a batch run.');
|
||||
await expect(dialog(page).locator('.dialog-actions button')).toHaveText(['Cancel']);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
// Under Skip the fixture's only prompt is already tagged: nothing to start either.
|
||||
await page.locator('#tag-existing').selectOption('skip');
|
||||
await reviewButton(page).click();
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Every prompt with an output image is already tagged.');
|
||||
await expect(dialog(page).locator('.dialog-actions button')).toHaveText(['Cancel']);
|
||||
expect(await serverJson(request, '/__test__/tagging/snapshot')).toEqual(before);
|
||||
expect((await serverJson(request, '/prompt_manager/jobs')).jobs.filter(item => item.kind === 'autotag')).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
import { readCoverageTable, parseControl, CONTROL_EXEMPTIONS } from '../../tools/vendor/legacy-coverage.mjs';
|
||||
|
||||
// System › Overview (#249 System S2 Task 6) on the concept-shaped real fixture: the metrics, rows, Statistics dialog and
|
||||
// Health report read the server, never sample values (D93, D95, D96). The hooks are workspace_system_fixture.py's.
|
||||
@@ -412,15 +413,16 @@ test('out-of-order overview responses never repaint a newer section', async ({ p
|
||||
await nav.getByRole('button', { name: 'Appearance', exact: true }).click();
|
||||
await nav.getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect.poll(held).toBe(2);
|
||||
// Diagnostics still hands off (S12), so its body is one known panel.
|
||||
// Diagnostics & logs is native (S12): its body is its two surfaces.
|
||||
await nav.getByRole('button', { name: 'Diagnostics & logs', exact: true }).click();
|
||||
await expect(layout.locator(':scope > section.surface')).toHaveCount(2);
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await expect.poll(held).toBe(0);
|
||||
// Both late answers reached a section that had left: Diagnostics keeps its own body.
|
||||
await expect(layout.locator(':scope > section.handoff')).toBeVisible();
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
await expect(layout.locator('.summary-metrics')).toHaveCount(0);
|
||||
expect(await layout.evaluate(node => [...node.children].map(child => child.className))).toEqual(['surface handoff', 'save-row']);
|
||||
expect(await layout.evaluate(node => [...node.children].map(child => `${child.localName}.${child.className}`)))
|
||||
.toEqual(['section.surface', 'section.surface', 'div.save-row']);
|
||||
// Back on Overview, one fresh read paints one set of metrics.
|
||||
await nav.getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect(layout.locator('.summary-metrics')).toHaveCount(1);
|
||||
@@ -793,3 +795,45 @@ test('reset restores defaults without touching other keys', async ({ page }) =>
|
||||
expect(stored).toEqual({ gridColumns: 5, defaultLimit: 200, thumbnailPromptDismissed: 'kept', workspaceLayout: 'grid',
|
||||
videoMute: true, showImageInfo: true });
|
||||
});
|
||||
|
||||
// TESTING.md "Legacy System capability coverage" (#249 System S12, #329): each `covered` row's native control is on its
|
||||
// System section, found by role and accessible name (a `term` by its text, a disclosure by its summary). Collapsed
|
||||
// `details` are opened first, so a control inside one counts once it is shown. tests/vendor/legacy-coverage.test.mjs
|
||||
// checks the rest of each row.
|
||||
test('every covered legacy System capability has its native control', async ({ page, request }) => {
|
||||
// Download is offered for a model that is not on disk ("Download again" otherwise): the selected default's file is
|
||||
// missing here.
|
||||
await expect(await request.post('/__test__/tagging/models', { data: { unavailable: ['wd14-swinv2'] } })).toBeOK();
|
||||
try {
|
||||
await findControls(page);
|
||||
} finally {
|
||||
await request.post('/__test__/tagging/reset', { data: {} });
|
||||
}
|
||||
});
|
||||
|
||||
async function findControls(page) {
|
||||
const covered = readCoverageTable().filter(row => row.Status === 'covered')
|
||||
.map(row => ({ capability: row.Capability, ...parseControl(row['Native control']) }));
|
||||
// A covered row without a control is one of the recorded exemptions, never silently skipped.
|
||||
expect(covered.filter(row => !row.section).map(row => row.capability)).toEqual(Object.keys(CONTROL_EXEMPTIONS));
|
||||
const controls = covered.filter(row => row.section);
|
||||
expect(controls.length).toBeGreaterThan(40);
|
||||
const sections = [...new Set(controls.map(row => row.section))];
|
||||
for (const section of sections) {
|
||||
await page.goto(`${workspace}#/system/${section}`);
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(layoutOf(page).locator(':scope > *').first()).toBeVisible();
|
||||
for (const details of await main.locator('details:not([open]) > summary').all()) await details.click();
|
||||
for (const { capability, role, name, row } of controls.filter(item => item.section === section)) {
|
||||
// A name several rows share is looked up inside the row the table names.
|
||||
const within = row ? main.locator('.operation').filter({ has: page.locator(`h3:text-is("${row}")`) }) : main;
|
||||
const exact = new RegExp(`^${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`);
|
||||
// A `dt` (term) and a `details` disclosure (group) have no name from content: the term's text, the summary's.
|
||||
const control = role === 'term' ? within.getByRole('term').filter({ hasText: exact })
|
||||
: role === 'group' ? within.locator('details').filter({ has: page.locator(':scope > summary', { hasText: exact }) })
|
||||
.or(within.getByRole('group', { name, exact: true }))
|
||||
: within.getByRole(role, { name, exact: true });
|
||||
await expect(control.first(), `${capability}: ${section} · ${role} · "${name}"${row ? ` in "${row}"` : ''}`).toBeVisible();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -513,6 +513,41 @@ test('search finds an image by its image tag', async ({ page, request }) => {
|
||||
await expect(card(page, UNLINKED)).toBeVisible();
|
||||
});
|
||||
|
||||
// #379 CUJ 8: the Tags filter finds images by their own image tags through the tag index; no prompt carries them, and
|
||||
// selecting a match shows its prompt. A linked prompt's tag still combines with the image tags.
|
||||
test('the Tags filter finds images by image tags only, and selecting one shows its prompt (#379)', async ({ page, request }) => {
|
||||
await saveImageTags(request, FIRST, ['beach', '1girl', 'waves']);
|
||||
await saveImageTags(request, SECOND, ['beach', 'waves']);
|
||||
await saveImageTags(request, UNLINKED, ['beach', '1girl', 'waves', 'gulls']);
|
||||
await page.goto(images);
|
||||
await expect(page.locator('.gallery-body article[data-image-key]')).toHaveCount(manifest.files.length);
|
||||
const filters = page.locator('#workspace-main search.toolbar').getByRole('button', { name: /^Filters(?: · \d+)?$/ });
|
||||
const refine = page.getByRole('dialog', { name: 'Refine this view', exact: true });
|
||||
const applyTags = async value => {
|
||||
await filters.click();
|
||||
await refine.getByLabel('Tags', { exact: true }).fill(value);
|
||||
const listing = page.waitForRequest(item => item.url().includes('/images/output?')
|
||||
&& new URL(item.url()).searchParams.get('tags') !== null);
|
||||
await refine.getByRole('button', { name: 'Apply filters', exact: true }).click();
|
||||
await listing;
|
||||
};
|
||||
await applyTags('beach, 1girl, waves');
|
||||
// Every listed image whose image tags hold all three; SECOND lacks 1girl.
|
||||
await expect(page.locator('.gallery-body article[data-image-key]')).toHaveCount(2);
|
||||
await expect(card(page, FIRST)).toBeVisible();
|
||||
await expect(card(page, UNLINKED)).toBeVisible();
|
||||
await expect(card(page, SECOND)).toHaveCount(0);
|
||||
// The newest match is selected; the inspector shows its linked prompt.
|
||||
await infoButton(page, FIRST).click();
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
await expect(panel(page).locator('.panel-heading h2')).toHaveText(manifest.title);
|
||||
// The linked positive's own tag combines: SECOND has beach and waves itself and minimal through its prompt.
|
||||
await applyTags(`beach, waves, ${manifest.prompt_tags[0]}`);
|
||||
await expect(page.locator('.gallery-body article[data-image-key]')).toHaveCount(2);
|
||||
await expect(card(page, FIRST)).toBeVisible();
|
||||
await expect(card(page, SECOND)).toBeVisible();
|
||||
});
|
||||
|
||||
// Task C4 rulings: a save re-reads the selected file's details without a loading flash, and under a Tags filter or a
|
||||
// search the loaded listing is re-read in place (scroll and selection kept while the file still matches).
|
||||
const cards = page => page.locator('.gallery-body article[data-image-key]');
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {pageInventoryOf,inventoryOf,diffInventory,mapVocabularyConcept,mapVocabularyDialog} from './vocabulary-canonical-comparison.mjs';
|
||||
|
||||
@@ -40,7 +41,7 @@ async function expectLibrary(page,{tags='',category='',count}) {
|
||||
|
||||
let manifest;
|
||||
test.beforeEach(async({page,request,browserName})=>{
|
||||
await startCoverage(page,browserName);manifest=await seed(request);
|
||||
await startCoverage(page,browserName);await armUnseenRestores(page);manifest=await seed(request);
|
||||
page.vocabularyErrors=[];page.on('pageerror',error=>page.vocabularyErrors.push(error.message));
|
||||
});
|
||||
test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.vocabularyErrors).toEqual([]);});
|
||||
@@ -264,7 +265,8 @@ const toast=page=>page.locator('#workspace-toast');
|
||||
const actionNames=scope=>scope.locator('.dialog-actions button').evaluateAll(nodes=>nodes.map(node=>node.textContent.trim()));
|
||||
const snapshot=async request=>{const response=await request.get('/__test__/vocabulary/snapshot');await expect(response).toBeOK();return response.json();};
|
||||
const backup=async request=>{const response=await request.get('/__test__/vocabulary/backup');await expect(response).toBeOK();return response.body();};
|
||||
const restore=async(request,buffer)=>expect(await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'vocabulary.db',mimeType:'application/octet-stream',buffer}}})).toBeOK();
|
||||
// A restore the open page has not seen, so its dialog stays to report it (#413).
|
||||
const restore=async(page,request,buffer)=>expect(await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'vocabulary.db',mimeType:'application/octet-stream',buffer}}}))).toBeOK();
|
||||
const tagNames=rows=>rows.tags.map(([,name])=>name);
|
||||
function trackWrites(page) {
|
||||
const writes=[];page.on('request',item=>{if(item.method()!=='GET')writes.push({method:item.method(),url:new URL(item.url()).pathname,body:item.postDataJSON()});});return writes;
|
||||
@@ -482,7 +484,7 @@ test('conflict and restore states keep the dialog and write nothing',async({page
|
||||
|
||||
const before=await snapshot(request);
|
||||
box=await openDialog(page,main(page).getByRole('button',{name:'+ Create tag',exact:true}),'Create tag');
|
||||
await restore(request,saved);
|
||||
await restore(page,request,saved);
|
||||
await field(box,'Tag name').fill('sunrise');await button(box,'Apply').click();
|
||||
await expect(note(box)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was changed.');
|
||||
await expect(button(box,'Apply')).toBeDisabled();
|
||||
@@ -622,7 +624,7 @@ test('delete confirmation read failure, missing tag and restore keep Delete disa
|
||||
await expect(button(box,'Delete')).toBeDisabled();await expect(tagRow(page,'calm').locator('td:nth-child(2)')).toHaveText('2');
|
||||
await button(box,'Cancel').click();
|
||||
|
||||
box=await openRename(page,'mist');await restore(request,saved);
|
||||
box=await openRename(page,'mist');await restore(page,request,saved);
|
||||
await button(box,'Delete tag…').click();
|
||||
await expect(note(box)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was changed.');
|
||||
await expect(button(box,'Delete')).toBeDisabled();expect(await actionNames(box)).toEqual(['Reload','Cancel','Delete']);
|
||||
|
||||
@@ -18,7 +18,7 @@ const routes = [
|
||||
['tags', 'Tags & categories', null],
|
||||
// #249 local Inspect: the native Inspect a file page; Overview's tool card still links the legacy metadata page.
|
||||
['inspect', 'Inspect a file', null],
|
||||
// #249 System S2: native System; its undelivered sections hand off per section (D94, asserted below).
|
||||
// #249 System S2: native System; every section native since S12 (D94 retired, asserted below).
|
||||
['system', 'System', null],
|
||||
// #249 D51: SAVED VIEWS' Favorites, Images filtered to favorites.
|
||||
['favorites', 'Favorites', null],
|
||||
@@ -395,7 +395,7 @@ const SURFACE = {
|
||||
intro: { fontSize: '12px', marginTop: '7px' },
|
||||
};
|
||||
|
||||
for (const width of [1600, 390]) test(`overview and handoff render with concept surfaces ${width}`, async ({ page }) => {
|
||||
for (const width of [1600, 390]) test(`overview renders with concept surfaces and no System section hands off ${width}`, async ({ page }) => {
|
||||
await page.setViewportSize({ width, height: 900 });
|
||||
await page.goto(`${workspace}#/overview`);
|
||||
const main = page.locator('#workspace-main');
|
||||
@@ -412,23 +412,23 @@ for (const width of [1600, 390]) test(`overview and handoff render with concept
|
||||
expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true);
|
||||
|
||||
await sidebarLink(page, 'System').click();
|
||||
// System's Overview is native (S2 Task 6); a section not delivered yet still hands off (D94).
|
||||
// Every System section is native (S12 retired D94): the last one, Diagnostics & logs, shows concept surfaces.
|
||||
await main.locator('.section-nav').getByRole('button', { name: 'Diagnostics & logs', exact: true }).click();
|
||||
const handoff = main.locator('section.handoff');
|
||||
await expect(handoff).toHaveClass(/\bsurface\b/);
|
||||
expect(await handoff.evaluate(surfaceStyle)).toMatchObject(SURFACE);
|
||||
await expect(handoff.locator('a[href="/prompt_manager/admin"]')).toBeVisible();
|
||||
const logs = main.locator('.settings-layout > section.surface').last();
|
||||
await expect(logs.locator(':scope > h2')).toHaveText('Logs');
|
||||
expect(await logs.evaluate(surfaceStyle)).toMatchObject(SURFACE);
|
||||
await expect(main.locator('.handoff')).toHaveCount(0);
|
||||
await expect(main.locator('.panel')).toHaveCount(0);
|
||||
expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true);
|
||||
});
|
||||
|
||||
// #249 System S2 Task 5: the concept's renderSystem shell (app.js 698–706) at `#/system/<section>`, every section a D94
|
||||
// hand-off until its PR, and the Save row's truthful status (D92).
|
||||
// #249 System S2 Task 5: the concept's renderSystem shell (app.js 698–706) at `#/system/<section>`, every section native
|
||||
// (S12 retired the D94 hand-off), and the Save row's truthful status (D92).
|
||||
const SYSTEM_SECTION_LABELS = ['Overview', 'Appearance', 'Sources & scans', 'Integrations', 'Auto-tagging', 'Maintenance',
|
||||
'Backup & export', 'Diagnostics & logs'];
|
||||
const SYSTEM_HASHES = ['overview', 'appearance', 'sources', 'integrations', 'tagging', 'maintenance', 'backups', 'diagnostics'];
|
||||
|
||||
test('System is native: the concept shell, section navigation, sections, hand-offs and the Save row', async ({ page }) => {
|
||||
test('System is native: the concept shell, section navigation, sections and the Save row', async ({ page }) => {
|
||||
const mutations = [];
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) mutations.push(item.url()); });
|
||||
await page.goto(`${workspace}#/system`);
|
||||
@@ -466,7 +466,8 @@ test('System is native: the concept shell, section navigation, sections, hand-of
|
||||
// (S4 Task 11, workspace-system-sources.spec.mjs, its update notice shown in a fresh browser), Integrations (S10
|
||||
// Task 27, workspace-system-integrations.spec.mjs), Maintenance (S7 Task 19,
|
||||
// workspace-system-maintenance.spec.mjs), Backup & export (S9 Task 24, workspace-system-backups.spec.mjs) and
|
||||
// Auto-tagging (S11 Task 6, workspace-system-tagging.spec.mjs); every other section hands off.
|
||||
// Auto-tagging (S11 Task 6, workspace-system-tagging.spec.mjs) and Diagnostics & logs (S12,
|
||||
// workspace-system-diagnostics.spec.mjs): none hands off.
|
||||
if (hash === 'tagging') await expect(layout.locator('#tag-model')).toBeVisible();
|
||||
if (hash === 'overview') await expect(layout.locator(':scope > .summary-metrics')).toBeVisible();
|
||||
if (hash === 'sources') await expect(layout.locator('#source-list .operation')).toHaveCount(2);
|
||||
@@ -476,13 +477,9 @@ test('System is native: the concept shell, section navigation, sections, hand-of
|
||||
integrations: ['section.surface', 'section.surface', 'div.save-row'],
|
||||
maintenance: ['div.notice', 'section.surface', 'section.surface', 'div.save-row'],
|
||||
backups: ['div.two-col', 'section.surface', 'div.save-row'],
|
||||
tagging: ['section.surface', 'div.save-row'] }[hash]
|
||||
?? ['section.surface handoff', 'div.save-row']);
|
||||
if (['overview', 'appearance', 'sources', 'integrations', 'tagging', 'maintenance', 'backups'].includes(hash)) continue;
|
||||
const handoff = layout.locator('section.handoff');
|
||||
await expect(handoff.locator('h2')).toHaveText('Your current tools are ready');
|
||||
expect(await handoff.locator('a.workspace-link').evaluateAll(links => links.map(link => link.getAttribute('href'))))
|
||||
.toEqual(['/prompt_manager/admin']);
|
||||
tagging: ['section.surface', 'div.save-row'],
|
||||
diagnostics: ['section.surface', 'section.surface', 'div.save-row'] }[hash]);
|
||||
await expect(layout.locator('.handoff')).toHaveCount(0);
|
||||
await expect(layout.locator('#settings-status')).toHaveText('No unsaved changes.');
|
||||
await expect(page.locator('aside.sidebar a[href="#/system"]')).toHaveAttribute('aria-current', 'page');
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@ import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -34,6 +33,7 @@ from utils.execution_contexts import (
|
||||
from utils.execution_history_events import make_finish, make_start
|
||||
from utils.execution_outputs import OutputBundle, OutputReference, snapshot_output_roots
|
||||
from utils.pair_execution import EXECUTION_KEY
|
||||
from tests.browser.folder_trash import replace_folder
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
@@ -300,8 +300,7 @@ def _clear(api: PromptManagerAPI, root: Path) -> None:
|
||||
for table in _TABLES:
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for name in ("output", "input"):
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
replace_folder(root / name)
|
||||
|
||||
|
||||
def _record_run(
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
"""System › Diagnostics & logs test hooks for the browser tests (#249 System S12).
|
||||
|
||||
The logger writes into ``<server root>/logs`` (``repoint_logs``, called by
|
||||
``create_app`` before any record is written), so no test reads or deletes a
|
||||
real log folder. Every route refuses a log folder outside the server root.
|
||||
|
||||
- ``POST /__test__/logs/reset`` restores the default logging configuration,
|
||||
empties the active log, removes every other entry of the log folder (a
|
||||
link is removed, never followed) and clears the in-memory buffer. The
|
||||
global ``/__test__/reset`` calls it too;
|
||||
- ``POST /__test__/logs/seed`` ``{records, filler, rotated, active, link,
|
||||
other}`` writes buffer records with fixed times (``records``: the concept's
|
||||
four lines, then one DEBUG, ERROR and CRITICAL record, older; ``filler``
|
||||
more INFO records older still), ``active`` bytes into the active log,
|
||||
rotated copies ``prompt_manager.log.<n>`` of the listed sizes (fixed
|
||||
times), on POSIX a symlink ``prompt_manager.log.<n+1>`` to a file outside
|
||||
the folder (``link``) and a user's ``prompt_manager.log.bak`` (``other``).
|
||||
``{mode: "concept"}`` reproduces the concept's Logs sample (app.js 695)
|
||||
for the comparison: the buffer holds only its four lines, the active log
|
||||
is 1.4 MB with three rotated copies ("prompt_manager.log · 1.4 MB · rotated
|
||||
copies: 3") and console logging is off, as the concept's toggle;
|
||||
- ``POST /__test__/logs/rotate`` rolls the copies over as the handler does
|
||||
(``.2`` → ``.3``, ``.1`` → ``.2``, a new ``.1``), so a reviewed plan no
|
||||
longer matches;
|
||||
- ``GET /__test__/logs/state`` answers the folder's entries (name, size, link)
|
||||
and the buffer's length.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
ACTIVE = "prompt_manager.log"
|
||||
SEED_DAY = datetime.datetime(2026, 10, 26)
|
||||
# The concept's four lines (app.js 695), oldest first: the viewer shows them
|
||||
# newest first.
|
||||
CONCEPT_RECORDS = (
|
||||
("09:38:52", logging.INFO, "Source monitor started: output"),
|
||||
("09:39:04", logging.INFO, "Database backup verified"),
|
||||
("09:42:18", logging.WARNING, "3 images have ambiguous prompt metadata"),
|
||||
("09:42:18", logging.INFO, "Scan completed: 12 enriched, 0 duplicate links"),
|
||||
)
|
||||
# Older than the concept's lines, so they come after them: the levels the
|
||||
# concept does not show.
|
||||
LEVEL_RECORDS = (
|
||||
("09:30:01", logging.DEBUG, "Thumbnail cache lookup"),
|
||||
("09:30:02", logging.ERROR, "A source folder could not be read"),
|
||||
("09:30:03", logging.CRITICAL, "The database could not be opened"),
|
||||
)
|
||||
ROTATED_TIME = datetime.datetime(2026, 10, 25, 14, 2)
|
||||
# The concept's "prompt_manager.log · 1.4 MB · rotated copies: 3": the active
|
||||
# log stays under 1.45 MB while the server writes its own lines.
|
||||
CONCEPT_ACTIVE = int(1.4 * 1024 * 1024)
|
||||
CONCEPT_ROTATED = (3072, 2048, 1024)
|
||||
MAX_SEED_BYTES = 64 * 1024 * 1024
|
||||
DEFAULT_CONFIG = {
|
||||
"level": "INFO",
|
||||
"max_file_size": 10 * 1024 * 1024,
|
||||
"backup_count": 5,
|
||||
"console_logging": True,
|
||||
"file_logging": True,
|
||||
"buffer_size": 1000,
|
||||
}
|
||||
|
||||
|
||||
def _manager():
|
||||
from utils.logging_config import get_logger_manager
|
||||
|
||||
return get_logger_manager()
|
||||
|
||||
|
||||
def repoint_logs(root: Path) -> Path:
|
||||
"""Point the logger at ``<root>/logs`` before anything is logged.
|
||||
|
||||
The logger is built while ``logs_dir`` answers that folder, so its first
|
||||
handler never opens a log file anywhere else (a node folder with a legacy
|
||||
database would otherwise name a real log folder).
|
||||
"""
|
||||
from utils import library_location
|
||||
|
||||
folder = Path(root) / "logs"
|
||||
folder.mkdir(exist_ok=True)
|
||||
original = library_location.logs_dir
|
||||
library_location.logs_dir = lambda **_kwargs: str(folder)
|
||||
try:
|
||||
manager = _manager()
|
||||
finally:
|
||||
library_location.logs_dir = original
|
||||
with manager._files_lock:
|
||||
manager.log_dir = folder
|
||||
manager._setup_loggers()
|
||||
return folder
|
||||
|
||||
|
||||
def _require_inside(root: Path, folder: Path) -> None:
|
||||
resolved = Path(os.path.realpath(folder))
|
||||
if Path(os.path.realpath(root)) not in resolved.parents:
|
||||
raise web.HTTPForbidden(text="The log folder is outside the test server root")
|
||||
|
||||
|
||||
def _record(time_text, level, message, index):
|
||||
record = logging.LogRecord(
|
||||
"prompt_manager.test", level, __file__, index, message, None, None
|
||||
)
|
||||
hours, minutes, seconds = (int(part) for part in time_text.split(":"))
|
||||
moment = SEED_DAY.replace(hour=hours, minute=minutes, second=seconds)
|
||||
# Equal times keep their order: each later record is a microsecond newer.
|
||||
record.created = moment.timestamp() + index / 1e6
|
||||
return record
|
||||
|
||||
|
||||
def _touch(path: Path, size: int, moment: datetime.datetime) -> None:
|
||||
# A link at this name is replaced, never written through.
|
||||
if path.is_symlink():
|
||||
path.unlink()
|
||||
with open(path, "wb") as handle:
|
||||
handle.write(b"x" * size)
|
||||
stamp = moment.timestamp()
|
||||
os.utime(path, (stamp, stamp))
|
||||
|
||||
|
||||
def _append_active(manager, log_dir: Path, size: int) -> None:
|
||||
"""Grow the active log as this process's handler does (#389).
|
||||
|
||||
A second handle appending to the file the handler writes is what Clear
|
||||
all logs refuses as another program writing to it, so the bytes go
|
||||
through the handler's own stream; with file logging off, to the file.
|
||||
"""
|
||||
|
||||
def same(path) -> bool:
|
||||
return os.path.normcase(os.path.realpath(path)) == os.path.normcase(
|
||||
os.path.realpath(log_dir / ACTIVE)
|
||||
)
|
||||
|
||||
for handler in manager._file_handlers():
|
||||
stream = getattr(handler, "stream", None)
|
||||
if stream is None or not same(handler.baseFilename):
|
||||
continue
|
||||
handler.acquire()
|
||||
try:
|
||||
stream.write("x" * size)
|
||||
stream.flush()
|
||||
finally:
|
||||
handler.release()
|
||||
return
|
||||
with open(log_dir / ACTIVE, "ab") as handle:
|
||||
handle.write(b"x" * size)
|
||||
|
||||
|
||||
def _bounded(value, name, top=MAX_SEED_BYTES):
|
||||
if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= top:
|
||||
raise web.HTTPBadRequest(text=f"{name} must be an integer 0..{top}")
|
||||
return value
|
||||
|
||||
|
||||
def register_diagnostics_commands(app, routes, api, root: Path):
|
||||
"""Register the hooks; returns ``reset()`` for the global reset."""
|
||||
|
||||
def folder() -> Path:
|
||||
path = Path(_manager().log_dir)
|
||||
_require_inside(root, path)
|
||||
return path
|
||||
|
||||
def reset() -> None:
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
if manager.get_config() != DEFAULT_CONFIG:
|
||||
manager.update_config(dict(DEFAULT_CONFIG))
|
||||
with manager._files_lock:
|
||||
held = manager._acquire_file_handlers()
|
||||
try:
|
||||
with os.scandir(log_dir) as entries:
|
||||
found = list(entries)
|
||||
for entry in found:
|
||||
path = Path(entry.path)
|
||||
if entry.name == ACTIVE and entry.is_file(follow_symlinks=False):
|
||||
manager._truncate_main_log(path)
|
||||
elif entry.is_dir(follow_symlinks=False):
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
os.unlink(path)
|
||||
finally:
|
||||
manager._release_handlers(held)
|
||||
with manager._buffer_lock:
|
||||
manager._log_buffer.clear()
|
||||
|
||||
def seed_concept() -> None:
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
manager.update_config({"console_logging": False})
|
||||
with manager._buffer_lock:
|
||||
manager._log_buffer.clear()
|
||||
for index, (time_text, level, message) in enumerate(CONCEPT_RECORDS):
|
||||
record = _record(time_text, level, message, index)
|
||||
manager.add_to_buffer(record, record.getMessage())
|
||||
_append_active(manager, log_dir, CONCEPT_ACTIVE)
|
||||
for number, size in enumerate(CONCEPT_ROTATED, start=1):
|
||||
moment = ROTATED_TIME - datetime.timedelta(hours=number)
|
||||
_touch(log_dir / f"{ACTIVE}.{number}", size, moment)
|
||||
|
||||
def seed(data) -> None:
|
||||
if data.get("mode") == "concept":
|
||||
seed_concept()
|
||||
return
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
with_records = data.get("records", True) is not False
|
||||
filler = _bounded(data.get("filler", 0), "filler", 900)
|
||||
older = [
|
||||
(f"08:{index // 60:02}:{index % 60:02}", logging.INFO, f"Filler {index}")
|
||||
for index in range(filler)
|
||||
]
|
||||
named = list(LEVEL_RECORDS) + list(CONCEPT_RECORDS) if with_records else []
|
||||
ordered = older + named
|
||||
for index, (time_text, level, message) in enumerate(ordered):
|
||||
record = _record(time_text, level, message, index)
|
||||
manager.add_to_buffer(record, record.getMessage())
|
||||
active = _bounded(data.get("active", 0), "active")
|
||||
if active:
|
||||
_append_active(manager, log_dir, active)
|
||||
rotated = data.get("rotated", [])
|
||||
if not isinstance(rotated, list) or len(rotated) > 20:
|
||||
raise web.HTTPBadRequest(text="rotated must be a list of sizes")
|
||||
for number, size in enumerate(rotated, start=1):
|
||||
moment = ROTATED_TIME - datetime.timedelta(hours=number)
|
||||
_touch(log_dir / f"{ACTIVE}.{number}", _bounded(size, "size"), moment)
|
||||
if data.get("link") and os.name != "nt":
|
||||
target = Path(root) / "outside-log-target.txt"
|
||||
target.write_bytes(b"not a log of this folder")
|
||||
os.symlink(target, log_dir / f"{ACTIVE}.{len(rotated) + 1}")
|
||||
if data.get("other"):
|
||||
_touch(log_dir / f"{ACTIVE}.bak", 0, ROTATED_TIME)
|
||||
|
||||
def rotate() -> None:
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
with manager._files_lock:
|
||||
held = manager._acquire_file_handlers()
|
||||
try:
|
||||
for number in (2, 1):
|
||||
source = log_dir / f"{ACTIVE}.{number}"
|
||||
if source.is_file() and not source.is_symlink():
|
||||
os.replace(source, log_dir / f"{ACTIVE}.{number + 1}")
|
||||
_touch(log_dir / f"{ACTIVE}.1", 4096, datetime.datetime.now())
|
||||
finally:
|
||||
manager._release_handlers(held)
|
||||
|
||||
def state():
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
entries = []
|
||||
with os.scandir(log_dir) as listing:
|
||||
found = sorted(listing, key=lambda item: item.name)
|
||||
for entry in found:
|
||||
info = entry.stat(follow_symlinks=False)
|
||||
entries.append(
|
||||
{"name": entry.name, "size": info.st_size, "link": entry.is_symlink()}
|
||||
)
|
||||
with manager._buffer_lock:
|
||||
buffered = len(manager._log_buffer)
|
||||
return {"entries": entries, "buffer": buffered, "config": manager.get_config()}
|
||||
|
||||
async def body(request):
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
if not isinstance(data, dict):
|
||||
raise web.HTTPBadRequest()
|
||||
return data
|
||||
|
||||
@routes.post("/__test__/logs/reset")
|
||||
async def logs_reset(request: web.Request) -> web.Response:
|
||||
await api._run_in_executor(reset)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/logs/seed")
|
||||
async def logs_seed(request: web.Request) -> web.Response:
|
||||
data = await body(request)
|
||||
await api._run_in_executor(seed, data)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/logs/rotate")
|
||||
async def logs_rotate(request: web.Request) -> web.Response:
|
||||
await api._run_in_executor(rotate)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.get("/__test__/logs/state")
|
||||
async def logs_state(request: web.Request) -> web.Response:
|
||||
return web.json_response(await api._run_in_executor(state))
|
||||
|
||||
return reset
|
||||
@@ -3,11 +3,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from aiohttp import web
|
||||
from PIL import Image
|
||||
from tests.browser.folder_trash import replace_folder
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
@@ -51,8 +51,7 @@ def seed_library_editor(api: PromptManagerAPI, root: Path) -> dict:
|
||||
for table in _TABLES:
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for name in ("output", "input"):
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
replace_folder(root / name)
|
||||
epoch = db.get_execution_epoch()
|
||||
receipt = db.create_library_recipe(
|
||||
EDITOR_POSITIVE,
|
||||
|
||||
@@ -52,6 +52,7 @@ from typing import TYPE_CHECKING
|
||||
from aiohttp import web
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
from tests.browser.folder_trash import replace_folder
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
@@ -225,8 +226,7 @@ def seed_review(
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
api.close_annotation_captures()
|
||||
for name in ("output", "input"):
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
replace_folder(root / name)
|
||||
shutil.rmtree(root / "temp" / "review", ignore_errors=True)
|
||||
output = root / "output"
|
||||
files = []
|
||||
|
||||
@@ -35,7 +35,8 @@ read. These per-server hooks only change what a System test needs on top:
|
||||
previews for Convert outdated;
|
||||
- ``POST /__test__/system/jxl-encoder`` ``{available}`` (#317) makes the
|
||||
server read as having (``cjxl``) or lacking a JPEG XL encoder without
|
||||
probing; reset restores the thumbnail settings and the real detection.
|
||||
probing, whatever the host has installed (#371: ``false`` stands in for the
|
||||
detector itself); reset restores the thumbnail settings and the real detection.
|
||||
|
||||
- ``POST /__test__/system/duplicate-prompts`` (S7) adds two prompts whose
|
||||
text differs only in case and spacing (a duplicate prompt text group);
|
||||
@@ -118,6 +119,11 @@ def register_system_commands(app, routes, api, root: Path):
|
||||
}
|
||||
from py.api import thumbnail_encode
|
||||
|
||||
real_detect = thumbnail_encode._detect
|
||||
|
||||
def _no_encoder():
|
||||
return None
|
||||
|
||||
monitor = getattr(image_monitor, "_monitor_instance", None)
|
||||
failures: dict[str, int] = {}
|
||||
|
||||
@@ -147,6 +153,7 @@ def register_system_commands(app, routes, api, root: Path):
|
||||
api._health_report = None
|
||||
for (owner, name), value in host_values.items():
|
||||
setattr(owner, name, value)
|
||||
thumbnail_encode._detect = real_detect
|
||||
thumbnail_encode.reset_encoders()
|
||||
image_monitor._monitor_instance = monitor
|
||||
api.__dict__.pop("_library_resolver", None)
|
||||
@@ -298,7 +305,12 @@ def register_system_commands(app, routes, api, root: Path):
|
||||
available = data.get("available")
|
||||
if not isinstance(available, bool):
|
||||
raise web.HTTPBadRequest()
|
||||
# Test-only: stand in the cached detection (no probe).
|
||||
# Test-only: the fixture, not the host, decides (#371). Without an encoder the
|
||||
# detector itself stands in, so a re-probe (a "none found" is probed again on
|
||||
# every settings read) cannot find the host's real encoder; with one the cached
|
||||
# answer stands, and a failed write still probes the real detector as before.
|
||||
# Reset restores the real detector.
|
||||
thumbnail_encode._detect = real_detect if available else _no_encoder
|
||||
with thumbnail_encode._detect_lock:
|
||||
thumbnail_encode._detected = "cjxl" if available else None
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@@ -19,6 +19,7 @@ from typing import TYPE_CHECKING
|
||||
from aiohttp import web
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
from tests.browser.folder_trash import replace_folder
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
@@ -98,8 +99,7 @@ def seed_tagging(api: PromptManagerAPI, root: Path, extra_images: int = 0) -> di
|
||||
for table in _TABLES:
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for name in ("output", "input"):
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
replace_folder(root / name)
|
||||
output = root / "output"
|
||||
_png(output / FILES[0][0], (196, 140, 92))
|
||||
_png(output / FILES[1][0], (150, 110, 80))
|
||||
|
||||
@@ -5,13 +5,13 @@ from __future__ import annotations
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from tests.browser.folder_trash import replace_folder
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
@@ -129,8 +129,7 @@ def seed_workspace_visual(api: PromptManagerAPI, root: Path) -> None:
|
||||
):
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for name in ("output", "input"):
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
replace_folder(root / name)
|
||||
for parent_index, (positive, negatives) in enumerate(zip(_POSITIVES, _NEGATIVES)):
|
||||
category = ("Architecture", "Landscape", "Abstract")[parent_index % 3]
|
||||
tags = ["geometric", ("warm", "cool")[parent_index % 2]]
|
||||
|
||||
@@ -7,10 +7,10 @@ in concept order, so tag creation order and category first use match the mock.
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from aiohttp import web
|
||||
from tests.browser.folder_trash import replace_folder
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
@@ -120,8 +120,7 @@ def seed_vocabulary(
|
||||
for table in _TABLES:
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
for name in ("output", "input"):
|
||||
shutil.rmtree(root / name)
|
||||
(root / name).mkdir()
|
||||
replace_folder(root / name)
|
||||
epoch = db.get_execution_epoch()
|
||||
recipes = []
|
||||
for recipe in CONCEPT_RECIPES:
|
||||
|
||||
@@ -292,7 +292,8 @@ async def image_monitor(request: web.Request) -> web.Response:
|
||||
config.MONITORING_ENABLED = True
|
||||
monitor.start_monitoring([folder_paths.get_output_directory()])
|
||||
else:
|
||||
# Joins the image worker and the observer (up to 5 s each): off the loop.
|
||||
# Joins the image worker and the observer (one shared 5 s deadline),
|
||||
# so it runs on a worker thread, not on the event loop.
|
||||
await asyncio.to_thread(monitor.stop_monitoring)
|
||||
config.MONITORING_ENABLED = False
|
||||
return web.json_response({"monitoring": monitor.is_monitoring})
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Hang guards for waits on disk-bound work (tests only).
|
||||
|
||||
A test that waits for a job or handoff whose work is file and SQLite I/O
|
||||
(a safety backup with fsync, a restore, hashing and unlinking files) waits
|
||||
on that state; the ceiling below only ends a real hang. Hosted Windows
|
||||
runners stall disk I/O for tens of seconds at a time: a restore took over
|
||||
10 s (merge queue, #382), and on PR #405 every test setup in a 50 s window
|
||||
took up to 9 s against 0.1-0.6 s around it while the test bodies stayed
|
||||
fast, so a one-file duplicate cleanup missed a 10 s bound (#409).
|
||||
|
||||
The ceiling is per wait and stays under the suite's 120 s per-test hang
|
||||
watchdog. A test with several such waits in a row can reach the watchdog
|
||||
first; either way a real hang fails, the watchdog with every thread's stack.
|
||||
"""
|
||||
|
||||
DISK_BOUND_SECONDS = 90
|
||||
Vendored
+16
@@ -113,3 +113,19 @@ by their own suites. The expectations start as the observed baseline. A change
|
||||
that alters an outcome edits the vector in the same commit.
|
||||
`tests/js/legacy-metadata-characterization.test.js` pins what
|
||||
the legacy viewers show on the same files.
|
||||
|
||||
# Legacy contract snapshot
|
||||
|
||||
`legacy_contract_snapshot.json` records what the legacy admin and gallery pages
|
||||
talk to while System S13 (#330) moves their tools to native System: every
|
||||
registered route as `"<METHOD> <path>"` (`routes`) and the key paths of
|
||||
`GET /prompt_manager/settings` to depth 2 (`settings_keys`; list items give the
|
||||
union of their keys as `name[].key`). `tests/test_legacy_contract_snapshot.py`
|
||||
compares a temp-database API against it. Record it again only for an intended
|
||||
change, and say why in the PR body:
|
||||
|
||||
```sh
|
||||
PM_RECORD_LEGACY_SNAPSHOT=1 python3 -m unittest tests.test_legacy_contract_snapshot
|
||||
```
|
||||
|
||||
Cutover (#334) removes the test and this fixture.
|
||||
|
||||
+200
@@ -0,0 +1,200 @@
|
||||
{
|
||||
"routes": [
|
||||
"DELETE /prompt_manager/delete/{prompt_id}",
|
||||
"DELETE /prompt_manager/image-annotations/capture/{token}",
|
||||
"DELETE /prompt_manager/images/favorite",
|
||||
"DELETE /prompt_manager/images/{image_id}",
|
||||
"DELETE /prompt_manager/pairs/{pair_id}",
|
||||
"DELETE /prompt_manager/prompts/{prompt_id}/tags",
|
||||
"DELETE /prompt_manager/restore/preview/{token}",
|
||||
"DELETE /prompt_manager/saved-views/{view_id}",
|
||||
"DELETE /prompt_manager/tags/{tag_name}",
|
||||
"GET /prompt_manager/admin",
|
||||
"GET /prompt_manager/autotag/models",
|
||||
"GET /prompt_manager/backup",
|
||||
"GET /prompt_manager/backup/status",
|
||||
"GET /prompt_manager/categories",
|
||||
"GET /prompt_manager/diagnostics",
|
||||
"GET /prompt_manager/export",
|
||||
"GET /prompt_manager/gallery",
|
||||
"GET /prompt_manager/gallery.html",
|
||||
"GET /prompt_manager/gallery/subfolders",
|
||||
"GET /prompt_manager/history/runs/{run_id}",
|
||||
"GET /prompt_manager/history/runs/{run_id}/output/current/inspection",
|
||||
"GET /prompt_manager/history/runs/{run_id}/output/current/media",
|
||||
"GET /prompt_manager/history/runs/{run_id}/output/current/thumbnail",
|
||||
"GET /prompt_manager/history/runs/{run_id}/outputs",
|
||||
"GET /prompt_manager/image-annotations/capture/{token}",
|
||||
"GET /prompt_manager/images/all",
|
||||
"GET /prompt_manager/images/inspect/{image_path}",
|
||||
"GET /prompt_manager/images/output",
|
||||
"GET /prompt_manager/images/prompt/{image_path}",
|
||||
"GET /prompt_manager/images/recent",
|
||||
"GET /prompt_manager/images/search",
|
||||
"GET /prompt_manager/images/serve/{filepath}",
|
||||
"GET /prompt_manager/images/thumbnails/plan",
|
||||
"GET /prompt_manager/images/thumbnails/status",
|
||||
"GET /prompt_manager/images/{image_id}/file",
|
||||
"GET /prompt_manager/jobs",
|
||||
"GET /prompt_manager/jobs/{job_id}",
|
||||
"GET /prompt_manager/jobs/{job_id}/events",
|
||||
"GET /prompt_manager/jobs/{job_id}/report",
|
||||
"GET /prompt_manager/js/{filepath}",
|
||||
"GET /prompt_manager/lib/{filepath}",
|
||||
"GET /prompt_manager/library",
|
||||
"GET /prompt_manager/library/export",
|
||||
"GET /prompt_manager/library/options",
|
||||
"GET /prompt_manager/library/{positive_id}",
|
||||
"GET /prompt_manager/library/{positive_id}/history",
|
||||
"GET /prompt_manager/library/{positive_id}/history/pairs/{pair_id}/newest",
|
||||
"GET /prompt_manager/library/{positive_id}/images",
|
||||
"GET /prompt_manager/logs",
|
||||
"GET /prompt_manager/logs/config",
|
||||
"GET /prompt_manager/logs/download/{filename}",
|
||||
"GET /prompt_manager/logs/files",
|
||||
"GET /prompt_manager/logs/stats",
|
||||
"GET /prompt_manager/lora/detect",
|
||||
"GET /prompt_manager/lora/status",
|
||||
"GET /prompt_manager/lora/trigger-words",
|
||||
"GET /prompt_manager/maintenance/duplicates/{job_id}",
|
||||
"GET /prompt_manager/pairs",
|
||||
"GET /prompt_manager/pairs/export",
|
||||
"GET /prompt_manager/pairs/{pair_id}",
|
||||
"GET /prompt_manager/prompts/{prompt_id}/images",
|
||||
"GET /prompt_manager/recent",
|
||||
"GET /prompt_manager/review/events",
|
||||
"GET /prompt_manager/review/items",
|
||||
"GET /prompt_manager/review/items/{item_id}",
|
||||
"GET /prompt_manager/saved-views",
|
||||
"GET /prompt_manager/scan/status",
|
||||
"GET /prompt_manager/scan_duplicates",
|
||||
"GET /prompt_manager/scan_duplicates/status",
|
||||
"GET /prompt_manager/scan_output_dir",
|
||||
"GET /prompt_manager/search",
|
||||
"GET /prompt_manager/settings",
|
||||
"GET /prompt_manager/stats",
|
||||
"GET /prompt_manager/subfolders",
|
||||
"GET /prompt_manager/system/diagnostics",
|
||||
"GET /prompt_manager/system/health",
|
||||
"GET /prompt_manager/system/library",
|
||||
"GET /prompt_manager/system/overview",
|
||||
"GET /prompt_manager/system/statistics",
|
||||
"GET /prompt_manager/tags",
|
||||
"GET /prompt_manager/tags/filter",
|
||||
"GET /prompt_manager/tags/stats",
|
||||
"GET /prompt_manager/tags/suggest",
|
||||
"GET /prompt_manager/tags/{tag_name}/prompts",
|
||||
"GET /prompt_manager/test",
|
||||
"GET /prompt_manager/vocabulary/categories",
|
||||
"GET /prompt_manager/vocabulary/tags",
|
||||
"GET /prompt_manager/vocabulary/tags/{tag_id}/impact",
|
||||
"GET /prompt_manager/web",
|
||||
"GET /prompt_manager/workspace",
|
||||
"PATCH /prompt_manager/library/{positive_id}",
|
||||
"PATCH /prompt_manager/pairs/{pair_id}",
|
||||
"PATCH /prompt_manager/saved-views/{view_id}",
|
||||
"PATCH /prompt_manager/vocabulary/tags/{tag_id}",
|
||||
"POST /prompt_manager/autotag/apply",
|
||||
"POST /prompt_manager/autotag/batch/preview",
|
||||
"POST /prompt_manager/autotag/download/{model_type}",
|
||||
"POST /prompt_manager/autotag/single",
|
||||
"POST /prompt_manager/autotag/start",
|
||||
"POST /prompt_manager/autotag/unload",
|
||||
"POST /prompt_manager/bulk/category",
|
||||
"POST /prompt_manager/bulk/delete",
|
||||
"POST /prompt_manager/bulk/tags",
|
||||
"POST /prompt_manager/cleanup",
|
||||
"POST /prompt_manager/delete_duplicate_images",
|
||||
"POST /prompt_manager/diagnostics/test-link",
|
||||
"POST /prompt_manager/gallery/rescan",
|
||||
"POST /prompt_manager/image-annotations/observe",
|
||||
"POST /prompt_manager/image-annotations/suggest",
|
||||
"POST /prompt_manager/images/clear-thumbnails",
|
||||
"POST /prompt_manager/images/generate-thumbnails",
|
||||
"POST /prompt_manager/images/generate-thumbnails/progress",
|
||||
"POST /prompt_manager/images/link",
|
||||
"POST /prompt_manager/images/reveal",
|
||||
"POST /prompt_manager/jobs/autotag",
|
||||
"POST /prompt_manager/jobs/duplicates",
|
||||
"POST /prompt_manager/jobs/image-tags",
|
||||
"POST /prompt_manager/jobs/model-download",
|
||||
"POST /prompt_manager/jobs/scan",
|
||||
"POST /prompt_manager/jobs/thumbnails",
|
||||
"POST /prompt_manager/jobs/{job_id}/cancel",
|
||||
"POST /prompt_manager/library",
|
||||
"POST /prompt_manager/library/{positive_id}/variants",
|
||||
"POST /prompt_manager/logs/config",
|
||||
"POST /prompt_manager/logs/truncate",
|
||||
"POST /prompt_manager/lora/enable",
|
||||
"POST /prompt_manager/lora/import/apply",
|
||||
"POST /prompt_manager/lora/import/preview",
|
||||
"POST /prompt_manager/lora/refresh-cache",
|
||||
"POST /prompt_manager/lora/scan",
|
||||
"POST /prompt_manager/maintenance",
|
||||
"POST /prompt_manager/maintenance/apply",
|
||||
"POST /prompt_manager/maintenance/preview",
|
||||
"POST /prompt_manager/pairs",
|
||||
"POST /prompt_manager/prompts/tags",
|
||||
"POST /prompt_manager/prompts/{prompt_id}/tags",
|
||||
"POST /prompt_manager/restore",
|
||||
"POST /prompt_manager/restore/apply",
|
||||
"POST /prompt_manager/restore/preview",
|
||||
"POST /prompt_manager/review/items/{item_id}/decision",
|
||||
"POST /prompt_manager/save",
|
||||
"POST /prompt_manager/saved-views",
|
||||
"POST /prompt_manager/scan",
|
||||
"POST /prompt_manager/scan/cancel",
|
||||
"POST /prompt_manager/scan_duplicates/stream",
|
||||
"POST /prompt_manager/settings",
|
||||
"POST /prompt_manager/settings/validate-source",
|
||||
"POST /prompt_manager/system/health",
|
||||
"POST /prompt_manager/system/library/keep",
|
||||
"POST /prompt_manager/system/library/move",
|
||||
"POST /prompt_manager/tags/merge",
|
||||
"POST /prompt_manager/vocabulary/categories/apply",
|
||||
"POST /prompt_manager/vocabulary/categories/preview",
|
||||
"POST /prompt_manager/vocabulary/tags",
|
||||
"POST /prompt_manager/vocabulary/tags/{tag_id}/delete",
|
||||
"PUT /prompt_manager/image-annotations/tags",
|
||||
"PUT /prompt_manager/images/favorite",
|
||||
"PUT /prompt_manager/prompts/{prompt_id}",
|
||||
"PUT /prompt_manager/prompts/{prompt_id}/rating",
|
||||
"PUT /prompt_manager/tags/{tag_name}"
|
||||
],
|
||||
"settings_keys": [
|
||||
"monitor",
|
||||
"monitor.observer_alive",
|
||||
"monitor.present",
|
||||
"monitor.scheduled",
|
||||
"revision",
|
||||
"settings",
|
||||
"settings.autotag_character_threshold",
|
||||
"settings.autotag_existing",
|
||||
"settings.autotag_general_threshold",
|
||||
"settings.autotag_model_type",
|
||||
"settings.autotag_suggestion_limit",
|
||||
"settings.autotag_template",
|
||||
"settings.cpu_count",
|
||||
"settings.gallery_root_path",
|
||||
"settings.gallery_root_paths",
|
||||
"settings.infinite_scroll",
|
||||
"settings.monitored_directories",
|
||||
"settings.monitoring_enabled",
|
||||
"settings.result_timeout",
|
||||
"settings.safety_backup",
|
||||
"settings.thumbnail_edge",
|
||||
"settings.thumbnail_format",
|
||||
"settings.thumbnail_quality",
|
||||
"settings.webui_display_mode",
|
||||
"settings.worker_threads",
|
||||
"sources",
|
||||
"sources.auto_detected",
|
||||
"sources.configured",
|
||||
"sources.effective",
|
||||
"success",
|
||||
"thumbnail_encoders",
|
||||
"thumbnail_encoders.jpeg",
|
||||
"thumbnail_encoders.jxl",
|
||||
"thumbnail_encoders.webp"
|
||||
]
|
||||
}
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
{
|
||||
"generation_params": [
|
||||
"case:basic",
|
||||
"case:shared-prompt-pipeline",
|
||||
"tests/fixtures/inspection_parity/f02.png",
|
||||
"tests/fixtures/inspection_parity/f03.png",
|
||||
"tests/fixtures/inspection_parity/f06c.png",
|
||||
"tests/fixtures/inspection_parity/f06e.png",
|
||||
"tests/fixtures/inspection_parity/f07a.png",
|
||||
"tests/fixtures/inspection_parity/f07b.png",
|
||||
"tests/fixtures/inspection_parity/f08.png",
|
||||
"tests/fixtures/inspection_parity/f09.png",
|
||||
"tests/fixtures/inspection_parity/f10.png",
|
||||
"tests/fixtures/inspection_parity/f11.png",
|
||||
"tests/fixtures/inspection_parity/f15.png",
|
||||
"tests/fixtures/inspection_parity/f16.png",
|
||||
"tests/fixtures/inspection_parity/f23.png"
|
||||
],
|
||||
"inspection_parity": [
|
||||
"f02",
|
||||
"f03",
|
||||
"f06c",
|
||||
"f06e",
|
||||
"f07a",
|
||||
"f07b",
|
||||
"f08",
|
||||
"f09",
|
||||
"f10",
|
||||
"f11",
|
||||
"f14",
|
||||
"f15",
|
||||
"f16",
|
||||
"f23"
|
||||
],
|
||||
"local_resolver": [
|
||||
"anchor-first-stage-no-latent",
|
||||
"anchor-same-links",
|
||||
"anchor-same-links-unanchored",
|
||||
"built-chars",
|
||||
"class-type-missing",
|
||||
"class-type-number",
|
||||
"combine-equal",
|
||||
"controlnet-slot1",
|
||||
"custom-cfg",
|
||||
"duplicate-key",
|
||||
"encode",
|
||||
"int-id",
|
||||
"latent-encode",
|
||||
"overwritten-bad-workflow",
|
||||
"pair-execution",
|
||||
"pair-execution-bad-json",
|
||||
"pair-execution-mismatch",
|
||||
"pair-execution-negative-changed",
|
||||
"pair-execution-other-node",
|
||||
"pair-execution-truncated",
|
||||
"pair-execution-two-records",
|
||||
"pair-execution-version-float",
|
||||
"pair-execution-version-true",
|
||||
"pair-graph",
|
||||
"pass-through-unknown-role-inputs",
|
||||
"passthrough",
|
||||
"prompt-and-parameters",
|
||||
"prompt-manager",
|
||||
"select-detailer-chain-same",
|
||||
"select-detailer-no-sampler-upstream",
|
||||
"select-detailer-output-walk-exhausted",
|
||||
"select-detailer-same-prompt",
|
||||
"select-edge-hires-fix-denoised-slot",
|
||||
"select-edge-hires-fix-lineage-cycle",
|
||||
"select-edge-hires-fix-two-refiners",
|
||||
"select-flux-concat",
|
||||
"select-flux-reroute-input",
|
||||
"select-flux-three-stage",
|
||||
"select-hires-fix",
|
||||
"select-hires-fix-both-saved",
|
||||
"select-hires-fix-reroute",
|
||||
"select-hires-fix-unlisted",
|
||||
"select-multistage-latent",
|
||||
"select-preview-dropped",
|
||||
"select-preview-through-bad-node",
|
||||
"select-runtime-flux-other-full-prompt",
|
||||
"select-runtime-flux-trigger",
|
||||
"select-runtime-search-list-append",
|
||||
"select-runtime-through-chained",
|
||||
"select-runtime-through-concat",
|
||||
"select-runtime-through-text-encode",
|
||||
"select-runtime-traced-prepend",
|
||||
"select-runtime-trigger-append-single",
|
||||
"select-runtime-trigger-pair",
|
||||
"select-runtime-trigger-prepend",
|
||||
"select-three-identical-one-advanced",
|
||||
"select-two-identical",
|
||||
"select-unconsumed-stage-kept",
|
||||
"several-samplers",
|
||||
"smz-false",
|
||||
"text-at-limit",
|
||||
"unknown-encoder",
|
||||
"workflow-type-dict",
|
||||
"workflow-type-in-subgraph",
|
||||
"workflow-type-list",
|
||||
"workflow-type-without-id",
|
||||
"zeroed"
|
||||
],
|
||||
"unknown_nodes": [
|
||||
"unknown-role-inputs"
|
||||
]
|
||||
}
|
||||
+1
-1
File diff suppressed because one or more lines are too long
@@ -36,33 +36,48 @@ function fixture() {
|
||||
const reply = (index, data) => requests[index].resolve({ json: async () => data });
|
||||
return { manager, requests, reply, notifications };
|
||||
}
|
||||
test('gallery review retains root identity and applies tags with the returned prompt ID', async () => {
|
||||
const SHA = 'a'.repeat(64);
|
||||
test('gallery review retains root identity and applies the tags to the image by its sha256 (#379)', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
assert.equal(requests[0].body.root_index, 1);
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'] });
|
||||
assert.equal(requests[0].body.existing, 'merge');
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'], sha256: SHA, image_tags: [] });
|
||||
await load;
|
||||
manager.loadNextReviewImage = () => {};
|
||||
const apply = manager.applyReviewTags();
|
||||
assert.deepEqual(requests[1].body, { prompt_id: 42, tags: ['smile'] });
|
||||
assert.equal(requests[1].url, '/prompt_manager/autotag/apply');
|
||||
assert.deepEqual(requests[1].body, ImageHelpers.autotagApplyBody(
|
||||
{ id: 'digest', path: 'same.png', root_index: 1 }, SHA, ['smile']));
|
||||
assert.equal('prompt_id' in requests[1].body, false);
|
||||
reply(1, { success: true });
|
||||
await apply;
|
||||
});
|
||||
test('unlinked images do not submit an invalid apply request or advance', async () => {
|
||||
test('an unlinked image applies to the image like a linked one', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
reply(0, { success: true, prompt_id: null, tags: ['smile'] });
|
||||
reply(0, { success: true, prompt_id: null, tags: ['smile'], sha256: SHA, image_tags: [] });
|
||||
await load;
|
||||
manager.loadNextReviewImage = () => {};
|
||||
const apply = manager.applyReviewTags();
|
||||
if (requests[1]) reply(1, { success: false, error: "Unexpected apply" });
|
||||
assert.equal(requests[1].body.sha256, SHA);
|
||||
assert.deepEqual(requests[1].body.tags, ['smile']);
|
||||
reply(1, { success: true });
|
||||
await apply;
|
||||
assert.equal(requests.length, 1);
|
||||
assert.equal(manager.autoTagState.reviewIndex, 0);
|
||||
});
|
||||
test('an answer without a captured sha256 never sends an apply', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'] });
|
||||
await load;
|
||||
manager.loadNextReviewImage = () => {};
|
||||
await manager.applyReviewTags();
|
||||
assert.equal(requests.filter(r => r.url === '/prompt_manager/autotag/apply').length, 0);
|
||||
});
|
||||
test('a failed apply keeps the current image available for retry', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'] });
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'], sha256: SHA, image_tags: [] });
|
||||
await load;
|
||||
const apply = manager.applyReviewTags();
|
||||
reply(1, { success: false, error: 'Try again' });
|
||||
|
||||
@@ -5,6 +5,8 @@ const {
|
||||
formatImageCaption,
|
||||
workflowDocumentHtml,
|
||||
autotagSingleBody,
|
||||
autotagApplyBody,
|
||||
autotagCompleteMessage,
|
||||
fallbackImageSource,
|
||||
} = require("../../web/js/image-helpers.js");
|
||||
|
||||
@@ -121,3 +123,27 @@ test('metadata lookup preserves encoded filename characters and works behind a p
|
||||
assert.equal(metadataLookupUrl('/prompt_manager/images/serve/a.png?v=123'), '/prompt_manager/images/prompt/a.png');
|
||||
assert.equal(metadataLookupUrl('/elsewhere/a.png'), null);
|
||||
});
|
||||
|
||||
test("autotagSingleBody asks to skip auto-tagged images only when told (#379)", () => {
|
||||
assert.equal(autotagSingleBody({ id: 1 }, WD14).existing, "merge");
|
||||
assert.equal(autotagSingleBody({ id: 1 }, { ...WD14, existing: "skip" }).existing, "skip");
|
||||
assert.equal(autotagSingleBody({ id: 1 }, { ...WD14, existing: "odd" }).existing, "merge");
|
||||
});
|
||||
|
||||
test("autotagApplyBody names the image, its bytes and the tags, never a prompt (#379)", () => {
|
||||
const sha = "a".repeat(64);
|
||||
assert.deepEqual(autotagApplyBody({ id: 4, prompt_id: 7 }, sha, ["bird"]), { image_id: 4, sha256: sha, tags: ["bird"] });
|
||||
assert.deepEqual(autotagApplyBody({ id: "digest", path: "sub/a.png", root_index: 1 }, sha, []),
|
||||
{ path: "sub/a.png", root_index: 1, sha256: sha, tags: [] });
|
||||
assert.deepEqual(autotagApplyBody({ id: 2 }, undefined, null), { image_id: 2, sha256: "", tags: [] });
|
||||
const tags = ["x"];
|
||||
autotagApplyBody({ id: 1 }, sha, tags).tags.push("y");
|
||||
assert.deepEqual(tags, ["x"]);
|
||||
});
|
||||
|
||||
test("autotagCompleteMessage counts images (#379)", () => {
|
||||
assert.equal(autotagCompleteMessage(2), "Auto tagging complete! Tagged 2 images.");
|
||||
assert.equal(autotagCompleteMessage(1), "Auto tagging complete! Tagged 1 image.");
|
||||
assert.equal(autotagCompleteMessage(0), "Auto tagging complete! Tagged 0 images.");
|
||||
assert.equal(autotagCompleteMessage(undefined), "Auto tagging complete! Tagged 0 images.");
|
||||
});
|
||||
|
||||
@@ -97,6 +97,30 @@ test("connect reports a non-2xx response through onerror and never onmessage", a
|
||||
assert.equal(source.readyState, source.CLOSED);
|
||||
});
|
||||
|
||||
test("a non-2xx JSON answer's error sentence reaches onerror as serverMessage", async () => {
|
||||
let error = null;
|
||||
const fetchImpl = async () => new Response(
|
||||
JSON.stringify({ success: false, error: "Auto-tagging is already running" }),
|
||||
{ status: 409, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
const source = connect("/x", { fetchImpl });
|
||||
source.onerror = (err) => { error = err; };
|
||||
await until(() => error !== null);
|
||||
assert.match(error.message, /409/);
|
||||
assert.equal(error.status, 409);
|
||||
assert.equal(error.serverMessage, "Auto-tagging is already running");
|
||||
});
|
||||
|
||||
test("a non-2xx answer without a JSON error has no serverMessage", async () => {
|
||||
let error = null;
|
||||
const fetchImpl = async () => new Response("nope", { status: 500 });
|
||||
const source = connect("/x", { fetchImpl });
|
||||
source.onerror = (err) => { error = err; };
|
||||
await until(() => error !== null);
|
||||
assert.equal(error.status, 500);
|
||||
assert.equal(error.serverMessage, undefined);
|
||||
});
|
||||
|
||||
test("connect reports a network failure through onerror", async () => {
|
||||
let error = null;
|
||||
const source = connect("/x", { fetchImpl: async () => { throw new TypeError("Failed to fetch"); } });
|
||||
|
||||
@@ -40,7 +40,7 @@ const semantics = compileSemantics(JSON.parse(fs.readFileSync(path.join(here, '.
|
||||
const { extractGenerationFacts } = require(path.join(here, '..', '..', '..', 'web', 'js', 'comfy-metadata.js'));
|
||||
|
||||
/** The local evidence for one set of record texts. */
|
||||
const resolved = records => ({ evidence: resolveLocalEvidence(records, semantics) });
|
||||
const resolved = (records, fileName = null) => ({ evidence: resolveLocalEvidence(records, semantics, fileName) });
|
||||
|
||||
/** The local wire's evidence and six facts for one read (no legacy fill, as in the vectors' own harness). */
|
||||
function localPayload(read, file, classification) {
|
||||
@@ -59,14 +59,14 @@ const output = {};
|
||||
for (const entry of JSON.parse(Buffer.concat(chunks).toString('utf8'))) {
|
||||
const { name } = entry;
|
||||
if (entry.mode === 'resolve') {
|
||||
if (entry.records !== undefined) { output[name] = resolved(entry.records); continue; }
|
||||
if (entry.records !== undefined) { output[name] = resolved(entry.records, entry.fileName ?? null); continue; }
|
||||
const blob = await fs.openAsBlob(entry.path);
|
||||
const { classification, result } = await readLocalFile(blob, { pngMetadata });
|
||||
const read = classification === null ? null : toInspectionRead(classification, result).read;
|
||||
if (classification === null) { output[name] = { unreadable: true }; continue; }
|
||||
// The evidence is the local wire's own (buildLocalPayload), and it equals resolveLocalEvidence of the records.
|
||||
const local = localPayload(result, { name: path.basename(entry.path), size: blob.size }, classification);
|
||||
const direct = read === null ? unreadableEvidence() : resolved(read.records).evidence;
|
||||
const direct = read === null ? unreadableEvidence() : resolved(read.records, path.basename(entry.path)).evidence;
|
||||
if (JSON.stringify(direct) !== JSON.stringify(local.evidence)) throw new Error(`${name}: the wire's roles are not the resolver's`);
|
||||
output[name] = read === null ? { unreadable: true, ...local } : local;
|
||||
continue;
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const source = import('../../web/js/workspace/images-multiselect.mjs');
|
||||
|
||||
const KEYS = Object.freeze(['0:a', '0:b', '0:c', '0:d', '1:e']);
|
||||
|
||||
test('a range runs from the anchor to the target in display order, either way', async () => {
|
||||
const { rangeKeys } = await source;
|
||||
assert.deepEqual(rangeKeys(KEYS, '0:b', '0:d'), ['0:b', '0:c', '0:d']);
|
||||
assert.deepEqual(rangeKeys(KEYS, '1:e', '0:c'), ['0:c', '0:d', '1:e']);
|
||||
assert.deepEqual(rangeKeys(KEYS, '0:c', '0:c'), ['0:c']);
|
||||
});
|
||||
|
||||
test('a range without a listed anchor is the target alone, and an unlisted target is empty', async () => {
|
||||
const { rangeKeys } = await source;
|
||||
assert.deepEqual(rangeKeys(KEYS, null, '0:c'), ['0:c']);
|
||||
assert.deepEqual(rangeKeys(KEYS, '9:gone', '0:c'), ['0:c']);
|
||||
assert.deepEqual(rangeKeys(KEYS, '0:a', '9:gone'), []);
|
||||
});
|
||||
|
||||
test('applying keys adds or removes them and keeps the same frozen set when nothing changes', async () => {
|
||||
const { applyKeys } = await source;
|
||||
const start = Object.freeze(new Set(['0:a']));
|
||||
const added = applyKeys(start, ['0:b', '0:c'], true);
|
||||
assert.deepEqual([...added], ['0:a', '0:b', '0:c']);
|
||||
assert.ok(Object.isFrozen(added));
|
||||
assert.deepEqual([...start], ['0:a']);
|
||||
const removed = applyKeys(added, ['0:a', '0:b'], false);
|
||||
assert.deepEqual([...removed], ['0:c']);
|
||||
assert.equal(applyKeys(removed, ['0:c'], true), removed);
|
||||
assert.equal(applyKeys(removed, ['0:a'], false), removed);
|
||||
});
|
||||
|
||||
test('every listed key is checked only when each one is in the set', async () => {
|
||||
const { everyChecked } = await source;
|
||||
assert.equal(everyChecked(new Set(KEYS), KEYS), true);
|
||||
assert.equal(everyChecked(new Set(KEYS.slice(1)), KEYS), false);
|
||||
assert.equal(everyChecked(new Set(['x', ...KEYS.slice(1)]), KEYS), false);
|
||||
assert.equal(everyChecked(new Set(), []), false);
|
||||
});
|
||||
|
||||
test('the select-all-matching offer needs every loaded image checked and more to load', async () => {
|
||||
const { offersAllMatching } = await source;
|
||||
const all = new Set(KEYS);
|
||||
assert.equal(offersAllMatching(all, KEYS, { total: 9, hasMore: true }), true);
|
||||
assert.equal(offersAllMatching(all, KEYS, { total: 5, hasMore: false }), false);
|
||||
assert.equal(offersAllMatching(new Set(KEYS.slice(1)), KEYS, { total: 9, hasMore: true }), false);
|
||||
assert.equal(offersAllMatching(new Set(), [], { total: 9, hasMore: true }), false);
|
||||
});
|
||||
|
||||
test('the counts read "N selected" and "All N matching selected", grouped', async () => {
|
||||
const { matchingLabel, matchingOffer } = await source;
|
||||
assert.equal(matchingLabel(12345), 'All 12,345 matching selected');
|
||||
assert.equal(matchingOffer(12345), 'Select all 12,345 matching');
|
||||
});
|
||||
|
||||
function fakeTimers() {
|
||||
const timers = new Map();
|
||||
let next = 1;
|
||||
return {
|
||||
set: (callback, delay) => { timers.set(next, { callback, delay }); return next++; },
|
||||
clear: id => timers.delete(id),
|
||||
fire: () => { for (const [id, { callback }] of [...timers]) { timers.delete(id); callback(); } },
|
||||
pending: () => timers.size,
|
||||
delays: () => [...timers.values()].map(timer => timer.delay),
|
||||
};
|
||||
}
|
||||
|
||||
test('a held press fires once after the delay and swallows the click that follows', async () => {
|
||||
const { createLongPress, LONG_PRESS_MS } = await source;
|
||||
const timers = fakeTimers(), fired = [];
|
||||
const press = createLongPress({ onLongPress: target => fired.push(target), setTimer: timers.set, clearTimer: timers.clear });
|
||||
press.down(10, 10, 'card-a');
|
||||
assert.deepEqual(timers.delays(), [LONG_PRESS_MS]);
|
||||
assert.equal(LONG_PRESS_MS, 500);
|
||||
press.move(14, 13);
|
||||
timers.fire();
|
||||
assert.deepEqual(fired, ['card-a']);
|
||||
press.up();
|
||||
assert.equal(press.consumeClick(), true);
|
||||
assert.equal(press.consumeClick(), false);
|
||||
});
|
||||
|
||||
test('moving past the slop, lifting early or cancelling never fires', async () => {
|
||||
const { createLongPress, LONG_PRESS_SLOP } = await source;
|
||||
assert.equal(LONG_PRESS_SLOP, 10);
|
||||
const timers = fakeTimers(), fired = [];
|
||||
const press = createLongPress({ onLongPress: target => fired.push(target), setTimer: timers.set, clearTimer: timers.clear });
|
||||
press.down(0, 0, 'a'); press.move(0, 11);
|
||||
assert.equal(timers.pending(), 0);
|
||||
press.down(0, 0, 'b'); press.up();
|
||||
assert.equal(timers.pending(), 0);
|
||||
press.down(0, 0, 'c');
|
||||
assert.equal(press.pending(), true);
|
||||
press.cancel();
|
||||
assert.equal(press.pending(), false);
|
||||
assert.equal(timers.pending(), 0);
|
||||
press.move(0, 0); press.up(); press.cancel();
|
||||
assert.deepEqual(fired, []);
|
||||
assert.equal(press.consumeClick(), false);
|
||||
// A new press drops a long press whose click never came.
|
||||
press.down(0, 0, 'd'); timers.fire();
|
||||
press.down(0, 0, 'e');
|
||||
assert.equal(press.consumeClick(), false);
|
||||
timers.fire();
|
||||
assert.deepEqual(fired, ['d', 'e']);
|
||||
// A cancel (a mouse press on a hybrid device) forgets a fired press whose click never came.
|
||||
press.cancel();
|
||||
assert.equal(press.consumeClick(), false);
|
||||
});
|
||||
@@ -234,6 +234,23 @@ test('the count line reads N of M images and K positive prompts only when known'
|
||||
assert.equal(countLine({loaded:1200,total:3400,positiveCount:0}),'1,200 of 3,400 images · 0 positive prompts');
|
||||
});
|
||||
|
||||
test('the count line reads as loading, never as an empty result, until a first page answers (#391)', async () => {
|
||||
const {countLine,createImageStream}=await source;
|
||||
assert.equal(countLine({loaded:0,total:0,positiveCount:null,loading:true,error:null}),'Loading images…');
|
||||
// A failed first page and a settled empty one say what the server said; a later page keeps the loaded count.
|
||||
assert.equal(countLine({loaded:0,total:0,positiveCount:null,loading:false,error:new Error('x')}),'0 of 0 images');
|
||||
assert.equal(countLine({loaded:0,total:0,positiveCount:0,loading:false,error:null}),'0 of 0 images · 0 positive prompts');
|
||||
assert.equal(countLine({loaded:40,total:102,positiveCount:2,loading:true,error:null}),'40 of 102 images · 2 positive prompts');
|
||||
// While a reset's first page is in flight the line reads as loading (the cleared state before it is replaced in
|
||||
// the same task, so it is never painted).
|
||||
let answer;const lines=[];
|
||||
const stream=createImageStream({api:{get:()=>new Promise(resolve=>{answer=resolve;})},onChange:state=>lines.push(countLine(state))});
|
||||
const loading=stream.reset({tags:['cinematic']});
|
||||
assert.equal(lines.at(-1),'Loading images…');
|
||||
answer({images:[{path:'a.png',root_index:0}],total:1,has_more:false,next_cursor:null,positive_count:1});await loading;
|
||||
assert.equal(lines.at(-1),'1 of 1 image · 1 positive prompt');
|
||||
});
|
||||
|
||||
test('the caption date names the newest result day under Newest first only', async () => {
|
||||
const {captionDate}=await source;
|
||||
const now=new Date(2026,9,17,15,0);
|
||||
@@ -494,3 +511,11 @@ test('defaults apply to a new visit but not to the current view or a saved view'
|
||||
assert.deepEqual(initialImagesView({}), { layout: 'grid', density: 'comfortable' });
|
||||
assert.deepEqual(initialImagesView({ preferences: {} }), { layout: 'grid', density: 'comfortable' });
|
||||
});
|
||||
|
||||
test('listingFilter is the listing query as an object, without paging (#392)', async () => {
|
||||
const { listingFilter } = await source;
|
||||
assert.deepEqual(listingFilter({q:'cat', root:'1', subfolder:'a/b', category:'Animals', tags:[' x ','y'], media:'video', sort:'oldest', favorites:true}),
|
||||
{q:'cat', root:'1', subfolder:'a/b', category:'Animals', tags:'x,y', media:'video', sort:'oldest', favorites:'1'});
|
||||
assert.deepEqual(listingFilter({root:'all', q:'', tags:[], media:'all'}), {sort:'newest'});
|
||||
assert.throws(() => listingFilter({sort:'sideways'}), TypeError);
|
||||
});
|
||||
|
||||
@@ -196,3 +196,29 @@ test('a preset starts the Tag image dialog on the chosen model and options', asy
|
||||
assert.throws(() => createImageTaggingController({ api, target, preset: { model: 'gguf', options: { instructions: '' } } }), TypeError);
|
||||
assert.throws(() => createImageTaggingController({ api, target, preset: { model: 'gguf', options: { nope: 1 } } }), TypeError);
|
||||
});
|
||||
|
||||
test('all matching sends the listing filter instead of targets, up to 10,000 images (#392)', async () => {
|
||||
const { openAddTags, openAutoTag, MAX_FILTER_IMAGES, TOO_MANY_MATCHING } = module();
|
||||
assert.equal(MAX_FILTER_IMAGES, 10000);
|
||||
assert.equal(TOO_MANY_MATCHING, 'Narrow the view to at most 10,000 images.');
|
||||
const toasts = [], posted = [], tagged = [];
|
||||
let ended = null;
|
||||
const store = { start: async (kind, body) => { posted.push([kind, body]); return { job: { id: JOB_ID, state: 'running' } }; },
|
||||
find: () => ended, subscribe: () => () => {} };
|
||||
const filter = { q: 'cat', sort: 'newest' };
|
||||
// The loaded images can be more than 500: the filter, not the identities, is sent.
|
||||
const loaded = Array.from({ length: 600 }, (_, index) => image(`bulk/${index}.png`));
|
||||
let dialog = null;
|
||||
openAddTags({ scope: pageScope(), store, images: loaded, filter, count: 1234, notify: text => toasts.push(text),
|
||||
onTagged: keys => tagged.push(keys.size), openInput: args => { dialog = args; } });
|
||||
assert.ok(dialog, 'the Add tags dialog opens');
|
||||
ended = { id: JOB_ID, state: 'finished', counters: { tagged: 3 } };
|
||||
await dialog.onApply('sunset');
|
||||
assert.deepEqual(posted, [['image-tags', { mode: 'add', filter, tags: ['sunset'] }]]);
|
||||
// The loaded files are read again when the job tagged any.
|
||||
assert.deepEqual(tagged, [600]);
|
||||
openAddTags({ scope: pageScope(), store, images: loaded, filter, count: 10001, notify: text => toasts.push(text),
|
||||
openInput: () => assert.fail('no dialog over the cap') });
|
||||
openAutoTag({ api: {}, scope: pageScope(), store, images: loaded, filter, count: 10001, notify: text => toasts.push(text) });
|
||||
assert.deepEqual(toasts, [TOO_MANY_MATCHING, TOO_MANY_MATCHING]);
|
||||
});
|
||||
|
||||
@@ -155,7 +155,9 @@ test('committed server answers equal the local evidence', () => {
|
||||
value !== null && typeof value === 'object' ? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
||||
let compared = 0;
|
||||
for (const {name, records, evidence} of vectors) {
|
||||
assert.deepEqual(canonical(resolver.resolveLocalEvidence(records, semantics)), canonical(evidence), name);
|
||||
// A case may name the inspected file (`__file_name`, #381 saver anchoring); it is no record.
|
||||
const {__file_name: fileName = null, ...roles} = records;
|
||||
assert.deepEqual(canonical(resolver.resolveLocalEvidence(roles, semantics, fileName)), canonical(evidence), name);
|
||||
compared++;
|
||||
}
|
||||
assert.ok(compared > 280, `compared ${compared}`);
|
||||
|
||||
@@ -23,7 +23,7 @@ test('tags is a native route titled Tags; only unbuilt routes hand off to legacy
|
||||
assert.equal(routeTitle(route), route.label);
|
||||
}
|
||||
assert.equal(routeTitle(resolveRoute('#/library/7')), 'Prompt library');
|
||||
// System is native from S2 (#249 System): its undelivered sections hand off per section (D94), not the route.
|
||||
// System is native from S2 (#249 System), and every section of it from S12 (D94 retired), not the route.
|
||||
for (const id of ['images', 'library', 'tags', 'inspect', 'overview', 'activity', 'review', 'system']) assert.equal(isHandoff(resolveRoute(`#/${id}`)), false, id);
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
// System › Diagnostics & logs (#249 System S12, #329): the concept's two surfaces (app.js 694–695) with live values: the
|
||||
// meta-grid texts, the report's name and merge (D110), the log viewer's lines, Lines validation, statistics, retention
|
||||
// and file rows, and the Clear all logs copy (D109).
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const diagnostics = import('../../web/js/workspace/system-diagnostics.mjs');
|
||||
|
||||
const CONCEPT = fs.readFileSync(path.join(__dirname, '../../docs/concepts/workspace/app.js'), 'utf8');
|
||||
const SOURCE = fs.readFileSync(path.join(__dirname, '../../web/js/workspace/system-diagnostics.mjs'), 'utf8');
|
||||
const SYSTEM = fs.readFileSync(path.join(__dirname, '../../web/js/workspace/system.mjs'), 'utf8');
|
||||
|
||||
const REPORT = {
|
||||
checks: { database: { status: 'ok', prompt_count: 4210, has_images_table: true, image_count: 19956 } },
|
||||
database_info: { database_path: 'user/__promptmanager/prompts.db', database_size_bytes: 1717986918 },
|
||||
sources: { configured: 2, effective: 2, watching: 2, unavailable: 0 },
|
||||
viewerjs: '1.15.2',
|
||||
};
|
||||
|
||||
test('the section keeps the concept surfaces, terms, rows and labels; no sample copy ships', () => {
|
||||
const start = CONCEPT.indexOf('case "diagnostics":');
|
||||
const body = CONCEPT.slice(start, CONCEPT.indexOf('break;', start));
|
||||
const terms = [...body.matchAll(/<dt>([^<]+)<\/dt>/g)].map(([, term]) => term);
|
||||
assert.deepEqual(terms, ['Database', 'Sources', 'Viewer.js', 'Host renderers']);
|
||||
for (const text of ["'Diagnostics'", "'Logs'", "'Run checks'", "'Refresh'", "'Level'", "'Log files & retention'",
|
||||
"'Review log cleanup'", "'Runtime log level'", "'Console logging'", "'Image-link diagnostics'"]) {
|
||||
assert.ok(SOURCE.includes(text), text);
|
||||
}
|
||||
for (const id of ['runtime-level', 'console-log']) assert.ok(body.includes(`setting("${id}"`) && SOURCE.includes(`id: '${id}'`), id);
|
||||
assert.ok(SOURCE.includes("'toolbar spaced-16'") && body.includes('class="toolbar spaced-16"'));
|
||||
// Prototype copy is never shipped.
|
||||
for (const sample of ['sample', 'Sample']) assert.ok(!SOURCE.includes(`'${sample}`) && !/Download sample|Inspect sample/.test(SOURCE), sample);
|
||||
});
|
||||
|
||||
test('D94 is retired: Diagnostics & logs is a delivered section and nothing hands off', () => {
|
||||
assert.ok(SYSTEM.includes('diagnostics: mountDiagnostics'));
|
||||
assert.ok(!SYSTEM.includes('SECTION_HANDOFFS'));
|
||||
assert.ok(!SYSTEM.includes('handoff('));
|
||||
assert.ok(!SYSTEM.includes('NOT_DELIVERED'));
|
||||
});
|
||||
|
||||
test('database text covers readable, not readable, busy and loading', async () => {
|
||||
const { databaseText, LOADING, UNREADABLE } = await diagnostics;
|
||||
assert.equal(databaseText({ report: REPORT }), 'Readable · 1.6 GB · 19,956 images');
|
||||
assert.equal(databaseText({ report: { ...REPORT, checks: { database: { status: 'ok', image_count: 1 } },
|
||||
database_info: { database_size_bytes: 2048 } } }), 'Readable · 2.0 KB · 1 image');
|
||||
assert.equal(databaseText({ report: { checks: { database: { status: 'error', message: 'Database error: locked' } } } }),
|
||||
'Not readable: Database error: locked');
|
||||
assert.equal(databaseText({ loading: true }), LOADING);
|
||||
assert.equal(LOADING, 'Loading…');
|
||||
assert.equal(databaseText({ error: { code: 'busy', running: { id: 'j1', title: 'Restore' } } }), 'Busy: Restore is running.');
|
||||
assert.equal(databaseText({ error: { code: 'busy' } }), 'Busy: another job is running.');
|
||||
assert.equal(databaseText({ error: new Error('offline') }), UNREADABLE);
|
||||
assert.equal(UNREADABLE, 'Could not be read.');
|
||||
});
|
||||
|
||||
test('sources text follows the precedence table', async () => {
|
||||
const { sourcesText } = await diagnostics;
|
||||
const of = sources => sourcesText({ report: { sources } });
|
||||
assert.equal(of({ configured: 0, effective: 1, watching: 1, unavailable: 0 }), 'ComfyUI output folder · watching');
|
||||
assert.equal(of({ configured: 0, effective: 1, watching: 0, unavailable: 0 }), 'ComfyUI output folder · manual scan');
|
||||
// Unavailable wins over watching.
|
||||
assert.equal(of({ configured: 3, effective: 2, watching: 2, unavailable: 1 }), '3 sources · 1 not available');
|
||||
assert.equal(of({ configured: 2, effective: 2, watching: 2, unavailable: 0 }), '2 sources · 2 watching');
|
||||
assert.equal(of({ configured: 1, effective: 1, watching: 0, unavailable: 0 }), '1 source · 0 watching');
|
||||
assert.equal(of(undefined), 'ComfyUI output folder · manual scan');
|
||||
});
|
||||
|
||||
test('Viewer.js reads the pinned vendor version', async () => {
|
||||
const { viewerText } = await diagnostics;
|
||||
assert.equal(viewerText({ report: REPORT }), '1.15.2 · pinned vendor');
|
||||
assert.equal(viewerText({ report: {} }), 'unknown · pinned vendor');
|
||||
});
|
||||
|
||||
test('report filename uses local time', async () => {
|
||||
const { reportFilename } = await diagnostics;
|
||||
assert.equal(reportFilename(new Date(2026, 9, 6, 7, 5)), 'prompt-manager-diagnostics-20261006-0705.json');
|
||||
assert.equal(reportFilename(new Date(2026, 11, 31, 23, 59)), 'prompt-manager-diagnostics-20261231-2359.json');
|
||||
});
|
||||
|
||||
test('merged report adds only host_renderers', async () => {
|
||||
const { mergeReport } = await diagnostics;
|
||||
const merged = mergeReport(REPORT, 'Detected: Classic (1 ComfyUI window).');
|
||||
assert.deepEqual(Object.keys(merged).sort(), [...Object.keys(REPORT), 'host_renderers'].sort());
|
||||
assert.equal(merged.host_renderers, 'Detected: Classic (1 ComfyUI window).');
|
||||
assert.equal(mergeReport(REPORT, null).host_renderers, null);
|
||||
assert.equal(mergeReport(REPORT, undefined).host_renderers, null);
|
||||
// The server's fields are not changed.
|
||||
assert.equal(merged.sources, REPORT.sources);
|
||||
});
|
||||
|
||||
test('sizes read as the server labels them (1 KB = 1024 bytes)', async () => {
|
||||
const { sizeText } = await diagnostics;
|
||||
assert.equal(sizeText(0), '0 bytes');
|
||||
assert.equal(sizeText(1), '1 byte');
|
||||
assert.equal(sizeText(1000), '1,000 bytes');
|
||||
assert.equal(sizeText(2048), '2.0 KB');
|
||||
assert.equal(sizeText(10 * 2 ** 20), '10.0 MB');
|
||||
assert.equal(sizeText(1717986918), '1.6 GB');
|
||||
assert.equal(sizeText(undefined), '0 bytes');
|
||||
});
|
||||
|
||||
test("log line uses the concept's format", async () => {
|
||||
const { logLine } = await diagnostics;
|
||||
assert.equal(logLine({ timestamp: '2026-10-26T09:42:18.123456', level: 'INFO', message: 'Database backup verified' }),
|
||||
'09:42:18 INFO Database backup verified');
|
||||
assert.equal(logLine({ timestamp: '2026-10-26T09:42:18', level: 'WARNING', message: '3 images have ambiguous prompt metadata' }),
|
||||
'09:42:18 WARNING 3 images have ambiguous prompt metadata');
|
||||
assert.match(logLine({ timestamp: '2026-10-26T09:42:18', level: 'ERROR', message: 'x' }), /^\d\d:\d\d:\d\d [A-Z]+ +\S/);
|
||||
// The concept's sample lines read the same way.
|
||||
assert.ok(CONCEPT.includes('09:42:18 INFO Scan completed'));
|
||||
});
|
||||
|
||||
test('Lines accepts whole numbers 1 to 500 only', async () => {
|
||||
const { validLimit, logsUrl } = await diagnostics;
|
||||
for (const [value, expected] of [['1', 1], ['500', 500], [' 20 ', 20], ['100', 100]]) assert.equal(validLimit(value), expected, value);
|
||||
for (const value of ['0', '501', '2.5', '', ' ', '-1', '1e2', 'abc', undefined]) assert.equal(validLimit(value), null, String(value));
|
||||
assert.equal(logsUrl('', 20), '/prompt_manager/logs?limit=20');
|
||||
assert.equal(logsUrl('WARNING', 20), '/prompt_manager/logs?limit=20&level=WARNING');
|
||||
});
|
||||
|
||||
test('the level filter offers all six minimum levels; the runtime level adds CRITICAL after the concept four', async () => {
|
||||
const { LEVEL_OPTIONS, RUNTIME_LEVELS } = await diagnostics;
|
||||
assert.deepEqual(LEVEL_OPTIONS.map(([, label]) => label), ['All levels', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']);
|
||||
assert.equal(LEVEL_OPTIONS[0][0], '');
|
||||
assert.deepEqual(RUNTIME_LEVELS, ['INFO', 'WARNING', 'DEBUG', 'ERROR', 'CRITICAL']);
|
||||
assert.ok(CONCEPT.includes('["INFO", "WARNING", "DEBUG", "ERROR"]'));
|
||||
});
|
||||
|
||||
test('statistics text uses singular forms for one', async () => {
|
||||
const { statsText } = await diagnostics;
|
||||
assert.equal(statsText({ buffer_count: 1000, level_counts: { INFO: 980, WARNING: 12, ERROR: 2, CRITICAL: 1 },
|
||||
log_files_count: 5, total_log_size: 43201024 }), '1,000 entries in memory · 3 errors · 12 warnings · 5 files · 41.2 MB on disk');
|
||||
assert.equal(statsText({ buffer_count: 1, level_counts: { WARNING: 1, CRITICAL: 1 }, log_files_count: 1, total_log_size: 1 }),
|
||||
'1 entry in memory · 1 error · 1 warning · 1 file · 1 byte on disk');
|
||||
assert.equal(statsText({}), '0 entries in memory · 0 errors · 0 warnings · 0 files · 0 bytes on disk');
|
||||
});
|
||||
|
||||
test('retention, file line and file rows read the server values', async () => {
|
||||
const { retentionText, fileLine, fileDetail } = await diagnostics;
|
||||
assert.equal(retentionText({ max_file_size: 10485760, backup_count: 5, file_logging: true, buffer_size: 1000 }),
|
||||
'Rotates at 10 MB · keeps 5 rotated copies · file logging on · memory buffer 1,000 entries');
|
||||
assert.equal(retentionText({ max_file_size: 1572864, backup_count: 1, file_logging: false, buffer_size: 1 }),
|
||||
'Rotates at 1.5 MB · keeps 1 rotated copy · file logging off · memory buffer 1 entry');
|
||||
const files = [
|
||||
{ filename: 'prompt_manager.log', kind: 'active', size: 1468006, modified: '2026-10-26T09:42:18' },
|
||||
{ filename: 'prompt_manager.log.1', kind: 'rotated', size: 10485760, modified: '2026-10-25T14:02:00' },
|
||||
{ filename: 'prompt_manager.log.4', kind: 'link', size: 20, modified: '2026-10-25T14:02:00' },
|
||||
{ filename: 'prompt_manager.log.bak', kind: 'other', size: 0, modified: '2026-10-24T08:00:00' },
|
||||
];
|
||||
assert.equal(fileLine(files), 'prompt_manager.log · 1.4 MB · rotated copies: 1');
|
||||
assert.equal(fileLine([]), 'prompt_manager.log · not written yet · rotated copies: 0');
|
||||
assert.equal(fileDetail(files[1]), '10.0 MB · 2026-10-25 14:02 · Rotated copy');
|
||||
assert.equal(fileDetail(files[2]), '20 bytes · 2026-10-25 14:02 · Link, not downloaded');
|
||||
assert.equal(fileDetail(files[3]), '0 bytes · 2026-10-24 08:00 · Other file');
|
||||
assert.equal(fileDetail(files[0]), '1.4 MB · 2026-10-26 09:42 · Active log');
|
||||
});
|
||||
|
||||
test('Clear all logs states both counts and its toast says what was done', async () => {
|
||||
const { clearSentence, clearToast } = await diagnostics;
|
||||
const rotated = size => ({ kind: 'rotated', size });
|
||||
const files = [{ kind: 'active', size: 10 }, rotated(10 * 2 ** 20), rotated(10 * 2 ** 20), rotated(8.4 * 2 ** 20), { kind: 'other', size: 5 }];
|
||||
assert.equal(clearSentence(files), 'Empties prompt_manager.log, deletes 3 rotated copies (28.4 MB) and clears the lines shown '
|
||||
+ 'here. The other files in the log folder are kept. This cannot be undone.');
|
||||
assert.equal(clearSentence([{ kind: 'active', size: 1 }]), 'Empties prompt_manager.log and clears the lines shown here. '
|
||||
+ 'The other files in the log folder are kept. This cannot be undone.');
|
||||
assert.match(clearSentence(null), /^Empties prompt_manager.log, deletes its rotated copies and clears/);
|
||||
assert.equal(clearToast({ truncated: ['prompt_manager.log', 'prompt_manager.log.1', 'prompt_manager.log.2', 'prompt_manager.log.3'],
|
||||
errors: [] }), 'Logs cleared · 3 rotated copies deleted.');
|
||||
assert.equal(clearToast({ truncated: ['prompt_manager.log', 'prompt_manager.log.1', 'prompt_manager.log.2'],
|
||||
errors: ['Failed to truncate prompt_manager.log.3: in use'] }), 'Logs cleared · 2 rotated copies deleted · 1 could not be deleted.');
|
||||
assert.equal(clearToast({ truncated: ['prompt_manager.log', 'prompt_manager.log.1'], errors: [] }), 'Logs cleared · 1 rotated copy deleted.');
|
||||
assert.equal(clearToast(undefined), 'Logs cleared · 0 rotated copies deleted.');
|
||||
});
|
||||
|
||||
test('Clear all logs sends back exactly the files its confirmation showed (#389)', async () => {
|
||||
const { clearBody, clearFailureText, COPY } = await diagnostics;
|
||||
const files = [
|
||||
{ filename: 'prompt_manager.log', kind: 'active', identity: '10:1761000000123456789:9007199254740993' },
|
||||
{ filename: 'prompt_manager.log.1', kind: 'rotated', identity: '20:1761000000123456789:42' },
|
||||
{ filename: 'prompt_manager.log.4', kind: 'link', identity: '5:1:7' },
|
||||
{ filename: 'prompt_manager.log.bak', kind: 'other', identity: '1:1:8' },
|
||||
{ filename: 'prompt_manager.log.2', kind: 'rotated' },
|
||||
];
|
||||
// Identities stay strings: mtime_ns and large inodes are not exact JS numbers.
|
||||
assert.deepEqual(clearBody(files), { expected: [
|
||||
{ name: 'prompt_manager.log', identity: '10:1761000000123456789:9007199254740993' },
|
||||
{ name: 'prompt_manager.log.1', identity: '20:1761000000123456789:42' },
|
||||
] });
|
||||
assert.deepEqual(clearBody([]), { expected: [] });
|
||||
assert.deepEqual(clearBody(null), {});
|
||||
const refused = Object.assign(new Error('The log files changed since you confirmed; nothing was cleared.'), { status: 409 });
|
||||
assert.equal(clearFailureText(refused), refused.message);
|
||||
assert.equal(clearFailureText(Object.assign(new Error('boom'), { status: 500 })), COPY.clearFailed);
|
||||
assert.equal(clearFailureText(undefined), COPY.clearFailed);
|
||||
});
|
||||
|
||||
test('the test-link dialog names the prompt and how Missing-file links removes the record', async () => {
|
||||
const { testLinkSentence, promptName } = await diagnostics;
|
||||
assert.equal(testLinkSentence({ id: 812, text: 'Misty harbor' }), 'Writes a placeholder image record '
|
||||
+ '(prompt_manager_test_link.png) linked to prompt #812 “Misty harbor”. Maintenance › Missing-file links removes it later.');
|
||||
assert.equal(promptName({ text: ' a\n b ' }), 'a b');
|
||||
assert.equal(promptName({ text: 'x'.repeat(80) }).length, 60);
|
||||
});
|
||||
|
||||
test('runtime values and body map config.level to runtime_level and back', async () => {
|
||||
const { runtimeValues, runtimeBody } = await diagnostics;
|
||||
assert.deepEqual(runtimeValues({ level: 'INFO', console_logging: true, max_file_size: 1 }), { runtime_level: 'INFO', console_logging: true });
|
||||
assert.deepEqual(runtimeBody({ runtime_level: 'CRITICAL' }), { level: 'CRITICAL' });
|
||||
assert.deepEqual(runtimeBody({ console_logging: false }), { console_logging: false });
|
||||
assert.deepEqual(runtimeBody({ runtime_level: 'DEBUG', console_logging: true }), { level: 'DEBUG', console_logging: true });
|
||||
assert.deepEqual(runtimeBody({}), {});
|
||||
});
|
||||
@@ -169,3 +169,44 @@ test('a plan with nothing to do offers no Apply (S7 live test)', async () => {
|
||||
assert.equal(hasWork({ ...PLAN, scope: {} }), true);
|
||||
assert.equal(NOTHING_TO_APPLY, 'There is nothing to apply; nothing was changed.');
|
||||
});
|
||||
|
||||
// S12 (#329): the rotated-log cleanup plan changes no library record: its Backup reads "Not needed", the review shows no
|
||||
// D55 backup box, and its details are files.
|
||||
const LOG_PLAN = { operation: 'rotated_logs', token: 't', expires_at: '2026-10-26T23:15:00',
|
||||
scope: { count: 3, label: '3 rotated log copies · 28.4 MB' },
|
||||
protected: { count: 3, reasons: ['Kept: the active log', 'Kept: 1 link', 'Kept: 1 other file'] },
|
||||
backup: { policy: 'none', destination_label: null, expected_bytes: null },
|
||||
media: { deletes_files: true, label: 'Deletes only the listed log files. No image, video or library record changes.' },
|
||||
details: [{ label: 'prompt_manager.log.1', detail: '10.0 MB · 2026-10-25 14:02' }] };
|
||||
|
||||
test('a plan without a database change reads Not needed and shows no backup box', async () => {
|
||||
const { planGrid, backupText, backupBox, BACKUP_COPY, OPERATION_TITLES } = await review;
|
||||
assert.equal(backupText({ policy: 'none' }), 'Not needed: no library record changes');
|
||||
assert.deepEqual(planGrid(LOG_PLAN), [
|
||||
['Scope', '3 rotated log copies · 28.4 MB'],
|
||||
['Protected', 'Kept: the active log · Kept: 1 link · Kept: 1 other file'],
|
||||
['Backup', 'Not needed: no library record changes'],
|
||||
['Media on disk', 'Deletes only the listed log files. No image, video or library record changes.'],
|
||||
]);
|
||||
const onBackup = () => {};
|
||||
assert.equal(backupBox(LOG_PLAN, onBackup), null);
|
||||
// Every database plan keeps its D55 box.
|
||||
assert.deepEqual(backupBox({ ...LOG_PLAN, operation: 'optimize', backup: { policy: 'required' } }, onBackup), { copy: BACKUP_COPY, onBackup });
|
||||
assert.deepEqual(backupBox({ operation: 'missing_links' }, onBackup), { copy: BACKUP_COPY, onBackup });
|
||||
// Its title heads the dialog and lets Activity offer Review again.
|
||||
assert.equal(OPERATION_TITLES.rotated_logs, 'Rotated log copies');
|
||||
});
|
||||
|
||||
test('rotated_logs details read Listed files', async () => {
|
||||
const { detailsSummary, detailLines } = await review;
|
||||
assert.equal(detailsSummary(LOG_PLAN, 3), 'Listed files (3)');
|
||||
assert.equal(detailsSummary({ ...LOG_PLAN, scope: { count: 60 } }, 50), 'Listed files (50 of 60)');
|
||||
assert.equal(detailsSummary({ operation: 'missing_links', scope: { count: 2 } }, 2), 'Listed records (2)');
|
||||
assert.deepEqual(detailLines(LOG_PLAN), ['prompt_manager.log.1 · 10.0 MB · 2026-10-25 14:02']);
|
||||
});
|
||||
|
||||
test('a rotated-log cleanup the log overtook offers Review again (S12)', async () => {
|
||||
const { reviewAgainOf } = await jobView;
|
||||
assert.deepEqual(reviewAgainOf({ kind: 'maintenance', state: 'failed', scope: { operation: 'rotated_logs', failure: 'plan-changed' } }),
|
||||
{ operation: 'rotated_logs' });
|
||||
});
|
||||
|
||||
@@ -420,3 +420,12 @@ test('a partial host save keeps what saved, adopts its revision and fails only t
|
||||
assert.equal((await drafts.save()).host, 'failed');
|
||||
assert.equal(drafts.statusText(), SETTINGS_STATUS.partial);
|
||||
});
|
||||
|
||||
// S12 (#329): System's runtime domain is the running server's log level and console setting (`/logs/config`).
|
||||
test('runtime adapter maps config.level to runtime_level and back', async () => {
|
||||
const { runtimeValues, runtimeBody } = await import('../../web/js/workspace/system-diagnostics.mjs');
|
||||
assert.deepEqual(runtimeValues({ level: 'WARNING', console_logging: false, backup_count: 5 }),
|
||||
{ runtime_level: 'WARNING', console_logging: false });
|
||||
assert.deepEqual(runtimeBody({ runtime_level: 'WARNING' }), { level: 'WARNING' });
|
||||
assert.deepEqual(runtimeBody({ runtime_level: 'CRITICAL', console_logging: true }), { level: 'CRITICAL', console_logging: true });
|
||||
});
|
||||
|
||||
@@ -307,6 +307,16 @@ test('Preview tagging review opens the Tag image dialog with the section’s val
|
||||
assert.deepEqual(draftProblem({ ...HOST, autotag_general_threshold: 1.5 }),
|
||||
{ key: 'autotag_general_threshold', message: 'General-tag confidence must be between 0 and 1.' });
|
||||
assert.equal(draftProblem({ ...HOST, autotag_character_threshold: '' }).key, 'autotag_character_threshold');
|
||||
// #369: every page shows the thresholds in steps of 0.05, so a value off that grid cannot be saved or previewed.
|
||||
assert.deepEqual(draftProblem({ ...HOST, autotag_general_threshold: 0.37 }),
|
||||
{ key: 'autotag_general_threshold', message: 'General-tag confidence must be a multiple of 0.05.' });
|
||||
assert.deepEqual(draftProblem({ ...HOST, autotag_character_threshold: 0.33 }),
|
||||
{ key: 'autotag_character_threshold', message: 'Character-tag confidence must be a multiple of 0.05.' });
|
||||
assert.equal(draftProblem({ ...HOST, autotag_general_threshold: 0.15, autotag_character_threshold: 0.95 }), null);
|
||||
assert.equal(draftProblem({ ...HOST, autotag_general_threshold: 0, autotag_character_threshold: 1 }), null);
|
||||
// A computed value a hair off the grid passes and is handed on snapped to it, as the server stores it.
|
||||
assert.equal(draftProblem({ ...HOST, autotag_general_threshold: 0.1 + 0.2 }), null);
|
||||
assert.equal(taggingPreset({ ...HOST, autotag_general_threshold: 0.1 + 0.2 }).options.general_threshold, 0.3);
|
||||
assert.equal(draftProblem({ ...HOST, autotag_suggestion_limit: 2.5 }).key, 'autotag_suggestion_limit');
|
||||
assert.equal(draftProblem({ ...HOST, autotag_suggestion_limit: 0 }, { limit: false }), null);
|
||||
assert.deepEqual(draftProblem({ ...HOST, autotag_model_type: 'hf', autotag_template: ' ' }),
|
||||
@@ -342,25 +352,36 @@ test('Preview tagging review opens the Tag image dialog with the section’s val
|
||||
await button('Preview tagging review').dispatch('click');
|
||||
assert.deepEqual(calls.toasts, ['General-tag confidence must be between 0 and 1.']);
|
||||
assert.equal(calls.opened.length, 1);
|
||||
byId('threshold-general').value = '0.37'; await byId('threshold-general').dispatch('change');
|
||||
assert.equal(button('Preview tagging review').getAttribute('aria-disabled'), 'true');
|
||||
await button('Preview tagging review').dispatch('click');
|
||||
assert.deepEqual(calls.toasts.at(-1), 'General-tag confidence must be a multiple of 0.05.');
|
||||
assert.equal(calls.opened.length, 1);
|
||||
} finally { scope.dispose(); dom.restore(); }
|
||||
});
|
||||
|
||||
const PLAN = { images: 3, prompts: 2, already_tagged: 1, to_tag: 1,
|
||||
const PLAN = { images: 3, already_tagged: 1, to_tag: 2, estimate_seconds: 3.2, estimate_device: 'cpu', suggestion_limit: 10,
|
||||
model: { type: 'wd14-swinv2', label: 'WD14 SwinV2', size_label: '~400 MB', downloaded: true }, existing: 'skip',
|
||||
review_required: false, backup: { policy: 'on' }, disk: { free_bytes: 10 * 2 ** 30, required_bytes: 2 ** 20, sufficient: true } };
|
||||
|
||||
test('the batch review grid, body and Cancel-only reasons (D134, CUJ 4–5)', async () => {
|
||||
const { batchGrid, batchBody, batchCloseNote, existingLabel, REPLACE_NOTE, NOTHING_TO_TAG, NO_LINKED_IMAGES } = await tagging;
|
||||
test('the batch review grid, body and Cancel-only reasons (D134, #379, CUJ 4–5)', async () => {
|
||||
const { batchGrid, batchBody, batchCloseNote, existingLabel, REPLACE_NOTE, NOTHING_TO_TAG, NO_IMAGES } = await tagging;
|
||||
assert.deepEqual(batchGrid(PLAN, HOST), [['Model', 'WD14 SwinV2 · ~400 MB'], ['Existing tags', 'Skip already tagged'],
|
||||
['Images linked', '3'], ['Prompts', '2'], ['Already tagged', '1'], ['Prompts to tag', 'up to 1'],
|
||||
['Settings', 'General 0.35 · Character 0.85'], ['Backup', 'Create and verify before changes']]);
|
||||
['Images', '3'], ['Already auto-tagged', '1'], ['Images to tag', 'about 2'],
|
||||
['Time', 'about 1 min on a processor; a graphics card can be faster.'],
|
||||
['Settings', 'General 0.35 · Character 0.85 · up to 10 tags per image'], ['Backup', 'Create and verify before changes']]);
|
||||
const llm = { ...HOST, autotag_model_type: 'gguf' };
|
||||
assert.deepEqual(batchGrid({ ...PLAN, model: { type: 'gguf' }, backup: { policy: 'off' } }, llm).slice(-2),
|
||||
[['Settings', 'Template of 27 characters'], ['Backup', 'No safety backup (the policy is off)']]);
|
||||
assert.deepEqual(batchGrid({ ...PLAN, model: { type: 'gguf' }, backup: { policy: 'off' }, estimate_seconds: null,
|
||||
estimate_device: null, suggestion_limit: 4 }, llm).slice(-3),
|
||||
[['Time', 'Language-model engines take longer per image. Activity shows the time left once tagging starts.'],
|
||||
['Settings', 'Template of 27 characters · up to 4 tags per image'], ['Backup', 'No safety backup (the policy is off)']]);
|
||||
assert.equal(batchGrid({}, HOST)[2][1], '0');
|
||||
assert.equal(batchGrid({}, HOST)[6][1], 'General 0.35 · Character 0.85');
|
||||
assert.equal(existingLabel('odd'), 'odd');
|
||||
assert.deepEqual(batchBody(HOST), { model_type: 'wd14-swinv2', existing: 'skip',
|
||||
options: { general_threshold: 0.35, character_threshold: 0.85 } });
|
||||
// #369: a threshold a hair off the grid is sent snapped to it.
|
||||
assert.equal(batchBody({ ...HOST, autotag_character_threshold: 0.1 + 0.2 }).options.character_threshold, 0.3);
|
||||
assert.deepEqual(batchBody(llm), { model_type: 'gguf', existing: 'skip', options: { instructions: HOST.autotag_template } });
|
||||
assert.equal(batchCloseNote(PLAN), null);
|
||||
assert.equal(batchCloseNote({ ...PLAN, review_required: true }), REPLACE_NOTE);
|
||||
@@ -368,9 +389,31 @@ test('the batch review grid, body and Cancel-only reasons (D134, CUJ 4–5)', as
|
||||
assert.equal(batchCloseNote({ ...PLAN, model: { ...PLAN.model, downloaded: false } }), 'WD14 SwinV2 is not downloaded. Download it first.');
|
||||
assert.equal(batchCloseNote({ ...PLAN, disk: { free_bytes: 2 ** 20, required_bytes: 2 ** 21, sufficient: false } }),
|
||||
'Not enough free disk space: the safety backup needs about 2.0 MB and 1.0 MB is free. Nothing is written until there is room.');
|
||||
assert.equal(batchCloseNote({ ...PLAN, prompts: 0, to_tag: 0 }), NO_LINKED_IMAGES);
|
||||
assert.equal(batchCloseNote({ ...PLAN, images: 0, to_tag: 0 }), NO_IMAGES);
|
||||
assert.equal(batchCloseNote({ ...PLAN, images: undefined }), NO_IMAGES);
|
||||
assert.equal(NO_IMAGES, 'The output folders contain no images to tag.');
|
||||
assert.equal(batchCloseNote({ ...PLAN, to_tag: 0 }), NOTHING_TO_TAG);
|
||||
assert.equal(NOTHING_TO_TAG, 'Every prompt with an output image is already tagged.');
|
||||
assert.equal(NOTHING_TO_TAG, 'Every output image is already auto-tagged.');
|
||||
});
|
||||
|
||||
test('the review Time row (DEC11, #379): minutes, hours and the engine it runs on', async () => {
|
||||
const { durationText, timeText, TIME_UNKNOWN } = await tagging;
|
||||
assert.equal(durationText(1), 'about 1 min');
|
||||
assert.equal(durationText(0), 'about 1 min');
|
||||
assert.equal(durationText(89), 'about 89 min');
|
||||
assert.equal(durationText(90), 'about 1 h 30 min');
|
||||
assert.equal(durationText(120), 'about 2 h');
|
||||
assert.equal(durationText(631), 'about 10 h 31 min');
|
||||
assert.equal(durationText(Number.NaN), 'about 1 min');
|
||||
// 23,645 images at 1.6 s on a processor (the design's CUJ 1).
|
||||
assert.equal(timeText({ estimate_seconds: 37832, estimate_device: 'cpu' }),
|
||||
'about 10 h 31 min on a processor; a graphics card can be faster.');
|
||||
assert.equal(timeText({ estimate_seconds: 59, estimate_device: 'gpu' }), 'about 1 min on the graphics card.');
|
||||
assert.equal(timeText({ estimate_seconds: 0, estimate_device: 'cpu' }), 'about 1 min on a processor; a graphics card can be faster.');
|
||||
for (const plan of [{ estimate_seconds: null }, {}, { estimate_seconds: -1 }, { estimate_seconds: Infinity }, null]) {
|
||||
assert.equal(timeText(plan), TIME_UNKNOWN);
|
||||
}
|
||||
assert.equal(TIME_UNKNOWN, 'Language-model engines take longer per image. Activity shows the time left once tagging starts.');
|
||||
});
|
||||
|
||||
test('showBatchPlan opens the review with the backup box, or the Cancel-only review', async () => {
|
||||
|
||||
@@ -26,6 +26,8 @@ VECTORS_PATH = (
|
||||
/ "local-inspect"
|
||||
/ "resolver-vectors.json"
|
||||
)
|
||||
# A case may name the inspected file (#381 saver anchoring); it is no record.
|
||||
FILE_NAME = "__file_name"
|
||||
# Committed record sets stay small; the large bound cases are built at test time.
|
||||
MAX_VECTOR_CHARS = 20_000
|
||||
COMMITTED_FUZZ = (0, 60)
|
||||
@@ -906,6 +908,60 @@ def golden_cases():
|
||||
},
|
||||
),
|
||||
)
|
||||
# #381: unknown nodes between the prompt and the sampler pass through.
|
||||
try:
|
||||
from tests import unknown_node_graphs
|
||||
except ImportError: # pragma: no cover - discover -s tests puts tests/ first
|
||||
import unknown_node_graphs
|
||||
for name, graph in unknown_node_graphs.shapes().items():
|
||||
add(f"pass-through-{name}", graph)
|
||||
for name, records in {
|
||||
**unknown_node_graphs.recorded_shapes(),
|
||||
**unknown_node_graphs.review_shapes(),
|
||||
}.items():
|
||||
cases[f"pass-through-{name}"] = records
|
||||
try:
|
||||
from tests import saver_anchor_graphs as anchor
|
||||
except ImportError: # pragma: no cover - discover -s tests puts tests/ first
|
||||
import saver_anchor_graphs as anchor
|
||||
for name, graph, file_name in [
|
||||
("branch-3", anchor.branch_bench(), "s45_branch_3_00002_.png"),
|
||||
("branch-unanchored", anchor.branch_bench(), None),
|
||||
("branch-no-match", anchor.branch_bench(), "photo_00001_.png"),
|
||||
("branch-linked", anchor.linked_prefix_bench(), "s45_branch_2_00007_.png"),
|
||||
(
|
||||
"branch-unknown-saver",
|
||||
anchor.branch_bench(saver="VendorSaveImage"),
|
||||
"s45_branch_4_00001.webp",
|
||||
),
|
||||
("shared-prefix", anchor.shared_prefix(), "IMG_GEN_00371_.png"),
|
||||
("same-links", anchor.two_stage_same_conditioning(), "final_00003_.png"),
|
||||
("same-links-unanchored", anchor.two_stage_same_conditioning(), None),
|
||||
(
|
||||
"other-links",
|
||||
anchor.two_stage_same_conditioning(["91", 0]),
|
||||
"final_00003_.png",
|
||||
),
|
||||
("hidden-earlier", anchor.hidden_earlier_prompt(), "out_00001_.png"),
|
||||
(
|
||||
"hidden-override",
|
||||
anchor.hidden_earlier_prompt(override=True),
|
||||
"out_00001_.png",
|
||||
),
|
||||
] + [
|
||||
(f"first-stage-{kind}", anchor.first_stage_variant(kind), "final_00003_.png")
|
||||
for kind in (
|
||||
"no-latent",
|
||||
"literal-latent",
|
||||
"missing-latent",
|
||||
"cyclic-latent",
|
||||
"stage-shaped",
|
||||
"detailer",
|
||||
)
|
||||
]:
|
||||
cases[f"anchor-{name}"] = {"prompt": _dump(graph)}
|
||||
if file_name is not None:
|
||||
cases[f"anchor-{name}"][FILE_NAME] = file_name
|
||||
cases["parameters-only"] = {
|
||||
"parameters": "a fox\nNegative prompt: blur\nSteps: 20, Sampler: euler"
|
||||
}
|
||||
@@ -1059,7 +1115,8 @@ def boundary_cases():
|
||||
["7", 0],
|
||||
{
|
||||
"\U0001f600"
|
||||
* 70: {"class_type": long_name, "inputs": {"text": "x"}},
|
||||
# No text and no links: still named, not passed through (#381).
|
||||
* 70: {"class_type": long_name, "inputs": {"seed": 1}},
|
||||
"7": _encode("b"),
|
||||
},
|
||||
)
|
||||
@@ -1387,7 +1444,10 @@ def committed_vectors():
|
||||
"name": name,
|
||||
"records": records,
|
||||
"evidence": inspect_prompt_evidence(
|
||||
records, None, profile=INSPECTION_PROFILE
|
||||
{key: value for key, value in records.items() if key != FILE_NAME},
|
||||
None,
|
||||
profile=INSPECTION_PROFILE,
|
||||
file_name=records.get(FILE_NAME),
|
||||
),
|
||||
}
|
||||
for name, records in cases.items()
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
"""A temp SQLite database in a folder no other test process shares (#346).
|
||||
|
||||
``PromptManagerAPI`` sweeps its database's folder at startup for the stages,
|
||||
uploads and safety-backup copies an interrupted restore or backup left. A
|
||||
fixture database directly in the system temp folder made every other test
|
||||
process that started an API (another worktree's suite, a pre-push hook) delete
|
||||
this process's in-flight restore files, and left safety backups behind there.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
|
||||
|
||||
def private_temp_db(case):
|
||||
"""Return a closed ``NamedTemporaryFile`` handle for an empty ``.db`` file.
|
||||
|
||||
The file lives in a new folder that ``case`` removes at cleanup, after its
|
||||
own teardown has closed and unlinked the database (cleanups run after
|
||||
``tearDown``), together with anything a restore or backup left next to it.
|
||||
"""
|
||||
folder = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
|
||||
case.addCleanup(folder.cleanup)
|
||||
handle = tempfile.NamedTemporaryFile(delete=False, suffix=".db", dir=folder.name)
|
||||
handle.close()
|
||||
return handle
|
||||
@@ -0,0 +1,216 @@
|
||||
"""Synthetic graphs for saver anchoring and shared-conditioning stages (#381).
|
||||
|
||||
``branch_bench`` reproduces the maintainer's ``gpu_unlock_bench`` run: four
|
||||
KSampler branches with different prompts, each saved by its own SaveImage whose
|
||||
literal prefix names the branch (``bench/run/s45_branch_N``). ``shared_prefix``
|
||||
is the dataset run's shape: every branch's saver uses one prefix, so the file
|
||||
name cannot name the branch. ``two_stage_same_conditioning`` reproduces the
|
||||
ANIMA run: a hires stage whose latent passes an unlisted node and reads the
|
||||
same positive and negative links as the first stage. Texts are neutral.
|
||||
"""
|
||||
|
||||
BRANCH_TEXTS = (
|
||||
"a knit beanie on a wooden table",
|
||||
"an old fishing boat at dawn",
|
||||
"a red vintage car in the rain",
|
||||
"a cozy library with tall shelves",
|
||||
)
|
||||
NEGATIVE = "blurry, lowres"
|
||||
|
||||
|
||||
def _encode(text):
|
||||
return {"class_type": "CLIPTextEncode", "inputs": {"text": text, "clip": ["1", 1]}}
|
||||
|
||||
|
||||
def _ksampler(positive, negative, latent, denoise=1.0):
|
||||
return {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"seed": 1,
|
||||
"steps": 20,
|
||||
"cfg": 7.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "normal",
|
||||
"denoise": denoise,
|
||||
"model": ["1", 0],
|
||||
"positive": positive,
|
||||
"negative": negative,
|
||||
"latent_image": latent,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _base():
|
||||
return {
|
||||
"1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "m"}},
|
||||
"2": {"class_type": "EmptyLatentImage", "inputs": {"width": 64}},
|
||||
"3": _encode(NEGATIVE),
|
||||
}
|
||||
|
||||
|
||||
def branch_bench(prefixes=None, saver="SaveImage"):
|
||||
"""Four branches; ``prefixes`` replaces each saver's ``filename_prefix``."""
|
||||
prefixes = prefixes or [
|
||||
f"bench/run/s45_branch_{index}" for index in range(1, len(BRANCH_TEXTS) + 1)
|
||||
]
|
||||
graph = _base()
|
||||
for index, text in enumerate(BRANCH_TEXTS, start=1):
|
||||
base = index * 10
|
||||
graph[str(base)] = _encode(text)
|
||||
graph[str(base + 1)] = _ksampler([str(base), 0], ["3", 0], ["2", 0])
|
||||
graph[str(base + 2)] = {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": [str(base + 1), 0], "vae": ["1", 2]},
|
||||
}
|
||||
graph[str(base + 3)] = {
|
||||
"class_type": saver,
|
||||
"inputs": {
|
||||
"filename_prefix": prefixes[index - 1],
|
||||
"images": [str(base + 2), 0],
|
||||
},
|
||||
}
|
||||
return graph
|
||||
|
||||
|
||||
def linked_prefix_bench():
|
||||
"""The branch run with each prefix coming from a PrimitiveString."""
|
||||
graph = branch_bench(prefixes=[["90", 0], ["91", 0], ["92", 0], ["93", 0]])
|
||||
for index in range(4):
|
||||
graph[str(90 + index)] = {
|
||||
"class_type": "PrimitiveString",
|
||||
"inputs": {"value": f"bench/run/s45_branch_{index + 1}"},
|
||||
}
|
||||
return graph
|
||||
|
||||
|
||||
def shared_prefix():
|
||||
"""Every branch saved with one prefix: the file name names no branch."""
|
||||
return branch_bench(prefixes=["dataset/IMG_GEN"] * len(BRANCH_TEXTS))
|
||||
|
||||
|
||||
def two_stage_same_conditioning(second_positive=None):
|
||||
"""Base and hires stage on the same links; the hires latent passes an
|
||||
unlisted node (``VendorGpuCleaner``), so the stages' lineage is unknown."""
|
||||
graph = _base()
|
||||
graph["90"] = _encode(BRANCH_TEXTS[0])
|
||||
graph["63"] = _ksampler(["90", 0], ["3", 0], ["2", 0])
|
||||
graph["30"] = {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["63", 0], "vae": ["1", 2]},
|
||||
}
|
||||
graph["98"] = {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"filename_prefix": "draft", "images": ["30", 0]},
|
||||
}
|
||||
graph["110"] = {"class_type": "VendorGpuCleaner", "inputs": {"anything": ["63", 0]}}
|
||||
graph["108"] = {
|
||||
"class_type": "LatentUpscaleBy",
|
||||
"inputs": {
|
||||
"upscale_method": "nearest-exact",
|
||||
"scale_by": 1.2,
|
||||
"samples": ["110", 0],
|
||||
},
|
||||
}
|
||||
graph["114"] = _ksampler(second_positive or ["90", 0], ["3", 0], ["108", 0], 0.5)
|
||||
graph["113"] = {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["114", 0], "vae": ["1", 2]},
|
||||
}
|
||||
graph["115"] = {
|
||||
"class_type": "VendorGpuCleaner",
|
||||
"inputs": {"anything": ["113", 0]},
|
||||
}
|
||||
graph["107"] = {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"filename_prefix": "final", "images": ["115", 0]},
|
||||
}
|
||||
if second_positive is not None:
|
||||
graph["91"] = _encode(BRANCH_TEXTS[1])
|
||||
return graph
|
||||
|
||||
|
||||
def hidden_earlier_prompt(override=False):
|
||||
"""Review shape: the skipped same-links stage continues a different prompt.
|
||||
|
||||
10 (cat) -> 11 (dog) -> unlisted LatentBlend -> 13 (dog, same links as 11)
|
||||
-> VAEDecode -> SaveImage ``out``. With ``override``, 11 is a
|
||||
``KSampler (Efficient)`` with a linked ``script``.
|
||||
"""
|
||||
graph = _base()
|
||||
graph["5"] = _encode("cat X")
|
||||
graph["6"] = _encode("dog Y")
|
||||
graph["10"] = _ksampler(["5", 0], ["3", 0], ["2", 0])
|
||||
graph["11"] = _ksampler(["6", 0], ["3", 0], ["10", 0])
|
||||
if override:
|
||||
graph["11"] = _ksampler(["6", 0], ["3", 0], ["2", 0])
|
||||
graph["11"]["class_type"] = "KSampler (Efficient)"
|
||||
graph["11"]["inputs"]["script"] = ["40", 0]
|
||||
graph["40"] = {"class_type": "XY Plot", "inputs": {"grid_spacing": 0}}
|
||||
graph["12"] = {
|
||||
"class_type": "LatentBlend",
|
||||
"inputs": {"samples1": ["11", 0], "samples2": ["2", 0], "blend_factor": 0.5},
|
||||
}
|
||||
graph["13"] = _ksampler(["6", 0], ["3", 0], ["12", 0], 0.5)
|
||||
graph["14"] = {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["13", 0], "vae": ["1", 2]},
|
||||
}
|
||||
graph["15"] = {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"filename_prefix": "out", "images": ["14", 0]},
|
||||
}
|
||||
return graph
|
||||
|
||||
|
||||
def deep_prefixes(chain):
|
||||
"""Review shape: two branches whose savers both trace the prefix ``dup``.
|
||||
|
||||
Saver A's prefix runs through ``chain`` DisplayText nodes; saver B's
|
||||
through one more, so a short visit allowance can finish only one trace.
|
||||
"""
|
||||
graph = branch_bench(prefixes=[["p0", 0], ["q", 0], "x_1", "x_2"])
|
||||
graph["pz"] = {"class_type": "StringConstant", "inputs": {"string": "dup"}}
|
||||
previous = "pz"
|
||||
for index in range(chain - 1, -1, -1):
|
||||
graph[f"p{index}"] = {
|
||||
"class_type": "DisplayText",
|
||||
"inputs": {"text": [previous, 0]},
|
||||
}
|
||||
previous = f"p{index}"
|
||||
graph["q"] = {"class_type": "DisplayText", "inputs": {"text": ["p0", 0]}}
|
||||
return graph
|
||||
|
||||
|
||||
def first_stage_variant(kind):
|
||||
"""``two_stage_same_conditioning`` whose first stage (63) is not plain.
|
||||
|
||||
``no-latent`` drops its latent input (still plain); ``literal-latent``,
|
||||
``missing-latent``, ``cyclic-latent``, ``stage-shaped`` and ``detailer``
|
||||
each make it something other than a plain first stage, so the later
|
||||
stage stays unattributed.
|
||||
"""
|
||||
graph = two_stage_same_conditioning()
|
||||
inputs = graph["63"]["inputs"]
|
||||
if kind == "no-latent":
|
||||
del inputs["latent_image"]
|
||||
elif kind == "literal-latent":
|
||||
inputs["latent_image"] = "latent"
|
||||
elif kind == "missing-latent":
|
||||
inputs["latent_image"] = ["999", 0]
|
||||
elif kind == "cyclic-latent":
|
||||
graph["70"] = {"class_type": "VendorLatentOp", "inputs": {"samples": ["63", 0]}}
|
||||
inputs["latent_image"] = ["70", 0]
|
||||
elif kind == "stage-shaped":
|
||||
graph["70"] = {
|
||||
"class_type": "VendorSampler",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"positive": ["90", 0],
|
||||
"latent_image": ["2", 0],
|
||||
},
|
||||
}
|
||||
inputs["latent_image"] = ["70", 0]
|
||||
elif kind == "detailer":
|
||||
graph["63"]["class_type"] = "FaceDetailer"
|
||||
inputs["image"] = ["30", 0]
|
||||
return graph
|
||||
@@ -35,6 +35,8 @@ REVIEW_TABLES = (
|
||||
"media_review_items",
|
||||
"media_review_metadata",
|
||||
)
|
||||
# The auto-tag marks (#379) cite annotation bytes too; an old backup lacks them.
|
||||
CITING_TABLES = REVIEW_TABLES + ("media_image_autotags",)
|
||||
# The exact DDL the plan pins; the module must produce precisely these objects.
|
||||
DDL = {
|
||||
METADATA: (
|
||||
@@ -214,7 +216,7 @@ class AnnotationSchemaTests(unittest.TestCase):
|
||||
"""
|
||||
with closing(sqlite3.connect(path)) as raw, raw:
|
||||
raw.execute("PRAGMA foreign_keys=OFF")
|
||||
for table in REVIEW_TABLES + tuple(reversed(TABLES)):
|
||||
for table in CITING_TABLES + tuple(reversed(TABLES)):
|
||||
raw.execute(f"DROP TABLE {table}")
|
||||
|
||||
def test_fresh_install_and_reopen_is_idempotent(self):
|
||||
|
||||
+157
-7
@@ -430,7 +430,9 @@ class ReplaceImageTagsTests(AnnotationTestCase):
|
||||
self.assertTrue(
|
||||
self.db.refresh_image_fingerprint(loc, DIGEST_A, SIZE, MTIME + 1)
|
||||
)
|
||||
list(self.db.iter_image_tag_records())
|
||||
self.assertEqual(
|
||||
self.db.image_tag_locators(self.db.image_tag_ids(["img"], "")), {loc}
|
||||
)
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["prompt-tag"])
|
||||
|
||||
|
||||
@@ -551,14 +553,162 @@ class CurrentImageTagsTests(AnnotationTestCase):
|
||||
self.assertEqual(current, {locators[0]: ["z", "y"], locators[502]: ["x"]})
|
||||
lookups = [s for s in statements if "media_annotation_files" in s]
|
||||
self.assertEqual(len(lookups), 2)
|
||||
# The tag search finds every record's file by tag id, and the empty
|
||||
# set's file never.
|
||||
self.assertEqual(
|
||||
list(self.db.iter_image_tag_records()),
|
||||
[
|
||||
(locators[0], SIZE, MTIME, ["z", "y"]),
|
||||
(locators[250], SIZE, MTIME + 1, []),
|
||||
(locators[502], SIZE, MTIME + 2, ["x"]),
|
||||
],
|
||||
self.db.image_tag_locators(self.db.image_tag_ids(["z", "x"], "")),
|
||||
{locators[0], locators[502]},
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({locators[250]: (SIZE, MTIME + 1)}),
|
||||
{locators[250]: []},
|
||||
)
|
||||
|
||||
def test_tag_locators_cover_more_ids_than_one_chunk(self):
|
||||
# A broad text search can match more than 500 names: every id's file
|
||||
# comes back across the chunks.
|
||||
locators = [self.locator(f"t{i:04}.png") for i in range(503)]
|
||||
for i, loc in enumerate(locators):
|
||||
self.replace(loc, tags=[f"name_{i:04}"])
|
||||
ids = self.db.image_tag_ids([], "name_")
|
||||
self.assertEqual(len(ids), 503)
|
||||
self.assertEqual(self.db.image_tag_locators(ids), set(locators))
|
||||
|
||||
|
||||
class AutotagMarkTests(AnnotationTestCase):
|
||||
"""The auto-tagged mark (#379): per byte record, written with the tags."""
|
||||
|
||||
def marked(self):
|
||||
conn = self.db.model.get_connection()
|
||||
return sorted(
|
||||
row[0] for row in conn.execute("SELECT * FROM media_image_autotags")
|
||||
)
|
||||
|
||||
def byte_id(self, loc, digest=DIGEST_A):
|
||||
conn = self.db.model.get_connection()
|
||||
return conn.execute(
|
||||
"SELECT b.id FROM media_annotation_bytes b JOIN media_annotation_files f "
|
||||
"ON f.id=b.file_id WHERE f.locator=? AND b.sha256=?",
|
||||
(loc, digest),
|
||||
).fetchone()[0]
|
||||
|
||||
def test_hand_tags_never_mark(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=["keeper"])
|
||||
self.add(loc, tags=["more"])
|
||||
self.assertEqual(self.marked(), [])
|
||||
self.assertEqual(self.db.autotag_state({loc: (SIZE, MTIME)}), {loc: False})
|
||||
self.assertEqual(self.db.autotagged_locators(), set())
|
||||
|
||||
def test_model_union_marks_including_replayed_and_empty_results(self):
|
||||
loc = self.locator("a.png")
|
||||
receipt = self.add(loc, tags=["bird"], mark_autotagged=True)
|
||||
self.assertEqual((receipt["added"], receipt["replayed"]), (1, False))
|
||||
self.assertEqual(self.marked(), [self.byte_id(loc)])
|
||||
# A replayed union and an empty result still mark (idempotently).
|
||||
other = self.locator("b.png")
|
||||
self.replace(other, DIGEST_B, ["keeper"])
|
||||
replayed = self.add(other, DIGEST_B, ["KEEPER"], mark_autotagged=True)
|
||||
self.assertTrue(replayed["replayed"])
|
||||
empty = self.locator("c.png")
|
||||
nothing = self.add(empty, tags=[], mark_autotagged=True)
|
||||
self.assertEqual((nothing["tags"], nothing["added"]), ([], 0))
|
||||
self.add(empty, tags=[], mark_autotagged=True)
|
||||
self.assertEqual(
|
||||
self.marked(),
|
||||
sorted(
|
||||
[self.byte_id(loc), self.byte_id(other, DIGEST_B), self.byte_id(empty)]
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.autotag_state(
|
||||
{loc: (SIZE, MTIME), other: (SIZE, MTIME), empty: (SIZE, MTIME)}
|
||||
),
|
||||
{loc: True, other: True, empty: True},
|
||||
)
|
||||
self.schema_clean()
|
||||
|
||||
def test_empty_union_without_mark_is_refused(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.add(self.locator("a.png"), tags=[])
|
||||
self.assertEqual(self.counts()["media_annotation_files"], 0)
|
||||
|
||||
def test_tag_limit_union_writes_no_tag_and_no_mark(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=[f"t{i}" for i in range(100)])
|
||||
before = self.rows(*ANNOTATION_TABLES, "tags")
|
||||
with self.assertRaises(AnnotationCommandError) as ctx:
|
||||
self.add(loc, tags=["one more"], mark_autotagged=True)
|
||||
self.assertEqual(ctx.exception.code, "tag-limit")
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES, "tags"), before)
|
||||
self.assertEqual(self.marked(), [])
|
||||
|
||||
def test_replace_marks_on_saved_and_replayed_receipts_not_on_conflict(self):
|
||||
loc = self.locator("a.png")
|
||||
saved = self.replace(loc, tags=["a"], mark_autotagged=True)
|
||||
self.assertFalse(saved["replayed"])
|
||||
self.assertEqual(self.marked(), [self.byte_id(loc)])
|
||||
other = self.locator("b.png")
|
||||
self.replace(other, DIGEST_B, ["x"])
|
||||
replayed = self.replace(
|
||||
other, DIGEST_B, ["x"], revision=1, mark_autotagged=True
|
||||
)
|
||||
self.assertTrue(replayed["replayed"])
|
||||
third = self.locator("c.png")
|
||||
self.replace(third, tags=["y"])
|
||||
with self.assertRaises(AnnotationCommandError):
|
||||
self.replace(third, tags=["z"], revision=5, mark_autotagged=True)
|
||||
self.assertEqual(
|
||||
self.marked(), sorted([self.byte_id(loc), self.byte_id(other, DIGEST_B)])
|
||||
)
|
||||
|
||||
def test_mark_follows_the_bytes_not_the_file(self):
|
||||
loc = self.locator("a.png")
|
||||
self.add(loc, tags=["bird"], mark_autotagged=True)
|
||||
# The file's bytes changed: the new record is current and unmarked.
|
||||
self.replace(loc, DIGEST_B, ["hand"], mtime_ns=MTIME + 1)
|
||||
self.assertEqual(self.db.autotag_state({loc: (SIZE, MTIME + 1)}), {loc: False})
|
||||
self.assertEqual(self.db.autotag_state({loc: (SIZE, MTIME)}), {loc: True})
|
||||
# The review's estimate counts any marked record of the locator.
|
||||
self.assertEqual(self.db.autotagged_locators(), {loc})
|
||||
missing = self.locator("none.png")
|
||||
self.assertEqual(self.db.autotag_state({missing: (1, 1)}), {missing: False})
|
||||
|
||||
def test_state_and_tags_choose_the_same_current_record(self):
|
||||
loc = self.locator("a.png")
|
||||
# Two records with the same verified fingerprint: the newest wins.
|
||||
self.replace(loc, DIGEST_A, ["old bytes"])
|
||||
with patch.object(
|
||||
annotations_module, "utc_now_iso", return_value="2099-01-01T00:00:00"
|
||||
):
|
||||
self.add(loc, DIGEST_B, ["new bytes"], mark_autotagged=True)
|
||||
stats = {loc: (SIZE, MTIME)}
|
||||
self.assertEqual(self.db.current_image_tags(stats), {loc: ["new bytes"]})
|
||||
self.assertEqual(self.db.autotag_state(stats), {loc: True})
|
||||
with patch.object(
|
||||
annotations_module, "utc_now_iso", return_value="2199-01-01T00:00:00"
|
||||
):
|
||||
self.add(loc, DIGEST_A, ["older bytes again"])
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags(stats), {loc: ["old bytes", "older bytes again"]}
|
||||
)
|
||||
self.assertEqual(self.db.autotag_state(stats), {loc: False})
|
||||
|
||||
def test_marks_never_touch_prompt_rows(self):
|
||||
conn = self.db.model.get_connection()
|
||||
prompt_id = self.db.save_prompt("a beach", tags=["beach"])
|
||||
before = [
|
||||
[tuple(row) for row in conn.execute(f"SELECT * FROM {table} ORDER BY 1")]
|
||||
for table in ("prompts", "prompt_tags")
|
||||
]
|
||||
self.add(self.locator("a.png"), tags=["bird"], mark_autotagged=True)
|
||||
self.replace(self.locator("b.png"), DIGEST_B, ["waves"], mark_autotagged=True)
|
||||
after = [
|
||||
[tuple(row) for row in conn.execute(f"SELECT * FROM {table} ORDER BY 1")]
|
||||
for table in ("prompts", "prompt_tags")
|
||||
]
|
||||
self.assertIsNotNone(prompt_id)
|
||||
self.assertEqual(after, before)
|
||||
|
||||
|
||||
class AnnotationIsolationTests(AnnotationTestCase):
|
||||
|
||||
+2
-3
@@ -8,7 +8,6 @@ without requiring a full ComfyUI server.
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
@@ -18,6 +17,7 @@ from aiohttp.test_utils import AioHTTPTestCase
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
from py.api import PromptManagerAPI
|
||||
from tests.private_temp import private_temp_db
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
|
||||
@@ -25,8 +25,7 @@ class APITestCase(AioHTTPTestCase):
|
||||
"""Base class that stands up a test aiohttp app with PromptManager routes."""
|
||||
|
||||
async def get_application(self):
|
||||
self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db")
|
||||
self._temp_db.close()
|
||||
self._temp_db = private_temp_db(self)
|
||||
|
||||
app = web.Application()
|
||||
routes = web.RouteTableDef()
|
||||
|
||||
+13
-21
@@ -846,6 +846,14 @@ class TestSettingsReadMonitorThroughModule(AdminAPITestCase):
|
||||
self.assertEqual(data["settings"]["gallery_root_paths"], ["output"])
|
||||
|
||||
|
||||
def _wired_prompt(text):
|
||||
"""A sampler-less graph whose one encoder feeds a guider (#252)."""
|
||||
return {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": text}},
|
||||
"2": {"class_type": "BasicGuider", "inputs": {"conditioning": ["1", 0]}},
|
||||
}
|
||||
|
||||
|
||||
def _raise(*args, **kwargs):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
@@ -2186,11 +2194,7 @@ class TestOutputScanInternals(AdminAPITestCase):
|
||||
|
||||
def test_ingest_outcomes(self):
|
||||
f = Path("gen.png")
|
||||
meta = {
|
||||
"prompt": json.dumps(
|
||||
{"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat"}}}
|
||||
)
|
||||
}
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a cat"))}
|
||||
self.assertIsNone(self.api._ingest_scanned_file(f, {}))
|
||||
self.assertIsNone(self.api._ingest_scanned_file(f, {"prompt": "{}"}))
|
||||
self.assertEqual(
|
||||
@@ -2202,31 +2206,19 @@ class TestOutputScanInternals(AdminAPITestCase):
|
||||
|
||||
def test_ingest_link_failures_are_tolerated(self):
|
||||
f = Path("gen.png")
|
||||
meta = {
|
||||
"prompt": json.dumps(
|
||||
{"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a dog"}}}
|
||||
)
|
||||
}
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a dog"))}
|
||||
self.api.db.link_image_to_prompt = _raise
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "added")
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "found")
|
||||
|
||||
def test_ingest_save_failure_counts_as_found(self):
|
||||
f = Path("gen.png")
|
||||
meta = {
|
||||
"prompt": json.dumps(
|
||||
{"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a bird"}}}
|
||||
)
|
||||
}
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a bird"))}
|
||||
self.api.db.save_prompt = lambda *a, **k: None
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "found")
|
||||
|
||||
def test_batch_counts_and_per_file_errors(self):
|
||||
meta = {
|
||||
"prompt": json.dumps(
|
||||
{"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a fish"}}}
|
||||
)
|
||||
}
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a fish"))}
|
||||
self.api.db.get_prompt_by_hash = _raise
|
||||
counts = self.api._ingest_scan_batch_sync(
|
||||
[(Path("a.png"), meta), (Path("b.png"), {})]
|
||||
@@ -2915,7 +2907,7 @@ class TestScanReadsVideos(AdminAPITestCase):
|
||||
shutil.which("ffmpeg") and shutil.which("ffprobe"), "ffmpeg not installed"
|
||||
)
|
||||
async def test_scan_adds_prompts_found_in_videos(self):
|
||||
graph = {"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat"}}}
|
||||
graph = _wired_prompt("a cat")
|
||||
_write_mp4(
|
||||
self.output_dir / "clip.mp4", json.dumps({"prompt": json.dumps(graph)})
|
||||
)
|
||||
|
||||
@@ -363,23 +363,20 @@ class TestModelGate(SuggestTestCase):
|
||||
prompt_id = self._save_prompt("batch prompt")
|
||||
self.api.db.link_image_to_prompt(prompt_id, str(self.output_dir / "sub/a.png"))
|
||||
started, proceed = threading.Event(), threading.Event()
|
||||
|
||||
def slow_generate(image_path, **kwargs):
|
||||
started.set()
|
||||
proceed.wait(WAIT)
|
||||
return ["batch"]
|
||||
|
||||
self.tagger.generate_tags = slow_generate
|
||||
# The legacy batch is the autotag job (#379): it suggests per image.
|
||||
self.tagger.block = (started, proceed)
|
||||
batch = asyncio.ensure_future(
|
||||
self.client.post("/prompt_manager/autotag/start?model_type=wd14-vit")
|
||||
)
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, started.wait, WAIT)
|
||||
calls = len(self.tagger.calls)
|
||||
status, body = await self.suggest(self.suggest_body(observed, "gguf"))
|
||||
self.assertEqual((status, body["code"]), (409, "model-busy"))
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
self.assertEqual(len(self.tagger.calls), calls)
|
||||
self.assertEqual(self.tagger.loaded, "wd14-vit") # not swapped
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
response = await batch
|
||||
await response.read()
|
||||
|
||||
+80
-576
@@ -5,6 +5,7 @@ no network request is made.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
@@ -34,6 +35,8 @@ from aiohttp.test_utils import AioHTTPTestCase
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
from py.api import PromptManagerAPI
|
||||
from tests.private_temp import private_temp_db
|
||||
from py.api.autotag_routes import MODEL_BUSY_ERROR
|
||||
from py.api.jobs import JobBusy
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
@@ -70,6 +73,7 @@ class FakeAutotagService:
|
||||
self.status_calls = 0
|
||||
self.status_threads = []
|
||||
self.status_paths = {}
|
||||
self.suggest_calls = 0
|
||||
|
||||
def get_models_status(self):
|
||||
self.status_calls += 1
|
||||
@@ -115,14 +119,32 @@ class FakeAutotagService:
|
||||
raise self.generate_error
|
||||
return list(self.tags)
|
||||
|
||||
def generate_tag_suggestions(
|
||||
self,
|
||||
image_path,
|
||||
*,
|
||||
limit,
|
||||
prompt=None,
|
||||
general_threshold=None,
|
||||
character_threshold=None,
|
||||
):
|
||||
"""The suggestion entry point (#379): the fixed tags, at most ``limit``."""
|
||||
self.suggest_calls += 1
|
||||
tags = self.generate_tags(
|
||||
image_path,
|
||||
prompt=prompt,
|
||||
general_threshold=general_threshold,
|
||||
character_threshold=character_threshold,
|
||||
)
|
||||
return [(name, None) for name in tags[:limit]]
|
||||
|
||||
|
||||
class AutotagAPITestCase(AioHTTPTestCase):
|
||||
"""Stands up the PromptManager routes with a temp DB, temp output dir and a
|
||||
fake engine."""
|
||||
|
||||
async def get_application(self):
|
||||
self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db")
|
||||
self._temp_db.close()
|
||||
self._temp_db = private_temp_db(self)
|
||||
self.tmp_root = Path(tempfile.mkdtemp())
|
||||
self.output_dir = self.tmp_root / "output"
|
||||
self.output_dir.mkdir()
|
||||
@@ -175,8 +197,17 @@ class AutotagAPITestCase(AioHTTPTestCase):
|
||||
)
|
||||
|
||||
def _make_image(self, directory, name="img.png"):
|
||||
"""A small, valid image (the legacy review captures and probes it)."""
|
||||
from PIL import Image
|
||||
|
||||
path = directory / name
|
||||
path.write_bytes(b"\x89PNG\r\n\x1a\n")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shade = sum(name.encode()) % 256
|
||||
fmt = {".jpg": "JPEG", ".jpeg": "JPEG", ".webp": "WEBP"}.get(
|
||||
path.suffix.lower(), "PNG"
|
||||
)
|
||||
with Image.new("RGB", (4, 4), (shade, 64, 128)) as image:
|
||||
image.save(path, fmt)
|
||||
return path
|
||||
|
||||
def _link_image(self, prompt_id, path):
|
||||
@@ -216,10 +247,9 @@ class TestAutotagSingleContainment(AutotagAPITestCase):
|
||||
self.assertEqual(data["tags"], ["1girl", "smile"])
|
||||
self.assertEqual(data["prompt_id"], prompt_id)
|
||||
self.assertEqual(len(self.service.generate_calls), 1)
|
||||
self.assertEqual(
|
||||
os.path.realpath(self.service.generate_calls[0][0]),
|
||||
os.path.realpath(str(image)),
|
||||
)
|
||||
# #379: the model reads a private capture of the image, never the file.
|
||||
self.assertNotIn(str(self.output_dir), self.service.generate_calls[0][0])
|
||||
self.assertEqual(data["sha256"], hashlib.sha256(image.read_bytes()).hexdigest())
|
||||
|
||||
async def test_path_inside_output_dir_is_tagged(self):
|
||||
image = self._make_image(self.output_dir)
|
||||
@@ -449,238 +479,6 @@ class TestAutotagSideEffectRoutesArePostOnly(AutotagAPITestCase):
|
||||
self.assertTrue((await resp.json())["success"])
|
||||
|
||||
|
||||
class TestAutotagBatch(AutotagAPITestCase):
|
||||
"""Batch tagging over the DB image list with SSE progress reporting."""
|
||||
|
||||
def _seed(self):
|
||||
"""Three prompts: A (one image), B (two images), C (missing file)."""
|
||||
a = self._save_prompt("prompt a")
|
||||
b = self._save_prompt("prompt b")
|
||||
c = self._save_prompt("prompt c")
|
||||
self._link_image(a, self._make_image(self.output_dir, "a1.png"))
|
||||
self._link_image(b, self._make_image(self.output_dir, "b1.png"))
|
||||
self._link_image(b, self._make_image(self.output_dir, "b2.png"))
|
||||
self._link_image(c, self.output_dir / "missing.png")
|
||||
return a, b, c
|
||||
|
||||
async def _start(self, query="model_type=gguf", body=None):
|
||||
if body is None:
|
||||
resp = await self.client.request(
|
||||
"POST", f"/prompt_manager/autotag/start?{query}"
|
||||
)
|
||||
else:
|
||||
resp = await self._post_json(f"/prompt_manager/autotag/start?{query}", body)
|
||||
return resp, self._sse_events(await resp.text())
|
||||
|
||||
async def test_batch_tags_each_prompt_once_and_reports_counts(self):
|
||||
a, b, c = self._seed()
|
||||
|
||||
resp, events = await self._start()
|
||||
|
||||
self.assertEqual(resp.status, 200)
|
||||
done = events[-1]
|
||||
self.assertEqual(done["type"], "complete")
|
||||
self.assertEqual(done["processed"], 2)
|
||||
self.assertEqual(done["tagged"], 2)
|
||||
self.assertEqual(done["skipped"], 2)
|
||||
self.assertEqual(done["errors"], 0)
|
||||
self.assertEqual(len(self.service.generate_calls), 2)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(a)["tags"], ["1girl", "smile"])
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(b)["tags"], ["1girl", "smile"])
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(c)["tags"], [])
|
||||
progress = [e for e in events if e["type"] == "progress"]
|
||||
self.assertTrue(all(0 <= e["progress"] <= 100 for e in progress))
|
||||
self.assertTrue(any("processed" in e for e in progress))
|
||||
|
||||
async def test_rows_outside_output_dirs_are_skipped_and_counted(self):
|
||||
inside = self._save_prompt("inside")
|
||||
outside = self._save_prompt("outside")
|
||||
self._link_image(inside, self._make_image(self.output_dir, "in.png"))
|
||||
secret = self._make_image(self.secret_dir, "password.png")
|
||||
self._link_image(outside, secret)
|
||||
|
||||
_, events = await self._start()
|
||||
|
||||
done = events[-1]
|
||||
self.assertEqual(done["type"], "complete")
|
||||
self.assertEqual(done["processed"], 1)
|
||||
self.assertEqual(done["tagged"], 1)
|
||||
self.assertEqual(done["skipped"], 1)
|
||||
self.assertEqual(done["skipped_outside_roots"], 1)
|
||||
self.assertEqual(len(self.service.generate_calls), 1)
|
||||
self.assertEqual(
|
||||
os.path.realpath(self.service.generate_calls[0][0]),
|
||||
os.path.realpath(str(self.output_dir / "in.png")),
|
||||
)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(outside)["tags"], [])
|
||||
progress = [e for e in events if e["type"] == "progress" and "skipped" in e]
|
||||
self.assertTrue(progress)
|
||||
self.assertTrue(all("skipped_outside_roots" in e for e in progress))
|
||||
body = json.dumps(events)
|
||||
self.assertNotIn(str(self.secret_dir), body)
|
||||
|
||||
async def test_second_start_while_running_is_409(self):
|
||||
self._seed()
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_generate(image_path, **kwargs):
|
||||
started.set()
|
||||
release.wait(5)
|
||||
return ["slow"]
|
||||
|
||||
self.service.generate_tags = slow_generate
|
||||
first = asyncio.ensure_future(self._start())
|
||||
try:
|
||||
await asyncio.get_event_loop().run_in_executor(None, started.wait, 5)
|
||||
self.assertTrue(started.is_set())
|
||||
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?model_type=gguf"
|
||||
)
|
||||
|
||||
self.assertEqual(resp.status, 409)
|
||||
self.assertEqual(
|
||||
await resp.json(),
|
||||
{"success": False, "error": "Auto-tagging is already running"},
|
||||
)
|
||||
finally:
|
||||
release.set()
|
||||
resp1, events = await first
|
||||
self.assertEqual(resp1.status, 200)
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
|
||||
async def test_lock_is_released_after_a_run(self):
|
||||
self._seed()
|
||||
|
||||
resp1, _ = await self._start()
|
||||
resp2, events = await self._start()
|
||||
|
||||
self.assertEqual(resp1.status, 200)
|
||||
self.assertEqual(resp2.status, 200)
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
|
||||
async def test_lock_is_released_when_the_run_fails(self):
|
||||
self._seed()
|
||||
self.service.load_error = RuntimeError("no model")
|
||||
|
||||
_, events = await self._start()
|
||||
self.assertEqual(events[-1]["type"], "error")
|
||||
|
||||
self.service.load_error = None
|
||||
resp, events = await self._start()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
|
||||
async def test_already_tagged_prompts_are_skipped_unless_asked(self):
|
||||
p = self._save_prompt("tagged", tags=["cat"])
|
||||
self._link_image(p, self._make_image(self.output_dir, "t.png"))
|
||||
|
||||
_, events = await self._start("model_type=gguf&skip_tagged=true")
|
||||
self.assertEqual(events[-1]["skipped"], 1)
|
||||
self.assertEqual(self.service.generate_calls, [])
|
||||
|
||||
_, events = await self._start("model_type=gguf&skip_tagged=false")
|
||||
self.assertEqual(events[-1]["tagged"], 1)
|
||||
self.assertEqual(
|
||||
self.api.db.get_prompt_by_id(p)["tags"], ["cat", "1girl", "smile"]
|
||||
)
|
||||
|
||||
async def test_prompt_already_holding_the_tags_counts_as_skipped(self):
|
||||
p = self._save_prompt("same", tags=["1girl", "smile"])
|
||||
self._link_image(p, self._make_image(self.output_dir, "s.png"))
|
||||
|
||||
_, events = await self._start("model_type=gguf&skip_tagged=false")
|
||||
|
||||
self.assertEqual(events[-1]["skipped"], 1)
|
||||
self.assertEqual(events[-1]["tagged"], 0)
|
||||
|
||||
async def test_generation_errors_are_counted_and_do_not_abort(self):
|
||||
self._seed()
|
||||
self.service.generate_error = RuntimeError("boom")
|
||||
|
||||
_, events = await self._start()
|
||||
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
# a1, b1 and b2 each fail: a failed prompt is retried on its next image
|
||||
self.assertEqual(events[-1]["errors"], 3)
|
||||
self.assertEqual(events[-1]["tagged"], 0)
|
||||
|
||||
async def test_json_body_parameters_are_accepted(self):
|
||||
self._seed()
|
||||
|
||||
_, events = await self._start(
|
||||
"", {"model_type": "wd14-vit", "general_threshold": 0.5, "prompt": "p"}
|
||||
)
|
||||
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
self.assertEqual(self.service.load_calls, [("wd14-vit", True)])
|
||||
self.assertEqual(self.service.generate_calls[0][2], 0.5)
|
||||
self.assertEqual(self.service.custom_prompt, "p")
|
||||
|
||||
async def test_keep_in_memory_false_unloads_after_batch(self):
|
||||
self._seed()
|
||||
|
||||
_, events = await self._start("model_type=gguf&keep_in_memory=false")
|
||||
|
||||
self.assertEqual(events[-1]["model_loaded"], False)
|
||||
self.assertEqual(self.service.unload_calls, 1)
|
||||
|
||||
async def test_model_not_downloaded_is_an_error_event(self):
|
||||
self.service.downloaded = False
|
||||
|
||||
_, events = await self._start()
|
||||
|
||||
self.assertEqual(events[-1]["type"], "error")
|
||||
self.assertEqual(self.service.load_calls, [])
|
||||
|
||||
async def test_load_failure_is_an_error_event_without_details(self):
|
||||
self.service.load_error = RuntimeError("/srv/models/secret.gguf is corrupt")
|
||||
|
||||
_, events = await self._start()
|
||||
|
||||
self.assertEqual(events[-1]["type"], "error")
|
||||
self.assertNotIn("/srv/models", events[-1]["message"])
|
||||
|
||||
async def test_non_numeric_threshold_is_400(self):
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?general_threshold=abc"
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
async def test_invalid_json_body_is_400(self):
|
||||
resp = await self.client.request(
|
||||
"POST",
|
||||
"/prompt_manager/autotag/start",
|
||||
data="{not json",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
async def test_client_disconnect_stops_the_batch_and_unloads(self):
|
||||
self._seed()
|
||||
real_write = web.StreamResponse.write
|
||||
state = {"writes": 0}
|
||||
|
||||
async def flaky_write(response, data):
|
||||
state["writes"] += 1
|
||||
if b"Processing " in data:
|
||||
raise ConnectionResetError("Cannot write to closing transport")
|
||||
return await real_write(response, data)
|
||||
|
||||
with patch.object(web.StreamResponse, "write", flaky_write):
|
||||
try:
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?keep_in_memory=false"
|
||||
)
|
||||
await resp.read()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.assertEqual(len(self.service.generate_calls), 1)
|
||||
self.assertEqual(self.service.unload_calls, 1)
|
||||
|
||||
|
||||
class TestAutotagModelsAndDownloadErrors(AutotagAPITestCase):
|
||||
|
||||
async def test_models_error_is_500(self):
|
||||
@@ -834,7 +632,15 @@ class TestAutotagSingleEdgeCases(AutotagAPITestCase):
|
||||
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(self.service.load_calls, [("wd14-vit", True)])
|
||||
self.assertEqual(self.service.custom_prompt, "terse")
|
||||
# #379: the request's prompt is for this call only.
|
||||
self.assertEqual(self.service.custom_prompt, "")
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/single",
|
||||
{"path": str(image), "model_type": "gguf", "prompt": "terse"},
|
||||
)
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(self.service.generate_calls[-1][1], "terse")
|
||||
self.assertEqual(self.service.custom_prompt, "")
|
||||
|
||||
async def test_image_id_without_stored_path_is_404(self):
|
||||
prompt_id = self._save_prompt()
|
||||
@@ -850,64 +656,6 @@ class TestAutotagSingleEdgeCases(AutotagAPITestCase):
|
||||
self.assertEqual(resp.status, 404)
|
||||
|
||||
|
||||
class TestApplyAutotag(AutotagAPITestCase):
|
||||
|
||||
async def test_missing_prompt_id_is_400(self):
|
||||
resp = await self._post_json("/prompt_manager/autotag/apply", {"tags": ["a"]})
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
async def test_no_tags_is_a_noop(self):
|
||||
resp = await self._post_json("/prompt_manager/autotag/apply", {"prompt_id": 1})
|
||||
data = await resp.json()
|
||||
self.assertTrue(data["success"])
|
||||
self.assertEqual(data["message"], "No tags to apply")
|
||||
|
||||
async def test_unknown_prompt_is_404(self):
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply", {"prompt_id": 999, "tags": ["a"]}
|
||||
)
|
||||
self.assertEqual(resp.status, 404)
|
||||
|
||||
async def test_tags_are_merged_without_duplicates(self):
|
||||
prompt_id = self._save_prompt(tags=["cat"])
|
||||
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply",
|
||||
{"prompt_id": prompt_id, "tags": ["cat", "dog"]},
|
||||
)
|
||||
|
||||
data = await resp.json()
|
||||
self.assertEqual(data["added_tags"], ["dog"])
|
||||
self.assertEqual(data["total_tags"], 2)
|
||||
self.assertEqual(
|
||||
self.api.db.get_prompt_by_id(prompt_id)["tags"], ["cat", "dog"]
|
||||
)
|
||||
|
||||
async def test_string_tags_from_db_are_split(self):
|
||||
prompt_id = self._save_prompt()
|
||||
with patch.object(
|
||||
self.api.db,
|
||||
"get_prompt_by_id",
|
||||
return_value={"id": prompt_id, "tags": "cat, dog"},
|
||||
):
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply",
|
||||
{"prompt_id": prompt_id, "tags": ["dog", "owl"]},
|
||||
)
|
||||
data = await resp.json()
|
||||
self.assertEqual(data["added_tags"], ["owl"])
|
||||
self.assertEqual(data["total_tags"], 3)
|
||||
|
||||
async def test_db_error_is_500(self):
|
||||
with patch.object(
|
||||
self.api.db, "get_prompt_by_id", side_effect=RuntimeError("db")
|
||||
):
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply", {"prompt_id": 1, "tags": ["a"]}
|
||||
)
|
||||
self.assertEqual(resp.status, 500)
|
||||
|
||||
|
||||
class TestAutotagErrorBodiesHideServerPaths(AutotagAPITestCase):
|
||||
"""Every autotag handler reports exceptions through the shared public helper."""
|
||||
|
||||
@@ -932,9 +680,12 @@ class TestAutotagErrorBodiesHideServerPaths(AutotagAPITestCase):
|
||||
await self._assert_hidden(resp)
|
||||
|
||||
async def test_apply_error(self):
|
||||
with patch.object(self.api.db, "get_prompt_by_id", side_effect=self._locked()):
|
||||
image = self._make_image(self.output_dir)
|
||||
digest = hashlib.sha256(image.read_bytes()).hexdigest()
|
||||
with patch.object(self.api.db, "add_image_tags", side_effect=self._locked()):
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply", {"prompt_id": 1, "tags": ["a"]}
|
||||
"/prompt_manager/autotag/apply",
|
||||
{"path": str(image), "sha256": digest, "tags": ["a"]},
|
||||
)
|
||||
await self._assert_hidden(resp)
|
||||
|
||||
@@ -1008,10 +759,13 @@ class TestLegacyRoutesShareTheModelGate(AutotagAPITestCase):
|
||||
"POST", "/prompt_manager/autotag/download/gguf"
|
||||
)
|
||||
)
|
||||
await self._busy(
|
||||
await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?model_type=gguf"
|
||||
)
|
||||
# The legacy batch is read as a stream: one error frame (#379).
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?model_type=gguf"
|
||||
)
|
||||
self.assertEqual(
|
||||
self._sse_events(await resp.text()),
|
||||
[{"type": "error", "status": "error", "message": MODEL_BUSY_ERROR}],
|
||||
)
|
||||
await self._busy(
|
||||
await self.client.request("POST", "/prompt_manager/autotag/unload")
|
||||
@@ -1026,7 +780,7 @@ class TestLegacyRoutesShareTheModelGate(AutotagAPITestCase):
|
||||
self.assertEqual(self.service.unload_calls, 0)
|
||||
self.assertEqual(self.service.get_loaded_model_type(), "wd14-vit")
|
||||
|
||||
# A refused start released the batch lock: the next start runs.
|
||||
# A refused start left no job behind: the next start runs.
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?model_type=gguf"
|
||||
)
|
||||
@@ -1238,7 +992,7 @@ if __name__ == "__main__":
|
||||
|
||||
|
||||
class TestGalleryReviewContract(AutotagAPITestCase):
|
||||
async def test_root_relative_gallery_targets_generate_and_apply_to_their_prompt(
|
||||
async def test_root_relative_gallery_targets_generate_and_apply_to_their_image(
|
||||
self,
|
||||
):
|
||||
second = self.tmp_root / "second-output"
|
||||
@@ -1255,16 +1009,22 @@ class TestGalleryReviewContract(AutotagAPITestCase):
|
||||
self.assertEqual(response.status, 200)
|
||||
data = await response.json()
|
||||
self.assertEqual(data["prompt_id"], pid)
|
||||
self.assertEqual(
|
||||
self.service.generate_calls[-1][0], str((root / "same.png").resolve())
|
||||
)
|
||||
response = await self._post_json(
|
||||
"/prompt_manager/autotag/apply",
|
||||
{"prompt_id": data["prompt_id"], "tags": data["tags"]},
|
||||
{
|
||||
"path": "same.png",
|
||||
"root_index": index,
|
||||
"sha256": data["sha256"],
|
||||
"tags": data["tags"],
|
||||
},
|
||||
)
|
||||
self.assertEqual(response.status, 200)
|
||||
# The image's own tags; the prompt is never written (#379).
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(pid)["tags"], [])
|
||||
locator = os.path.normcase(os.path.realpath(str(root / "same.png")))
|
||||
self.assertEqual(
|
||||
set(self.api.db.get_prompt_by_id(pid)["tags"]), set(data["tags"])
|
||||
self.api.db.read_image_tags(locator, data["sha256"])["tags"],
|
||||
data["tags"],
|
||||
)
|
||||
|
||||
async def test_gallery_unlinked_image_returns_no_prompt_id(self):
|
||||
@@ -1893,13 +1653,15 @@ class TestTagOutputImagesPreview(BatchJobTestCase):
|
||||
self.assertEqual(self.service.generate_calls, [])
|
||||
self.assertEqual(self.service.load_calls, [])
|
||||
|
||||
async def test_preview_counts_skip_and_merge(self):
|
||||
async def test_preview_counts_listed_still_images_not_prompts(self):
|
||||
self._seed()
|
||||
skip = await (await self._preview(existing="skip")).json()
|
||||
# t, f1, f2 and m are listed; gone.png is missing, o.png is outside.
|
||||
self.assertEqual(
|
||||
{k: skip[k] for k in ("images", "prompts", "already_tagged", "to_tag")},
|
||||
{"images": 6, "prompts": 5, "already_tagged": 1, "to_tag": 4},
|
||||
{k: skip[k] for k in ("images", "already_tagged", "to_tag")},
|
||||
{"images": 4, "already_tagged": 0, "to_tag": 4},
|
||||
)
|
||||
self.assertNotIn("prompts", skip)
|
||||
self.assertEqual(
|
||||
skip["model"],
|
||||
{
|
||||
@@ -1916,7 +1678,7 @@ class TestTagOutputImagesPreview(BatchJobTestCase):
|
||||
set(skip["disk"]), {"free_bytes", "required_bytes", "sufficient"}
|
||||
)
|
||||
merge = await (await self._preview(existing="merge")).json()
|
||||
self.assertEqual(merge["to_tag"], 5)
|
||||
self.assertEqual(merge["to_tag"], 4)
|
||||
leaked = [s for s in _strings(skip) if _absolute(s)]
|
||||
self.assertEqual(leaked, [])
|
||||
|
||||
@@ -1924,7 +1686,7 @@ class TestTagOutputImagesPreview(BatchJobTestCase):
|
||||
self._seed()
|
||||
data = await (await self._preview(existing="replace_after_review")).json()
|
||||
self.assertTrue(data["review_required"])
|
||||
self.assertEqual(data["prompts"], 5)
|
||||
self.assertEqual(data["images"], 4)
|
||||
|
||||
async def test_preview_refuses_bad_bodies(self):
|
||||
for body in (
|
||||
@@ -1979,119 +1741,6 @@ class TestTagOutputImagesJob(BatchJobTestCase):
|
||||
"WD14 ViT is not downloaded. Download it in System › Auto-tagging.",
|
||||
)
|
||||
|
||||
async def test_job_writes_what_legacy_writes(self):
|
||||
ids = self._seed()
|
||||
await self._run_job(existing="skip")
|
||||
native = {name: sorted(self._tags(pid)) for name, pid in ids.items()}
|
||||
# The same library through the legacy batch, in a fresh database.
|
||||
for name, pid in ids.items():
|
||||
self.api.db.update_prompt_metadata(
|
||||
pid,
|
||||
tags=(
|
||||
["existing"]
|
||||
if name == "tagged"
|
||||
else (["auto-scanned"] if name == "marked" else [])
|
||||
),
|
||||
)
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/autotag/start?model_type=wd14-vit&skip_tagged=true"
|
||||
)
|
||||
self.assertEqual(self._sse_events(await resp.text())[-1]["type"], "complete")
|
||||
legacy = {name: sorted(self._tags(pid)) for name, pid in ids.items()}
|
||||
self.assertEqual(native, legacy)
|
||||
self.assertEqual(native["tagged"], ["existing"])
|
||||
self.assertEqual(native["fresh"], ["1girl", "smile"])
|
||||
|
||||
async def test_job_outcomes_follow_the_mapping(self):
|
||||
ids = self._seed()
|
||||
self.api.db.update_prompt_metadata(
|
||||
ids["marked"], tags=["auto-scanned", "1girl", "smile"]
|
||||
)
|
||||
|
||||
def details(job):
|
||||
return {
|
||||
group: sorted(row["detail"] for row in rows)
|
||||
for group, rows in job.outcomes.items()
|
||||
}
|
||||
|
||||
merged = await self._run_job(existing="merge")
|
||||
self.assertEqual(merged.state, "finished")
|
||||
self.assertEqual(
|
||||
details(merged),
|
||||
{
|
||||
"tagged": ["Added 2 tags", "Added 2 tags"],
|
||||
"unchanged": ["No new tags"],
|
||||
"skipped": [
|
||||
"File not found",
|
||||
"Outside the output folders",
|
||||
"Prompt handled by another image",
|
||||
],
|
||||
"errors": [],
|
||||
},
|
||||
)
|
||||
self.assertEqual(merged.status_line, "Tagged 2 prompts")
|
||||
|
||||
skipped = await self._run_job(existing="skip")
|
||||
self.assertEqual(details(skipped)["tagged"], [])
|
||||
self.assertEqual(details(skipped)["skipped"].count("Prompt already tagged"), 3)
|
||||
self.assertEqual(skipped.status_line, "Tagged 0 prompts")
|
||||
|
||||
async def test_tagging_errors_are_fixed_strings(self):
|
||||
self._seed()
|
||||
self.service.generate_error = RuntimeError("/srv/secret/model exploded")
|
||||
job = await self._run_job(existing="merge")
|
||||
self.assertTrue(job.outcomes["errors"])
|
||||
self.assertEqual(
|
||||
{r["detail"] for r in job.outcomes["errors"]}, {"Tagging failed"}
|
||||
)
|
||||
self.assertNotIn("/srv/secret", json.dumps(job.report()))
|
||||
|
||||
async def test_job_progress_is_determinate(self):
|
||||
self._seed()
|
||||
seen = []
|
||||
original = self.service.generate_tags
|
||||
|
||||
def generate(*args, **kwargs):
|
||||
job = self.api._jobs.active("autotag")
|
||||
seen.append((job.processed, job.total))
|
||||
return original(*args, **kwargs)
|
||||
|
||||
self.service.generate_tags = generate
|
||||
job = await self._run_job(existing="merge")
|
||||
self.assertEqual((job.processed, job.total, job.progress), (6, 6, 100))
|
||||
self.assertTrue(all(total == 6 for _p, total in seen[1:]), seen)
|
||||
self.assertTrue(job.cancellable)
|
||||
|
||||
async def test_job_cancel_stops_before_the_next_image_and_keeps_writes(self):
|
||||
ids = self._seed()
|
||||
started = threading.Event()
|
||||
proceed = threading.Event()
|
||||
calls = []
|
||||
|
||||
def slow(image_path, **kwargs):
|
||||
calls.append(image_path)
|
||||
started.set()
|
||||
proceed.wait(30)
|
||||
return ["slow"]
|
||||
|
||||
self.service.generate_tags = slow
|
||||
resp = await self._start_job(existing="merge")
|
||||
job = self.api._jobs.get((await resp.json())["job"]["id"])
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, started.wait, 30)
|
||||
cancel = await self.client.post(f"/prompt_manager/jobs/{job.id}/cancel")
|
||||
self.assertTrue((await cancel.json())["cancellation_requested"])
|
||||
finally:
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), 30)
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.status_line, "Cancelled. Tags already added were kept.")
|
||||
self.assertEqual(len(calls), 1)
|
||||
tagged = [n for n, pid in ids.items() if "slow" in self._tags(pid)]
|
||||
self.assertEqual(len(tagged), 1)
|
||||
self.assertIsNone(model_gate.holder())
|
||||
self.assertEqual(self.service.unload_calls, 0)
|
||||
|
||||
async def test_job_refused_while_legacy_batch_holds(self):
|
||||
async with self.api._jobs.hold(
|
||||
"autotag", "Tag output images (legacy page)", admission="records"
|
||||
@@ -2103,41 +1752,6 @@ class TestTagOutputImagesJob(BatchJobTestCase):
|
||||
self.assertEqual(data["running"]["title"], "Tag output images (legacy page)")
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_legacy_refused_while_job_runs(self):
|
||||
self._seed()
|
||||
started, proceed = threading.Event(), threading.Event()
|
||||
|
||||
def slow(image_path, **kwargs):
|
||||
started.set()
|
||||
proceed.wait(30)
|
||||
return ["slow"]
|
||||
|
||||
self.service.generate_tags = slow
|
||||
resp = await self._start_job(existing="merge")
|
||||
job = self.api._jobs.get((await resp.json())["job"]["id"])
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, started.wait, 30)
|
||||
self.assertTrue(model_gate.holder().startswith("autotag-job:"))
|
||||
legacy = await self.client.post("/prompt_manager/autotag/start")
|
||||
# Both legacy pages read a stream: the refusal is one error frame
|
||||
# that names the running job.
|
||||
self.assertEqual(legacy.status, 200)
|
||||
self.assertEqual(
|
||||
self._sse_events(await legacy.text()),
|
||||
[
|
||||
{
|
||||
"type": "error",
|
||||
"status": "error",
|
||||
"message": "Tag output images already running",
|
||||
}
|
||||
],
|
||||
)
|
||||
unload = await self.client.post("/prompt_manager/autotag/unload")
|
||||
self.assertEqual((await unload.json())["operation"], "Tag output images")
|
||||
finally:
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), 30)
|
||||
|
||||
async def test_job_releases_gate_when_start_is_refused(self):
|
||||
class Running:
|
||||
id = "other"
|
||||
@@ -2185,7 +1799,7 @@ class TestTagOutputImagesJob(BatchJobTestCase):
|
||||
("database folder", "the safety backup", 10, 1),
|
||||
10,
|
||||
)
|
||||
with patch("py.api.autotag_routes.disk_plan", return_value=short):
|
||||
with patch("py.api.image_tag_jobs.disk_plan", return_value=short):
|
||||
resp = await self._start_job()
|
||||
data = await resp.json()
|
||||
self.assertEqual(resp.status, 507)
|
||||
@@ -2203,116 +1817,6 @@ class TestTagOutputImagesJob(BatchJobTestCase):
|
||||
inside = [row for row in rows if row["root_index"] == 0]
|
||||
self.assertTrue(inside)
|
||||
|
||||
async def test_native_batch_leaves_custom_prompt_unchanged(self):
|
||||
self._seed()
|
||||
self.service.custom_prompt = "legacy page prompt"
|
||||
await self._run_job(existing="merge", model_type="gguf")
|
||||
self.assertEqual(self.service.custom_prompt, "legacy page prompt")
|
||||
prompts = {call[1] for call in self.service.generate_calls}
|
||||
self.assertEqual(prompts, {"Tags please."})
|
||||
|
||||
|
||||
# The legacy batch's per-image frames over BatchJobTestCase's library, newest
|
||||
# first, with every throttle check passing (pinned from main's generator).
|
||||
GOLDEN_LEGACY_ROWS = [
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 24,
|
||||
"status": "Processing 1/6...",
|
||||
"processed": 1,
|
||||
"tagged": 1,
|
||||
"skipped": 0,
|
||||
"skipped_outside_roots": 0,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 38,
|
||||
"status": "Skipping 2/6 (outside allowed directories)...",
|
||||
"processed": 1,
|
||||
"tagged": 1,
|
||||
"skipped": 1,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 52,
|
||||
"status": "Skipping 3/6 (file missing)...",
|
||||
"processed": 1,
|
||||
"tagged": 1,
|
||||
"skipped": 2,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 66,
|
||||
"status": "Processing 4/6...",
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 2,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 80,
|
||||
"status": "Skipping 5/6 (prompt already processed)...",
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 3,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 95,
|
||||
"status": "Skipping 6/6 (already tagged)...",
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 4,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
class TestLegacyBatchFrames(BatchJobTestCase):
|
||||
|
||||
async def test_legacy_batch_events_unchanged(self):
|
||||
"""Golden frames of the legacy batch over the seeded library."""
|
||||
self._seed()
|
||||
ticks = iter(range(1000)) # every throttle check sees a second pass
|
||||
with patch("py.api.autotag_routes._monotonic", side_effect=lambda: next(ticks)):
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/autotag/start?model_type=wd14-vit&skip_tagged=true"
|
||||
)
|
||||
frames = self._sse_events(await resp.text()) # the stream runs here
|
||||
self.assertEqual(
|
||||
frames,
|
||||
[
|
||||
{"type": "progress", "progress": 0, "status": "Loading model..."},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 5,
|
||||
"status": "Model loaded. Fetching all images from database...",
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 10,
|
||||
"status": "Found 6 images. Processing...",
|
||||
},
|
||||
*GOLDEN_LEGACY_ROWS,
|
||||
{
|
||||
"type": "complete",
|
||||
"progress": 100,
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 4,
|
||||
"errors": 0,
|
||||
"skipped_outside_roots": 1,
|
||||
"status": "Complete",
|
||||
"model_status": "Model kept in memory",
|
||||
"model_loaded": True,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class TestTagOutputImagesEdges(BatchJobTestCase):
|
||||
|
||||
@@ -2342,7 +1846,7 @@ class TestTagOutputImagesEdges(BatchJobTestCase):
|
||||
async def test_unexpected_error_fails_the_job_with_a_reason(self):
|
||||
self._seed()
|
||||
with patch.object(
|
||||
self.api.db, "autotag_batch_candidates", side_effect=[[], RuntimeError("x")]
|
||||
self.api, "_autotag_prepass", side_effect=RuntimeError("/srv/x")
|
||||
):
|
||||
job = await self._run_job(existing="merge")
|
||||
self.assertEqual(job.state, "failed")
|
||||
|
||||
+24
-92
@@ -1,5 +1,6 @@
|
||||
"""Tests for the core API module (py/api/__init__.py): helpers and base routes."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
@@ -165,8 +166,19 @@ class TestGzipMiddleware(AioHTTPTestCase):
|
||||
await resp.write_eof()
|
||||
return resp
|
||||
|
||||
async def declared_length(request):
|
||||
# The log download (#329) declares its length before the middleware
|
||||
# compresses it.
|
||||
body = b"x" * 3000
|
||||
return web.Response(
|
||||
body=body,
|
||||
content_type="text/plain",
|
||||
headers={"Content-Length": str(len(body))},
|
||||
)
|
||||
|
||||
app = web.Application(middlewares=[api_module._gzip_middleware])
|
||||
app.router.add_get("/prompt_manager/big", big_json)
|
||||
app.router.add_get("/prompt_manager/declared", declared_length)
|
||||
app.router.add_get("/prompt_manager/small", small_json)
|
||||
app.router.add_get("/prompt_manager/encoded", already_encoded)
|
||||
app.router.add_get("/prompt_manager/binary", binary)
|
||||
@@ -193,6 +205,18 @@ class TestGzipMiddleware(AioHTTPTestCase):
|
||||
|
||||
self.assertIn(b'"data"', gzip.decompress(await resp.read()))
|
||||
|
||||
async def test_compressed_body_declares_its_own_length(self):
|
||||
# A stale Content-Length (the uncompressed size) makes a browser wait
|
||||
# for bytes that never come (#329: Download log hung on any log > 1 KB).
|
||||
resp = await self._get("/prompt_manager/declared")
|
||||
self.assertEqual(resp.headers.get("Content-Encoding"), "gzip")
|
||||
# Bounded: a stale length would otherwise wait for the missing bytes.
|
||||
body = await asyncio.wait_for(resp.read(), timeout=10)
|
||||
self.assertEqual(int(resp.headers["Content-Length"]), len(body))
|
||||
import gzip
|
||||
|
||||
self.assertEqual(gzip.decompress(body), b"x" * 3000)
|
||||
|
||||
async def test_small_body_not_compressed(self):
|
||||
resp = await self._get("/prompt_manager/small")
|
||||
self.assertIsNone(resp.headers.get("Content-Encoding"))
|
||||
@@ -811,7 +835,6 @@ class TestMetadataParsing(unittest.TestCase):
|
||||
self.assertEqual(parsed["positive_prompt"], "a cat")
|
||||
self.assertEqual(parsed["negative_prompt"], "dog")
|
||||
self.assertEqual(parsed["parameters"]["parameters"], meta["parameters"])
|
||||
self.assertEqual(self.api._extract_readable_prompt(parsed), "a cat")
|
||||
|
||||
def test_parse_comfyui_fields(self):
|
||||
meta = {
|
||||
@@ -832,97 +855,6 @@ class TestMetadataParsing(unittest.TestCase):
|
||||
self.assertEqual(parsed["prompt"], "plain text")
|
||||
self.assertEqual(parsed["workflow"], {"nodes": []})
|
||||
|
||||
def test_extract_readable_prompt_variants(self):
|
||||
extract = self.api._extract_readable_prompt
|
||||
self.assertEqual(extract({"prompt": "plain"}), "plain")
|
||||
self.assertEqual(extract({"prompt": ["a", "", "b"]}), "a b")
|
||||
self.assertEqual(extract({"prompt": 42}), "42")
|
||||
self.assertEqual(
|
||||
extract({"prompt": None, "parameters": {"positive": ["x", "y"]}}), "x y"
|
||||
)
|
||||
self.assertIsNone(extract({"prompt": {}, "parameters": {}}))
|
||||
|
||||
graph = {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "pos"}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": "neg"}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["2", 0]},
|
||||
},
|
||||
}
|
||||
self.assertEqual(extract({"prompt": graph}), "pos")
|
||||
self.assertEqual(extract({"prompt": {"x": 1}, "workflow": graph}), "pos")
|
||||
|
||||
def test_node_inputs_and_text_lookup(self):
|
||||
self.assertEqual(self.api._get_node_inputs("nope"), {})
|
||||
self.assertEqual(self.api._get_node_inputs({"inputs": 5}), {})
|
||||
self.assertEqual(self.api._get_node_inputs({"inputs": {"a": 1}}), {"a": 1})
|
||||
listed = self.api._get_node_inputs(
|
||||
{"inputs": [{"name": "text", "link": None}, {"nope": 1}, "junk"]}
|
||||
)
|
||||
self.assertEqual(list(listed), ["text"])
|
||||
|
||||
self.assertIsNone(self.api._find_text_in_node(None))
|
||||
self.assertEqual(self.api._find_text_in_node({"inputs": {"text": "hi"}}), "hi")
|
||||
self.assertEqual(
|
||||
self.api._find_text_in_node(
|
||||
{"type": "CLIPTextEncode", "widgets_values": ["w"]}
|
||||
),
|
||||
"w",
|
||||
)
|
||||
self.assertIsNone(
|
||||
self.api._find_text_in_node(
|
||||
{"type": "CLIPTextEncode", "widgets_values": [" "]}
|
||||
)
|
||||
)
|
||||
self.assertIsNone(self.api._find_text_in_node({"type": "KSampler"}))
|
||||
|
||||
def test_positive_prompt_from_nodes_array_and_fallbacks(self):
|
||||
extract = self.api._extract_positive_prompt_from_comfyui_data
|
||||
self.assertIsNone(extract("x"))
|
||||
self.assertIsNone(extract({}))
|
||||
self.assertIsNone(extract({"nodes": []}))
|
||||
|
||||
workflow = {
|
||||
"nodes": [
|
||||
{"id": 1, "type": "CLIPTextEncode", "widgets_values": ["pos"]},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "CLIPTextEncode",
|
||||
"widgets_values": ["neg"],
|
||||
"title": "Negative",
|
||||
},
|
||||
{"id": 3, "type": "KSampler", "inputs": [{"name": "seed", "link": 4}]},
|
||||
"junk",
|
||||
]
|
||||
}
|
||||
self.assertEqual(extract(workflow), "pos")
|
||||
|
||||
only_negative = {
|
||||
"nodes": [
|
||||
{
|
||||
"id": 2,
|
||||
"type": "CLIPTextEncode",
|
||||
"widgets_values": ["neg"],
|
||||
"title": "neg",
|
||||
},
|
||||
]
|
||||
}
|
||||
self.assertEqual(extract(only_negative), "neg")
|
||||
|
||||
bad_link = {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "pos"}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": "oops", "negative": []},
|
||||
},
|
||||
"4": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["zz", 0], "negative": []},
|
||||
},
|
||||
}
|
||||
self.assertEqual(extract(bad_link), "pos")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user