Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2f19908d9a | ||
|
|
954fee739b | ||
|
|
d7893b6e8e | ||
|
|
f707a1abe5 | ||
|
|
c160b186d7 | ||
|
|
2fe9686ffc | ||
|
|
116a66f755 |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"required_status_checks": {
|
||||
"strict": false,
|
||||
"strict": true,
|
||||
"contexts": [
|
||||
"py3.12 on ubuntu-latest",
|
||||
"py3.13 on ubuntu-latest",
|
||||
|
||||
@@ -4,7 +4,6 @@ permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
+14
-38
@@ -4,7 +4,6 @@ permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
@@ -64,18 +63,16 @@ jobs:
|
||||
comfyui:
|
||||
name: real ComfyUI compatibility
|
||||
runs-on: ubuntu-latest
|
||||
# Chromium and its system libraries come from the pinned Playwright image (tag = @playwright/test in
|
||||
# tools/vendor/package-lock.json; tests/vendor/browser-image.test.mjs checks it). The runner install
|
||||
# pulled them from Ubuntu's apt mirror on every run, and three runs (#349, #378, #382) crawled at
|
||||
# ~26 kB/s for 11-12 minutes until the job hit its limit after every test had passed.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 24
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: npm
|
||||
cache-dependency-path: tools/vendor/package-lock.json
|
||||
- name: Read the reviewed runtime pins
|
||||
id: runtime
|
||||
run: |
|
||||
@@ -100,42 +97,21 @@ jobs:
|
||||
ref: ${{ steps.runtime.outputs.kikoRevision }}
|
||||
path: .ci-kikotools
|
||||
persist-credentials: false
|
||||
# The image's own Python and Node 24: setup-python/setup-node install the runner's toolcache builds,
|
||||
# whose entry points name host paths that do not exist in the container. The image has no ensurepip
|
||||
# and the job has no root, so a checksum-pinned pip wheel bootstraps a user-space venv. The image
|
||||
# Python must be the reviewed host Python, so a pin or image bump that disagrees fails here.
|
||||
- name: Python environment on the reviewed host Python
|
||||
env:
|
||||
PIP_WHEEL: https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl
|
||||
PIP_SHA256: 71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e
|
||||
PINNED_PYTHON: ${{ steps.runtime.outputs.pythonVersion }}
|
||||
run: |
|
||||
image="$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])')"
|
||||
[ "$image" = "$PINNED_PYTHON" ] || { echo "::error::image Python $image is not the pinned $PINNED_PYTHON"; exit 1; }
|
||||
venv="$RUNNER_TEMP/venv" wheel="$RUNNER_TEMP/pip-26.2.1-py3-none-any.whl"
|
||||
python3 -m venv --without-pip "$venv"
|
||||
curl -fsSL --retry 3 -o "$wheel" "$PIP_WHEEL"
|
||||
echo "$PIP_SHA256 $wheel" | sha256sum -c -
|
||||
"$venv/bin/python" "$wheel/pip" install --no-index --no-cache-dir "$wheel"
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
# pip's download cache (torch alone is ~200 MB); inside the container ~ is the job's HOME (/github/home).
|
||||
- name: Cache pip downloads
|
||||
uses: actions/cache@v6
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: pip-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('requirements-dev.txt', '.ci-comfyui/requirements.txt', 'tools/vendor/comfyui-runtime.json') }}
|
||||
python-version: ${{ steps.runtime.outputs.pythonVersion }}
|
||||
cache: pip
|
||||
cache-dependency-path: |
|
||||
requirements-dev.txt
|
||||
.ci-comfyui/requirements.txt
|
||||
tools/vendor/comfyui-runtime.json
|
||||
- name: Install CPU runtime and test dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}" --index-url https://download.pytorch.org/whl/cpu
|
||||
pip install -r .ci-comfyui/requirements.txt -r requirements-dev.txt "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}"
|
||||
# npm's download cache, keyed on the lockfile. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- run: npx --prefix tools/vendor playwright install --with-deps chromium
|
||||
- name: Classic and Nodes 2.0 startup and compatibility
|
||||
env:
|
||||
PM_COMFY_ROOT: ${{ github.workspace }}/.ci-comfyui
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
name: Viewer.js vendor and compatibility
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
|
||||
+35
-446
@@ -745,55 +745,11 @@ with a literal true `strip_newlines` is `replace("\n", "").strip()`;
|
||||
(`N` = `inputcount`) after the literal delimiter unless missing or exactly `""`, and
|
||||
ignores later inputs. A linked setting is `unsupported-node`; a list output
|
||||
(`return_list` true or linked) and `PromptSearchList` are `runtime-value` ("Text is
|
||||
decided at run time"), never one list item. Every
|
||||
decided at run time"), never one list item. Unverified text nodes (LoRAfromText,
|
||||
RegexReplace, XYZPrompt, ImpactWildcardProcessor) stay `unsupported-node`. Every
|
||||
intermediate text is length-checked and charged to the scanned-character budget. If
|
||||
the table cannot be loaded, graph inspection is `unresolved` ("Inspection node
|
||||
semantics are unavailable") and ingestion is unaffected, because it never imports the table.
|
||||
Unknown nodes pass through (#381, maintainer-approved 2026-10-27; no per-node
|
||||
inventory): an output whose class the table, its `facts`, `lineage` and the code's
|
||||
adapters never name is read through, not refused. A `text` input is its text (the
|
||||
legacy encoder rule, so RegexReplace or LoRAfromText show their input text), except
|
||||
that a string output of a node with a seed input (`seed`, `noise_seed`,
|
||||
`sampling_mode.seed`) and no linked `conditioning`/`positive`/`negative` input is
|
||||
generated at run time (a language model such as TextGenerate, a wildcard):
|
||||
`runtime-value` naming it. Else, at output slot 0 or 1 of a node with a `positive` or
|
||||
`negative` input, only `positive` (slot 1, `negative`) is followed, nothing when it
|
||||
is not linked; else a linked `conditioning` alone; else every linked input except
|
||||
the table's `pass_through.skip_inputs` (core type inputs: model, clip, vae, image,
|
||||
latent, ...), each traced with kind `any`. A followed input with no text adds
|
||||
nothing; limits and selector choices propagate; another failure (`runtime-value`,
|
||||
`composite`, `malformed`) blocks, unless a PromptManager node was reached: this
|
||||
repository's own nodes are authoritative, and then only their texts count. One
|
||||
distinct text resolves; several make the sampler `ambiguous` "Several prompts reach
|
||||
the sampler" with one candidate per distinct outcome, in input order (at most 32,
|
||||
else `work-limit`). A PromptManagerPair's execution record is what ran: while its
|
||||
inputs are checked against it, a walk through an unknown node that fails, diverges
|
||||
or passes text through is unverifiable and never overrides the record. No text
|
||||
at all is `unsupported-node` naming the outermost unknown node. An unlisted output
|
||||
of a listed class (a sampler, a loader, a lineage node) stays `unsupported-node`, so
|
||||
the walk never reads another stage's prompt. `INSPECTION_RESOLVER_VERSION` is 2 for
|
||||
this rule; ingestion keeps the default profile (`RESOLVER_VERSION` 1). The legacy
|
||||
parity gate (`tests/test_legacy_parity_gate.py`) records, per corpus (the
|
||||
generation-params vectors, the inspection parity fixtures, the local resolver golden
|
||||
and selection cases, the unknown-node shapes in `tests/unknown_node_graphs.py`),
|
||||
every graph whose inspection positive equals the legacy parser's; a graph may join
|
||||
that list (`python3 -m tests.test_legacy_parity_gate --write`), never leave it. A graph
|
||||
may read differently from legacy only when `LEGACY_WRONG` there states why (legacy's
|
||||
first-sampler pick on a branch the file name identifies).
|
||||
Saver anchoring (#381): an inspected file's name (`path.name` on the server, the chosen
|
||||
file's name locally) anchors an unanchored inspection on the one node whose
|
||||
`filename_prefix` (literal, or traced as a string) has a last path part that makes the
|
||||
file's stem `<prefix>_<digits>` with an optional trailing `_`. That node's linked inputs
|
||||
are walked like a detailer's image ancestry, through any image node, and the listed
|
||||
samplers found are the candidates. No match, several matches, a walk that fails or no
|
||||
sampler found means no anchor: nothing is guessed and the file stays "several possible
|
||||
prompts". Ingestion ignores file names. Shared-conditioning stages: in a sampler's
|
||||
latent walk, an earlier sampler reading the very same conditioning links (positive and
|
||||
negative, or a guider; compared by node ID text and slot) is the same prompt, not an
|
||||
unattributed stage, when it is a plain first stage: not a detailer, no prompt override
|
||||
set, and no listed sampler in its own latent ancestry (else it stays unattributed) (`tests/test_resolver_saver_anchor.py`, shapes in
|
||||
`tests/saver_anchor_graphs.py` from the maintainer's `gpu_unlock_bench`, dataset and
|
||||
ANIMA runs; browser parity through the `anchor-*` cases, which carry `__file_name`).
|
||||
|
||||
`tools/inspection_corpus.py --root DIR --out JSONL [--workers N] [--baseline JSONL]
|
||||
[--summary TXT] [--code TREE]` measures a media folder read-only against the #240
|
||||
@@ -1245,8 +1201,7 @@ Recorded deviations on this page (spec §8):
|
||||
`#bulkbar` shows "N selected" and only the real actions in the concept's order (Add
|
||||
tags, then D85's Auto-tag selected, Set category, Export metadata, Clear). Add to
|
||||
collection waits for collections; "Remove links…" is dropped. (Amended by #249 PR D:
|
||||
Add tags ships, see "Bulk tagging from the selection bar"; #392 adds range, keyboard,
|
||||
touch and "all N matching" selection, see "Selecting many images", D135.) The selection survives Load more and
|
||||
Add tags ships, see "Bulk tagging from the selection bar".) The selection survives Load more and
|
||||
bounded re-mounting and is cleared whenever the collection is replaced (search,
|
||||
category, sort, Filters, source change, rescan, reload). A return to Images restores
|
||||
the saved collection unreplaced and keeps its selection; a full page reload starts
|
||||
@@ -1684,9 +1639,6 @@ targets of the longest legal path (1024 code points, even escaped as `\uXXXX`) f
|
||||
unchanged, `-` skipped and `!` error.
|
||||
- The progress line is the scan's: "Processing file N of M · about X min left", the
|
||||
estimate (D56's `_time_left`) shown once two files are done, each file one batch.
|
||||
An ended job never keeps it: its status line becomes "<Add tags | Auto-tag>
|
||||
<finished | cancelled> · N files processed" (or the storage-restored sentence),
|
||||
pinned by `test_an_ended_job_shows_no_time_left` (#379 live check).
|
||||
- Download report saves `prompt-manager-image-tags-<YYYYMMDD-HHMM>.json`.
|
||||
- The start toasts are "Tagging started." / "Auto-tagging started." (D85); the end
|
||||
toasts follow the design's pattern with the job's counts, as the scan's: "<title>
|
||||
@@ -1723,65 +1675,6 @@ busy". Units:
|
||||
| Copy without a concept line: no tags, start fallbacks, the panel heading | D118 |
|
||||
| Activity item and job view for image tagging jobs | D89 (plan D66) |
|
||||
|
||||
### Selecting many images (#392)
|
||||
|
||||
**D135** (beyond the concept, which draws only the per-card checkbox): Images selects
|
||||
like a file manager, in Grid, List (where the concept hides the checkbox) and Masonry,
|
||||
over every loaded batch. The selection is a set of image keys in display order, so
|
||||
ranges include cards that bounded rendering has unmounted.
|
||||
|
||||
- Clicking a checkbox toggles it, and that image becomes the anchor. Ctrl/Cmd+click
|
||||
anywhere on a card (image, star, Info, Tag) also toggles it and makes it the anchor;
|
||||
nothing opens, inspects or changes the favorite.
|
||||
- Shift+click on a card selects the range from the anchor to it. Shift+click on a
|
||||
checkbox gives its range the box's new state, so it can clear a range. The anchor
|
||||
stays where it is. Without an anchor, the clicked image becomes the anchor.
|
||||
- In the grid, Ctrl/Cmd+A selects every loaded image; in the search box it keeps its
|
||||
text meaning. Shift+Arrow, Shift+Home and Shift+End move focus as the plain keys do,
|
||||
then select from the anchor (the starting card when there is none) to the new card.
|
||||
Escape in the grid clears the selection. Escape in the bar clears it too, and focus
|
||||
moves to the heading, as after Clear.
|
||||
- Touch: a 500 ms long-press (at most 10 px of movement) selects the card and turns on
|
||||
touch selection mode. The context menu during the press and the click after it are
|
||||
swallowed, and the card has no image callout. Taps then toggle cards until the
|
||||
selection is empty. A keyboard Enter or Space keeps its plain meaning.
|
||||
- A plain click keeps its meaning: the image opens the viewer, and Info inspects.
|
||||
- When every loaded image is checked and the listing has more, the bar offers
|
||||
**Select all N matching** right after the count. Pressing it gives **All N matching
|
||||
selected**: later batches arrive checked, and unchecking any image goes back to the
|
||||
loaded images minus that one. Clear, Escape, and a new search, filter, sort or rescan
|
||||
end it. A refresh after a tag save keeps it. Return visits keep the loaded images,
|
||||
and the offer shows again.
|
||||
- In that mode, Add tags and Auto-tag selected start the `image-tags` job with
|
||||
`filter` (the listing's own query as an object: `q`, `subfolder`, `category`,
|
||||
`tags`, `root`, `media`, `sort`, `favorites`, and with a Source the loaded root's
|
||||
`root_scope`) instead of `targets`. The server resolves the filter at start, in
|
||||
listing order, using the listing's own admission code (`_admitted_entries`). That
|
||||
means "all matching" when the action starts: an output written since then that
|
||||
matches is included. A `root_scope` that no longer names its root (the sources were
|
||||
renumbered) answers 409 `source-changed`. The limit is 10,000 images (`MAX_LISTING_SELECTION`).
|
||||
Over the limit, the client toasts "Narrow the view to at most 10,000 images." and the
|
||||
server answers 400 `too-many-images`. No match is 400 `no-images`. Auto-tag in this
|
||||
mode does not offer Review each image, because the review walks images in the
|
||||
browser. Set category posts `{images_filter, category}` to `/prompt_manager/bulk/category`.
|
||||
The server sets the category on the distinct linked positives and answers
|
||||
`prompt_count` and `skipped`. A listing whose `positive_count` is 0 toasts the
|
||||
nothing-linked sentence and opens no dialog, and over 10,000 matches it toasts the
|
||||
limit sentence. Export metadata reads every image in
|
||||
the browser, so it is `aria-disabled` in this mode, and its `title` says why.
|
||||
|
||||
Tests: `tests/browser/workspace-images.spec.mjs` ("Shift+click selects a range across
|
||||
an appended batch in Grid, List and Masonry", "Ctrl/Cmd+click toggles a card without
|
||||
opening it, and a plain click still opens the viewer", "Ctrl/Cmd+A, Select all N
|
||||
matching and Add tags tag every matching image on the server", "Shift+arrow keys extend
|
||||
the selection from the focused card, and Escape in the bar clears it", "a touch
|
||||
long-press starts selection mode, taps then toggle, and an empty selection ends it");
|
||||
units `tests/js/workspace-images-multiselect.test.js`,
|
||||
`tests/js/workspace-images-tagging-bulk.test.js` ("all matching sends the listing
|
||||
filter instead of targets"), `tests/js/workspace-images-state.test.js`
|
||||
(`listingFilter`); Python `tests/test_image_tag_jobs.py` (`TestFilterStart`) and
|
||||
`tests/test_api_workspace_images.py` (`TestListingFilterSelection`).
|
||||
|
||||
Native image tagging (#249 PR B routes, PR C UI): `POST
|
||||
/prompt_manager/image-annotations/observe` (capture one listed file, its saved set and
|
||||
`tag_revision`), `GET`/`DELETE /prompt_manager/image-annotations/capture/<token>`
|
||||
@@ -2765,8 +2658,7 @@ the connection's `total_changes` is 0 at close and the data reads back unchanged
|
||||
`-shm` bytes are not compared: every WAL reader rewrites its read marks there, so no
|
||||
reader can keep it byte-identical (`test_health_check_never_writes`).
|
||||
|
||||
Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107, D109–D110 and D111–D114; D121–D134 assigned). D94,
|
||||
the per-section hand-off panel, was retired by S12 (#329): every section is native.
|
||||
Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107 and D111–D114; D121–D134 assigned):
|
||||
|
||||
- **D92** (shell): no "Sample configuration" chip. The Save status is truthful: "No
|
||||
unsaved changes.", "Unsaved changes.", "Saved in this browser." / "Saved to the server
|
||||
@@ -2798,6 +2690,10 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
"N output sources" with "Monitoring and enrichment share one source list.", prefixed
|
||||
by "M of N configured sources available." or "None configured is available; ComfyUI's
|
||||
output folder is used." when that is the case.
|
||||
- **D94** (all sections): a section not delivered yet shows the hand-off panel "Your
|
||||
current tools are ready" with a link to each legacy tool that still owns it (the
|
||||
plan's legacy coverage table); retired section by section, gone after S12. Since S11
|
||||
(#328) only Diagnostics & logs hands off (`SECTION_HANDOFFS`, to the admin page).
|
||||
- **D95** (health report): the concept's `health-check` dialog titled "Health report",
|
||||
with no sample note. It lists every check in the server's order: Integrity check,
|
||||
Foreign keys & pair references, Canonical hashes, Pair uniqueness (added), Missing
|
||||
@@ -2828,7 +2724,7 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
- **D123** (Maintenance): the concept's Maintenance notice
|
||||
("Check first. Review changes. Then apply." + Run health checks) was delivered ahead of
|
||||
the rest of Maintenance (S2), as the native place that runs the health check; since S7
|
||||
the section's two surfaces follow it and its hand-off panel is gone. A press shows "Checking…" (`aria-disabled`) and opens the report; a
|
||||
the section's two surfaces follow it and its D94 hand-off is gone. A press shows "Checking…" (`aria-disabled`) and opens the report; a
|
||||
failed check toasts "The health check could not finish. Nothing was changed."
|
||||
- **D97** (Appearance): added rows for the legacy gallery settings the concept leaves
|
||||
out, after its own rows: "Show image details on cards" (`showImageInfo`; off hides a
|
||||
@@ -2949,11 +2845,6 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
and the media hashes, `GET /__test__/system/digest`). `plan dialogs match the concept`
|
||||
(system-maintenance-concept.spec, both themes and sizes) compares each of the four
|
||||
operations' dialogs with the concept's, mapped by these values only.
|
||||
S12 (#329): the `rotated_logs` plan (Diagnostics & logs › Review log cleanup, `h3`
|
||||
"Rotated log copies") changes no library record: its Backup reads "Not needed: no
|
||||
library record changes" (`backup.policy: "none"`), it has no D55 backup box, and its
|
||||
`details` reads "Listed files (N)" (`a plan without a database change reads Not needed
|
||||
and shows no backup box`, `rotated_logs details read Listed files`).
|
||||
- **D101** (Maintenance, S7): Duplicate image files → Find duplicates starts the
|
||||
duplicate search (Activity, D111); its job view adds "Review groups" after Download
|
||||
report once it finished with groups. The concept's `duplicates` dialog ("Review
|
||||
@@ -3200,104 +3091,37 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
again"; an Unload that found nothing loaded only reads the status again. At phone width
|
||||
the concept's 120px select cap cuts every engine label, so `#tag-model` alone keeps its
|
||||
natural width there and its row wraps the select under the text (`workspace.css`
|
||||
`/* D132 */`). The concept's cap cuts Existing tags too ("Replace after review"); by the
|
||||
maintainer's decision (#398) that parity is not kept: `#tag-existing` never shrinks for
|
||||
its row text and at phone width keeps its natural width beside the text (its row does
|
||||
not wrap; the text column narrows). 1600 is as the concept draws it. The comparison
|
||||
expects `selectsFit` true for every select and measures each row's `flexWrap` and
|
||||
whether its control sits under the text. Pinned by `auto-tagging matches the concept`
|
||||
(both themes and sizes), `auto-tagging at 390 wraps only the engine row`, `auto-tagging
|
||||
selects fit every option at 390/320 wide` (every option of both selects, both themes, no
|
||||
page scroll) and the mutation check (a wrapped row, a squeezed engine and Existing tags
|
||||
select) (system-tagging-concept.spec), and the model, Download, Unload and Preview
|
||||
`/* D132 */`); Existing tags stays as the concept draws it, cut at 390 as there. The
|
||||
comparison maps `selectsFit` only for the engine select and measures each row's
|
||||
`flexWrap` and whether its control sits under the text. Pinned by `auto-tagging matches
|
||||
the concept` (both themes and sizes), `auto-tagging at 390 wraps only the engine row`,
|
||||
`auto-tagging engine select fits at 320 wide` and the mutation check (it includes a
|
||||
wrapped row) (system-tagging-concept.spec), and the model, Download, Unload and Preview
|
||||
tests of workspace-system-tagging.spec.
|
||||
- **D133** (Auto-tagging, S11): a "Suggestions per image" row (`#tag-suggestions`,
|
||||
number 1–100, host key `autotag_suggestion_limit`) follows the two confidence rows:
|
||||
the persisted count the Tag image dialog's Suggestions field (D91) starts from. Pinned
|
||||
by the comparison's mapping (it inserts the row) and `defaults save through Save
|
||||
preferences and the Tag image dialog uses them`.
|
||||
- **D134** (Auto-tagging, S11, OQ2/OQ3/OQ7/OQ8; amended by #379): an `.operation` "Tag
|
||||
output images" with "Review" follows `.row-actions`: the per-image batch (#379), whose
|
||||
description reads "Tag each output image from its own pixels with the model above.
|
||||
Prompts are not changed. Review the scope first.". Review is
|
||||
- **D134** (Auto-tagging, S11, OQ2/OQ3/OQ7/OQ8): an `.operation` "Tag output images"
|
||||
with "Review" follows `.row-actions`: the legacy prompt-tag batch. Review is
|
||||
`aria-disabled` (the press explains) while the chosen engine is not downloaded or a
|
||||
value it uses is invalid; otherwise it posts `/autotag/batch/preview` with the drafted
|
||||
engine and policy (read-only) and opens "Review tag output images" in the concept's
|
||||
record-changing review (its `maintenancePlan` shell): `h3` "Tag output images", the
|
||||
`dl.meta-grid` Model, Existing tags, Images, Already auto-tagged, Images to tag
|
||||
("about N"), Time, Settings ("General <g> · Character <c>" or "Template of N
|
||||
characters", then " · up to L tags per image" with the preview's saved
|
||||
`suggestion_limit`) and Backup, the estimate sentence ("Nothing is written until you
|
||||
start. The number to tag is an estimate: images changed since they were tagged are
|
||||
tagged again, and missing files are skipped. Activity reports what was tagged.") in the
|
||||
`p.small`, the D55 backup box and Cancel / "Start tagging", which starts the
|
||||
cancellable `autotag` job and opens it. Time (DEC11) is `m = max(1,
|
||||
ceil(estimate_seconds / 60))` read "about m min" under 90 and "about H h M min" (or
|
||||
"about H h") from 90, followed by " on a processor; a graphics card can be faster."
|
||||
or, when the preview says WD14 runs on CUDA, " on the graphics card."; the language
|
||||
models read "Language-model engines take longer per image. Activity shows the time
|
||||
left once tagging starts.". The option label "Skip already tagged" is the concept's;
|
||||
for this batch it skips images already auto-tagged (hand tags do not count). Replace
|
||||
after review, a model not on disk, a short disk, no image and nothing to tag each show
|
||||
the rows with only Cancel and the reason ("Replacing tags needs a review of each
|
||||
image. Use Preview tagging review, or choose Skip or Merge for a batch run.", "The
|
||||
output folders contain no images to tag.", "Every output image is already
|
||||
auto-tagged.", "Not enough free disk space: the safety backup needs about X MB and Y
|
||||
MB is free. Nothing is written until there is room.") in the close-only review
|
||||
Integrations shares (`openCloseOnlyReview`). Pinned by
|
||||
`dl.meta-grid` Model, Existing tags, Images linked, Prompts, Already tagged, Prompts
|
||||
to tag ("up to N"), Settings ("General <g> · Character <c>" or "Template of N
|
||||
characters") and Backup, the upper-bound sentence in the `p.small`, the D55 backup box
|
||||
and Cancel / "Start tagging", which starts the cancellable `autotag` job and opens it.
|
||||
Replace after review, a model not on disk, a short disk, no linked image and nothing to
|
||||
tag each show the rows with only Cancel and the reason ("Replacing tags needs a review
|
||||
of each image. Use Preview tagging review, or choose Skip or Merge for a batch run.",
|
||||
"Every prompt with an output image is already tagged.", "Not enough free disk space:
|
||||
the safety backup needs about X MB and Y MB is free. Nothing is written until there is
|
||||
room.") in the close-only review Integrations shares (`openCloseOnlyReview`). Pinned by
|
||||
`tag output images review matches the concept`, its mutation check and `the
|
||||
Cancel-only review matches the concept review shell` (system-tagging-concept.spec) and
|
||||
the batch tests of workspace-system-tagging.spec; the mutation check renames each grid
|
||||
label in turn.
|
||||
- **D109** (Logs, S12, #329): the concept's Logs surface on the live server. The toolbar
|
||||
keeps "Level", Refresh and the download (renamed "Download log": the active log,
|
||||
`/logs/download/prompt_manager.log`, through the api client), with all six minimum
|
||||
levels (All levels, DEBUG, INFO, WARNING, ERROR, CRITICAL; the concept has three) and an
|
||||
added `label` "Lines" + `input#log-limit[type=number]` 1–500 (default 100): a value that
|
||||
is not a whole number 1–500 after trimming gets `aria-invalid` and sends nothing, a
|
||||
Level change included. The `pre` shows the in-memory buffer newest first as `HH:MM:SS
|
||||
LEVEL message` ("No log entries at this level." / "The log could not be read."); a
|
||||
newer read wins. Added after it, `p.small#log-stats[role=status]` "Log statistics":
|
||||
"N entries in memory · E errors · W warnings · F files · S on disk" (errors are ERROR +
|
||||
CRITICAL, singular for 1, links not counted). "Runtime log level" adds CRITICAL after
|
||||
the concept's INFO, WARNING, DEBUG, ERROR; it and "Console logging" are runtime drafts
|
||||
saved by Save preferences (`POST /logs/config`, only the changed keys) and read the
|
||||
server's values; their descriptions say "Applied until the server restarts." (the
|
||||
concept's say a gallery toggle is replaced and output stays quiet). Inside "Log files &
|
||||
retention", after the concept's line ("prompt_manager.log · <size> · rotated copies:
|
||||
N", N the `rotated` rows) and Review log cleanup (D100's `rotated_logs` plan): a
|
||||
`p.small` "Rotates at M MB · keeps K rotated copies · file logging on|off · memory
|
||||
buffer B entries" (read-only), an `.operation-list` of `/logs/files` (`h3` name, `p`
|
||||
"<size> · <YYYY-MM-DD HH:MM> · Active log | Rotated copy | Other file | Link, not
|
||||
downloaded", "Download" named "Download <name>" except for a link), and the danger
|
||||
`.operation` "Clear all logs" / "Empty the active log and delete its rotated copies." /
|
||||
"Clear all logs…", which confirms ("Empties prompt_manager.log, deletes N rotated
|
||||
copies (S) and clears the lines shown here. The other files in the log folder are
|
||||
kept. This cannot be undone."), runs `POST /logs/truncate`, toasts "Logs cleared · N
|
||||
rotated copies deleted." (plus " · K could not be deleted") and reads the viewer,
|
||||
statistics, files and configuration again. Pinned by workspace-system-diagnostics.spec's
|
||||
Task 5 and Task 6 tests.
|
||||
- **D110** (Diagnostics, S12, #329): the concept's Diagnostics surface with live values.
|
||||
The `div.meta-grid` `dd`s read one `GET /system/diagnostics`: Database "Readable ·
|
||||
<size> · N images" or "Not readable: <message>"; Sources "N sources · M watching", "N
|
||||
sources · K not available", or with none configured "ComfyUI output folder · watching
|
||||
| manual scan"; Viewer.js "<pinned version> · pinned vendor". Host renderers is D106's
|
||||
renderer line from the send bridge. Each reads "Loading…" first, "Busy: <job> is
|
||||
running." when an exclusive job refuses the read (D57's naming) and "Could not be
|
||||
read." on any other failure. Run checks opens the shared Health report (D95). The
|
||||
buttons drop the concept's "sample": "Download diagnostic report" saves a fresh report
|
||||
plus `host_renderers` as `prompt-manager-diagnostics-YYYYMMDD-HHMM.json` (local time)
|
||||
and toasts "Diagnostic report downloaded." ("Busy: <job> is running. Try again when it
|
||||
ends." / "The diagnostic report could not be downloaded. Nothing was saved to this
|
||||
device."); "Inspect" opens `#/inspect`, which reads a file and writes nothing. An added
|
||||
`.operation` "Test image link" / "Write a placeholder image record to check that
|
||||
linking works." / "Create test link…" keeps the legacy synthetic link (S12-11): it
|
||||
reads the most recent prompt on mount (`aria-disabled` with "Add a prompt first." or
|
||||
"The most recent prompt could not be read." added to the description), confirms in
|
||||
"Create test link" naming the prompt and how Missing-file links removes the record,
|
||||
and toasts "Test link created (image record N)." or "The test link could not be
|
||||
created. Nothing was changed.". Pinned by workspace-system-diagnostics.spec's Task 4
|
||||
tests.
|
||||
the batch tests of workspace-system-tagging.spec.
|
||||
- **D124** (inspector): with "Load metadata on selection" off, the Images panel and
|
||||
the viewer's details show "Metadata loads when you ask for it." and a "Load metadata"
|
||||
button instead of reading the file. The concept has no such state, so no concept
|
||||
@@ -3830,9 +3654,7 @@ threshold off that grid is refused ("General-tag confidence must be a multiple o
|
||||
0.05.", `validate_saved_threshold`; `draftProblem` holds back Preview tagging review and
|
||||
the batch Review with the same sentence) and falls back on load; an accepted one is
|
||||
stored exactly on the grid (#369). A per-request threshold keeps its own check: `/image-annotations/suggest` and
|
||||
`/jobs/autotag` take 0–1; the legacy `/autotag/start` passes its options through
|
||||
`validate_options` and starts the `autotag` job titled "Tag output images (legacy page)";
|
||||
the legacy `/autotag/single` only requires a number. `GET /autotag/models` reports the
|
||||
`/jobs/autotag` take 0–1; the legacy `/autotag/single` and `/autotag/start` only require a number. `GET /autotag/models` reports the
|
||||
saved values (`wd14_general_threshold`, `wd14_character_threshold`, `suggestion_limit`,
|
||||
`default_prompt`, `model_type`, `existing`), so the Tag image and Auto-tag selected
|
||||
dialogs start from them; a preset (the bulk review, Preview tagging review) wins.
|
||||
@@ -3858,96 +3680,19 @@ images", "Tag output images (legacy page)", "Tagging one image (legacy page)",
|
||||
`409 busy`; the legacy SSE route attaches to the same job and keeps its frames.
|
||||
- **Unload**: `POST /prompt_manager/autotag/unload` answers `{success, message}` or
|
||||
`409 model-busy`.
|
||||
- **Tag output images** (per image since #379; prompt tags and prompt text are never
|
||||
written): its targets are every still image the output folders list (the Images
|
||||
listing of each root, linked to a prompt or not; videos are not targets; a file under
|
||||
two overlapping roots is one target). `POST /prompt_manager/autotag/batch/preview
|
||||
{model_type, existing}` writes nothing and opens or stats no image file (it may wait
|
||||
for the shared listing walk); it answers `images`, `already_tagged` (images with any
|
||||
auto-tagged byte record), `to_tag` (`images − already_tagged` under Skip, `images`
|
||||
under Merge; both estimates), `estimate_seconds` (WD14: `to_tag` × 1.6 s on the
|
||||
processor or × `WD14_GPU_SECONDS_PER_IMAGE` when ONNX Runtime offers CUDA, see
|
||||
`py/tagging_defaults.py`; `null` for GGUF and HF), `estimate_device` (`cpu`, `gpu`,
|
||||
or `null`), `suggestion_limit` (the saved count the run uses), `model {type, label,
|
||||
size_label, downloaded}`, `existing`, `review_required`, `backup {policy}` and
|
||||
`disk`. `POST
|
||||
- **Tag output images**: `POST /prompt_manager/autotag/batch/preview {model_type,
|
||||
existing}` writes nothing and opens no file; it answers `images`, `prompts`,
|
||||
`already_tagged`, `to_tag` (an upper bound), `model {type, label, size_label,
|
||||
downloaded}`, `existing`, `review_required`, `backup {policy}` and `disk`. `POST
|
||||
/prompt_manager/jobs/autotag {model_type, existing, options}` (WD14: both thresholds;
|
||||
GGUF/HF: `instructions`, the template) starts the cancellable `autotag` job (202),
|
||||
title "Tag output images", admission `records`, a safety backup by the policy first.
|
||||
The job runs the `image-tags` loop (`_tag_each_file`) on each image: a capture, at
|
||||
most `suggestion_limit` model tags, a re-hash, then a union into that image's tags
|
||||
that also marks its bytes auto-tagged (`media_image_autotags`, one row per byte
|
||||
record; an empty result and an unsupported still are marked too; File changed, File
|
||||
not found, Too large to verify, read, inference and tag-limit errors are not). Phases,
|
||||
each checked for Cancel: backup, model load, listing ("Listing output images…"), a
|
||||
pre-pass that stats each file without opening it (gone: "File not found", over 256
|
||||
MiB: "Too large to verify", under Skip a marked current record: "Already
|
||||
auto-tagged"; counted with `record_counted`, rows up to the cap, no log line each),
|
||||
then the loop ("Processing file {k + n} of {T}", the time left from the model's own
|
||||
rate, D56 unchanged). It ends "Tagged N images", "Cancelled. Tags already added were
|
||||
kept.", or failed with "Prompt storage was restored; start the job again.", "The
|
||||
output folders could not be listed. Try again." or the model's refusal.
|
||||
Refusals: `400 review-required` (Replace after review never runs as a batch), `409
|
||||
model-unavailable` ("<label> is not downloaded. Download it in System ›
|
||||
Auto-tagging."), `507 insufficient-space`, `409 busy`, `409 model-busy`. The section
|
||||
toasts `model-busy` with its operation, the other three with the server's sentence,
|
||||
`busy` with the running job (D57), anything else "<what> could not start. Try again.".
|
||||
Outcomes: Tagged / Unchanged / Skipped / Errors (the `image-tags` descriptor, D111).
|
||||
`tests/test_autotag_batch.py` pins it, including two images of one prompt getting
|
||||
different tags with no `prompts` or `prompt_tags` row changed; the marks' schema group
|
||||
is pinned by `tests/test_autotag_schema.py` (forward-only: one empty `WITHOUT ROWID`
|
||||
table, optional in old backups, refused without the annotation group).
|
||||
|
||||
- **The legacy pages (#379, PR 2)** write image tags, never prompt tags.
|
||||
`POST /autotag/start` (the admin and gallery batch) reads `model_type`, `prompt`,
|
||||
`skip_tagged` (Skip or Merge), `keep_in_memory` and the thresholds (else the saved
|
||||
ones), checks them with `validate_options` (WD14: both thresholds; GGUF/HF:
|
||||
`instructions` = the non-blank `prompt`, else the saved template; the saved
|
||||
"Suggestions per image") and starts the `autotag` job "Tag output images (legacy
|
||||
page)" (gate owner `legacy-batch`) through the native start, so the safety backup and
|
||||
the disk check apply. A second legacy batch answers JSON 409 `{success: false, error:
|
||||
"Auto-tagging is already running"}` (also when another legacy start wins the race past
|
||||
that check); `SseStream` hands a non-2xx JSON `error` to `onerror` as `serverMessage`,
|
||||
which both pages show instead of "Connection lost during auto tagging". Any other refusal (a job in the way: "<title>
|
||||
already running", `model-unavailable`, `insufficient-space`, `model-busy`, a bad
|
||||
option) is one `{type: "error", status: "error", message}` frame. The stream relays
|
||||
the job: the phases as `{type: "progress", progress: 0 | 5 | 8, status: "Taking a
|
||||
safety backup…" | "Loading model…" | "Listing output images…"}`, each image as
|
||||
`{type: "progress", progress: 10 + progress × 85 / 100, status, processed, tagged,
|
||||
skipped}` (at most every 0.5 s; `processed` = Tagged + Unchanged + Errors), and the end
|
||||
as `complete` (`processed`, `tagged`, `skipped`, `errors`, `skipped_outside_roots: 0`,
|
||||
`status` "Complete" or "No still images found", `model_status`, `model_loaded`),
|
||||
`{type: "cancelled", message: "Cancelled. Tags already added were kept."}` or an
|
||||
`error` frame with the job's status line. A job quiet for 5 s repeats its last frame,
|
||||
and a closed stream cancels the job (the legacy pages have no other stop).
|
||||
`POST /autotag/single` takes `existing` (`"skip"` or `"merge"`, default merge). Under
|
||||
Skip, an image whose current bytes are auto-tagged answers `{success, already_tagged:
|
||||
true, image_tags, prompt_id, filename}` after one stat, without the model gate.
|
||||
Otherwise the image is captured and the model runs on the capture under the gate with
|
||||
the saved cap; the answer adds `tags`, `sha256` (the captured bytes) and `image_tags`,
|
||||
and `service.custom_prompt` is never set. Refusals: `409 model-unavailable`, `413
|
||||
too-large`, `404 file-not-found`, `409 file-changed`, `503 busy`, `415 not-a-still`
|
||||
(the image is marked), `409 model-busy`. `POST /autotag/apply` takes exactly
|
||||
`{image_id, sha256, tags}` or `{path, root_index?, sha256, tags}` (64 KiB), re-hashes
|
||||
the file and unions the tags into that image (`add_image_tags`, marked): `{success,
|
||||
added_tags, total_tags}`; an empty list only marks the image. A body with `prompt_id`
|
||||
is `400` "This page is out of date. Reload it."; a changed file `409 file-changed`
|
||||
("The image changed since its tags were generated."); a full set `409 tag-limit`; a
|
||||
restore `409 storage-changed`. The admin review asks each image with Skip first: an
|
||||
auto-tagged one shows "Image Already Auto-tagged" / "A tagging model already tagged
|
||||
this image. Generate new tags?" with its image tags (Skip, Skip All Tagged, Re-tag asks
|
||||
again with Merge); the gallery review always merges. Apply & Next sends the image and
|
||||
its `sha256` (linked or not). The batch copy: "Skip images already auto-tagged",
|
||||
"Auto tagging complete! Tagged N images." ("1 image"), and under both progress bars
|
||||
"Closing this window stops tagging. Tags already added are kept. Prompt Manager's
|
||||
Activity page shows the time left.". Pinned by `tests/test_autotag_legacy.py`,
|
||||
`tests/js/image-helpers.test.js` and `legacy-autotag.spec`.
|
||||
- **Tag image dialog marks (DEC4).** A save after Generate suggestions ran on that
|
||||
capture marks the image auto-tagged (also a replayed save, and a saved retry after
|
||||
`tag-conflict`); a save without Generate does not. The API keeps the suggested capture
|
||||
tokens; each new one drops the tokens whose capture is gone (`expires_in`, which never
|
||||
resets a capture's idle clock), a release or a saved receipt drops its own, and
|
||||
closing the captures clears them (`TestDialogMarks`).
|
||||
|
||||
No browser test downloads or runs a model: the injected `TestModel` answers, its
|
||||
`download_model` raises unless `/__test__/tagging/download {mode: hold | succeed |
|
||||
@@ -3956,153 +3701,6 @@ it. Every workspace-system-tagging test asserts no `/autotag/download/` request.
|
||||
Changing the routes or `py/config.py` needs a ComfyUI restart; the section itself is
|
||||
HTML, CSS and JS only.
|
||||
|
||||
### Logs and diagnostics contract (#249 System S12)
|
||||
|
||||
System › Diagnostics & logs (`web/js/workspace/system-diagnostics.mjs`, D109–D110)
|
||||
reads and deletes only through the routes below; the legacy admin Logs and Diagnostics
|
||||
modals use the same routes and keep working.
|
||||
|
||||
- **Rule R** (`utils/logging_config.py`): a file may be deleted as a rotated copy only
|
||||
when it is a direct entry of the log folder named `prompt_manager.log.<n>` (`<n>` a
|
||||
decimal integer ≥ 1 with no leading zero, compared after `normcase`) and a regular
|
||||
file that is not a link or junction (`lstat`). Every other entry is `active`, `link`
|
||||
or `other` and is never deleted. `/logs/files` rows carry that `kind`; links are listed
|
||||
with their own `lstat` size and are not counted or sized by `/logs/stats`. A download
|
||||
is served only for a regular, non-link file of the folder (a link, a folder or a name
|
||||
not listed is 404), read off the event loop.
|
||||
- **Reviewed cleanup**: `POST /maintenance/preview {operation: "rotated_logs", options:
|
||||
{}}` lists every rotated copy (basenames, size and time) and writes nothing; `POST
|
||||
/maintenance/apply {token}` starts the `maintenance` job "Rotated log copies"
|
||||
(admission `read`, not cancellable). Under the manager's files lock and every file
|
||||
handler's lock it re-scans; when any listed file is missing, no longer rotated or
|
||||
differs in size, `mtime_ns` or a non-zero inode, it deletes nothing and fails
|
||||
`plan-changed` ("The log files changed since the review; nothing was deleted. Review
|
||||
again.", Activity offers Review again). Otherwise it deletes exactly the listed files;
|
||||
each is a Changed row ("Deleted · <size>"), a file that cannot be removed an Errors
|
||||
row. Two Prompt Manager processes sharing one log folder are not locked against each
|
||||
other: each file's identity is checked just before its unlink.
|
||||
- **Clear all logs** (`POST /logs/truncate`, also the admin page's Clear) empties the
|
||||
active log through its handler, deletes the rotated copies (Rule R) and clears the
|
||||
buffer; other files and links are kept. Response keys are unchanged.
|
||||
- **Runtime settings** (`GET/POST /logs/config`): the level (DEBUG…CRITICAL) and console
|
||||
logging apply until the server restarts; the retention values are shown read-only in
|
||||
the workspace and the API keeps accepting them.
|
||||
- **Report** (`GET /system/diagnostics`): built on the server from an explicit list of
|
||||
fields (version, Python, platform, Viewer.js version, the four legacy checks, database
|
||||
info, source counts, log configuration and statistics, the session's health summary
|
||||
and the non-path configuration with `has_civitai_api_key`). Every path is relative to
|
||||
a ComfyUI anchor or reads "outside ComfyUI"; a backstop scrub replaces any absolute
|
||||
path or the saved key and counts it in `redacted`, which the tests require to be 0. It
|
||||
takes a registry lease (409 `busy` while an exclusive job such as a restore runs).
|
||||
- **Compressed downloads**: the gzip middleware drops a handler's declared
|
||||
`Content-Length` before it compresses a body, so the response declares the compressed
|
||||
length. A stale length made a browser wait for bytes that never came: the log download
|
||||
of any file over 1 KB hung (`test_compressed_body_declares_its_own_length`).
|
||||
|
||||
The browser server writes its logs into `<server root>/logs`
|
||||
(`tests/browser/workspace_diagnostics_fixture.py`), whose hooks seed records with fixed
|
||||
times, rotated copies, a link (POSIX) and a user's `.bak`, and roll the copies over; no
|
||||
test reads or deletes a real log folder.
|
||||
|
||||
`tests/browser/system-diagnostics-concept.spec.mjs` compares the section and its log
|
||||
cleanup review with the concept (dark and light, 1600x1100 and 390x844, no page scroll;
|
||||
both selects show their whole option at 320 wide). The fixture's `{mode: "concept"}` seed
|
||||
reproduces the concept's Logs sample: only its four buffer lines, a 1.4 MB active log with
|
||||
three rotated copies and console logging off, so those values are compared as the concept
|
||||
prints them. `mapDiagnosticsConcept` applies only D109 and D110, and `mapCleanupConcept`
|
||||
D100 (the shared `mapPlanConcept`) with D109's rule for this plan (no backup box, "Not
|
||||
needed: no library record changes", "Listed files"). The mutation check asserts that the
|
||||
unmapped difference is exactly the deviated paths. The comparison found that the Database
|
||||
line read `database_info.size_bytes`, a key the report does not have, so it always said
|
||||
"0 bytes"; it reads `database_size_bytes`, the key `get_database_info()` returns.
|
||||
|
||||
### Legacy System capability coverage
|
||||
|
||||
Every System capability of the legacy admin and gallery pages has an owner (#249 System
|
||||
S12, #329). Status is `covered` (a native control replaces it), `kept in legacy` (it stays
|
||||
on the legacy page, which keeps working) or `approved removal` (a ruling retired it). Native
|
||||
control is `` `<section>` · <role> · "<accessible name>" ``, `<section>` a System hash id
|
||||
and `<role>` one of button, combobox, switch, slider, spinbutton, textbox, group, status or
|
||||
term, followed by ` · in "<row title>"` when several rows of the section share the name.
|
||||
Every `covered` row names one, except the rows in `CONTROL_EXEMPTIONS`
|
||||
(`tools/vendor/legacy-coverage.mjs`), each with its reason: today only "Gallery per-image
|
||||
auto-tag / review", which the Images selection bar owns, not a System section. A row that
|
||||
is not `covered` names none (`—`). Test is the exact title of a test that runs, then its
|
||||
file. Anchors are line numbers on `main` at this PR.
|
||||
|
||||
`tests/vendor/legacy-coverage.test.mjs` fails on a row with any of these problems:
|
||||
- no owner, or a status outside the three;
|
||||
- a malformed Native control cell;
|
||||
- a covered row without a control (unless exempt), or a control on a row that is not
|
||||
covered;
|
||||
- a test title its file does not declare: `test(` or `it(` (`.only` and `.slow` count),
|
||||
or a `def test_` at function or method indentation;
|
||||
- a title whose test is only declared with `.skip`, `.fixme` or `.fail`.
|
||||
|
||||
A title that appears only in a comment or a Python docstring does not count. CI running
|
||||
those tests green is what proves them. `every covered legacy System capability has its
|
||||
native control` (workspace-system.spec) opens each covered row's section and finds the
|
||||
control by role and accessible name, inside its row when the cell names one. The shared
|
||||
"Review plan" of the per-operation review row and the Sources "Refresh" of the monitoring
|
||||
list stay unscoped: any row's control proves them.
|
||||
|
||||
| Capability | Legacy anchor | Owner | Status | Native control | Test |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| Image scan directories add/remove | `admin.html:377–390`; `admin.js:200,1273–1323` | S4 Sources | covered | `sources` · button · "+ Add source" | Add source validates on the server, then marks the host draft dirty; Save persists (workspace-system-sources.spec.mjs) |
|
||||
| Currently monitoring list + Refresh | `admin.html:392–400`; `admin.js:1032` | S4 Sources | covered | `sources` · button · "Refresh" | watching only with a live observer; restart pending otherwise (workspace-system-sources.spec.mjs) |
|
||||
| Monitoring enabled | `py/config.py:212,248` | S4 Sources | covered | `sources` · switch · "Watch for new outputs" | Watch for new outputs saves monitoring_enabled; a restart applies it (workspace-system-sources.spec.mjs) |
|
||||
| Search results auto-hide `result_timeout` | `admin.html:407–412` | S3 Appearance | covered | `appearance` · spinbutton · "Node search auto-hide (seconds)" | host fields round-trip through /settings with revision (workspace-system.spec.mjs) |
|
||||
| Web UI display mode | `admin.html:414–422` | S3 Appearance | covered | `appearance` · combobox · "Open from ComfyUI" | host fields round-trip through /settings with revision (workspace-system.spec.mjs) |
|
||||
| Infinite scrolling default | `admin.html:424–430`; `admin.js:689–727` | Legacy prompt list (README continuous browsing); read-only in the report | kept in legacy | — | shouldLoadMore asks for the next page only near the bottom with pages left (list-state.test.js) |
|
||||
| Worker threads | `admin.html:437–445` | S4 Sources | covered | `sources` · slider · "Worker threads" | worker range max is cpu_count (workspace-system-sources.spec.mjs) |
|
||||
| LoraManager badge, enable, path, trigger words, CivitAI key, clear key | `admin.html:448–490`; `admin.js:1052–1156` | S10 Integrations | covered | `integrations` · switch · "LoraManager integration" | LoRA rows show detected / enabled / cache-loaded separately (workspace-system-integrations.spec.mjs) |
|
||||
| LoRA data import with progress | `admin.html:492–498,517–541`; `admin.js:1157` | S10 Integrations | covered | `integrations` · button · "Review import" | Review import shows replacement, blocked items and network effects with the backup box; Start opens the job (D107) (workspace-system-integrations.spec.mjs) |
|
||||
| Statistics | `admin.html:159`; `admin.js:1627` | S2 Overview | covered | `overview` · button · "Explore" | statistics dialog lists runs by category and library totals (workspace-system.spec.mjs) |
|
||||
| Export components JSON `/export` | `admin.html:163`; `admin.js:1607`; `py/api/prompts.py:238` | S9 Backup & export | covered | `backups` · button · "Export all prompts JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Saved pairs export | `py/api/pairs.py:84` | S9 Backup & export | covered | `backups` · button · "Export saved pairs JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Library export | `py/api/library.py:223` | S9 Backup & export | covered | `backups` · button · "Export JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Diagnostics run | `admin.html:171,659`; `admin.js:1984` | S12 Diagnostics | covered | `diagnostics` · button · "Run checks" | Run checks opens the shared Health report (workspace-system-diagnostics.spec.mjs) |
|
||||
| Synthetic image-link test | `admin.html:662`; `admin.js:2101` | S12 Diagnostics | covered | `diagnostics` · button · "Create test link…" | Create test link confirms first and names the prompt (workspace-system-diagnostics.spec.mjs) |
|
||||
| Maintenance cleanup_duplicates, vacuum, cleanup_orphaned_images, prune_orphaned_prompts | `admin.html:692–748`; `admin.js:1834` | S6/S7 Maintenance | covered | `maintenance` · button · "Review plan" · in "Duplicate prompt text" | Apply reviewed operation closes, toasts, opens the job in Activity (workspace-system-maintenance.spec.mjs) |
|
||||
| Maintenance check_hash_duplicates, check_consistency | `admin.html:692–748`; `admin.js:1834` | S2 Health report (Maintenance notice, D123) | covered | `maintenance` · button · "Run health checks" | Run health checks opens Health report and updates metric 4 (workspace-system.spec.mjs) |
|
||||
| Maintenance statistics (database info) | `admin.html:692–748`; `admin.js:1834` | S12 Diagnostics | covered | `diagnostics` · term · "Database" | meta-grid shows real database and source status, the pinned Viewer.js version and the host renderers (workspace-system-diagnostics.spec.mjs) |
|
||||
| Maintenance select all / clear all / run | `admin.html:754–762`; `admin.js:1822–1833` | S7 per-operation review | covered | `maintenance` · button · "Review plan" | plan dialog shows the server's scope, protected, backup policy and media lines with the backup box (D100); Cancel writes nothing (workspace-system-maintenance.spec.mjs) |
|
||||
| Prompt cleanup `/cleanup` | `py/api/admin.py:620` | S6/S7 Maintenance | covered | `maintenance` · button · "Review plan" · in "Duplicate prompt text" | test_duplicate_prompts_plan_counts_the_legacy_groups (test_maintenance_plans.py) |
|
||||
| Database backup | `admin.html:179,824`; `admin.js:2139,2454` | S9 Backup & export | covered | `backups` · button · "Back up" | Back up downloads through the api client and revokes the object URL (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Restore from file | `admin.html:183,616–642`; `admin.js:2177–2248` | S8/S9 Backup & export | covered | `backups` · button · "Review restore…" | Review restore stages the chosen file and shows the real summary with the backup box (D105) (workspace-system-backups.spec.mjs) |
|
||||
| Metadata scan with progress, cancel, pill | `admin.html:187,805–865`; `admin.js:2336–2500` | S4 Sources | covered | `sources` · button · "Review scan" | Review scan opens the shared Review metadata scan with all sources (workspace-system-sources.spec.mjs) |
|
||||
| Rescan-required notice + dismiss | `admin.html:1367–1410`; `admin.js:991–1011` | S4 Sources | covered | `sources` · button · "Later" | notice Later persists the 3.2.0 dismissal (pm_update_notice_dismissed) without scanning (workspace-system-sources.spec.mjs) |
|
||||
| View logs with level and limit | `admin.html:195,885–907,974`; `admin.js:2579–2641` | S12 Logs | covered | `diagnostics` · combobox · "Level" | level filter is minimum-level over all six choices (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log statistics | `admin.html:921–941`; `admin.js:2512` | S12 Logs | covered | `diagnostics` · status · "Log statistics" | log statistics show entries, errors, warnings, files and size (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log files list and download | `admin.html:910,982–985`; `admin.js:2545,2642,2679` | S12 Logs | covered | `diagnostics` · button · "Download log" | each listed file downloads by its returned name and a link has no Download (workspace-system-diagnostics.spec.mjs) |
|
||||
| Clear/truncate logs | `admin.html:915`; `admin.js:2704` | S12 Logs | covered | `diagnostics` · button · "Clear all logs…" | Clear all logs states both counts, confirms first and reads everything again (workspace-system-diagnostics.spec.mjs) |
|
||||
| Rotated-log cleanup (new, #329) | — | S12 Logs | covered | `diagnostics` · button · "Review log cleanup" | Review log cleanup lists only rotated copies and keeps the rest (workspace-system-diagnostics.spec.mjs) |
|
||||
| Set log level (incl. CRITICAL), console logging | `admin.html:948–966`; `admin.js:2530,2729` | S12 Logs | covered | `diagnostics` · combobox · "Runtime log level" | runtime level (including CRITICAL) and console logging save and say they apply until the server restarts (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log file_logging / max_file_size / backup_count / buffer_size | `py/api/logging_routes.py:56,285–299` | S12 Logs (read-only) | covered | `diagnostics` · group · "Log files & retention" | each listed file downloads by its returned name and a link has no Download (workspace-system-diagnostics.spec.mjs) |
|
||||
| Auto-tag model list with status | `admin.html:1095–1148`; `admin.js:3641` | S11 Auto-tagging | covered | `tagging` · combobox · "Tagging model" | model options are the four real engines with size and the description shows live status (workspace-system-tagging.spec.mjs) |
|
||||
| Auto-tag model download | `admin.html:1413–1438`; `admin.js:3680–3747`; `gallery.js:3412` | S11 Auto-tagging | covered | `tagging` · button · "Download" | Download starts the job and Activity shows it without Cancel (workspace-system-tagging.spec.mjs) |
|
||||
| Unload model | `gallery.js:3480` | S11 Auto-tagging | covered | `tagging` · button · "Unload" | Unload is enabled only while a model is loaded, and a held gate's refusal names the operation (workspace-system-tagging.spec.mjs) |
|
||||
| Tag template, WD14 thresholds, already-tagged mode | `admin.html:1150–1198`; `admin.js:3748` | S11 Auto-tagging | covered | `tagging` · combobox · "Existing tags" | defaults save through Save preferences and the Tag image dialog uses them (workspace-system-tagging.spec.mjs) |
|
||||
| Batch auto-tag output images | `admin.html:191,1210,1279–1318`; `admin.js:3748–3840` | S11 Auto-tagging | covered | `tagging` · button · "Review" | Tag output images reviews the images, tags each one and never a prompt (#379) (workspace-system-tagging.spec.mjs) |
|
||||
| Batch review loop | `admin.html:1206,1218–1364`; `admin.js:3841–4172` | S11 + image-tagging plan | covered | `tagging` · button · "Preview tagging review" | Preview tagging review opens the Tag image dialog for the selected image (workspace-system-tagging.spec.mjs) |
|
||||
| Gallery per-image auto-tag / review | `gallery.html:601,605`; `gallery.js:3506,3674–3753` | Image-tagging plan (selection bar) | covered | — | Auto-tag selected with Apply automatically runs a job with the chosen model, thresholds and suggestion count (workspace-tagging-bulk.spec.mjs) |
|
||||
| Gallery thumbnail generation | `gallery.js:1852,1967` | S5/S7 | covered | `sources` · button · "Review options" · in "Thumbnail generation" | Start closes the dialog, toasts, and opens the job in Activity (workspace-system-sources.spec.mjs) |
|
||||
| Clear server thumbnails | `gallery.js:2175–2192` | S7 Maintenance | covered | `maintenance` · button · "Review options" · in "Thumbnail cache" | Thumbnail cache opens the S5 dialog (workspace-system-maintenance.spec.mjs) |
|
||||
| Browser cache clear | `gallery.js:2194–2207` | S7 Maintenance | covered | `maintenance` · button · "Review plan" · in "Metadata cache" | Metadata cache reviews and clears only owned browser entries when there are some (D102) (workspace-system-maintenance.spec.mjs) |
|
||||
| Refresh file listing | `gallery.js:2209` | S4 Sources | covered | `sources` · button · "Refresh" · in "Refresh files" | Refresh files calls /gallery/rescan and toasts its counts (workspace-system-sources.spec.mjs) |
|
||||
| Duplicate image scan, review, delete | `gallery.js:2234–2577` | S6/S7 Maintenance | covered | `maintenance` · button · "Find duplicates" | Find duplicates starts a job; its view links Review groups; Keep radios build a duplicate_files plan (D101) (workspace-system-maintenance.spec.mjs) |
|
||||
| Default view mode, image quality | `gallery-settings.js:26–27` | S3 Appearance | covered | `appearance` · combobox · "Default layout" | defaults apply to a new visit but not to the current view or a saved view (workspace-system.spec.mjs) |
|
||||
| Auto-load metadata, cache metadata, show file paths | `gallery-settings.js:31–33` | S3 Appearance | covered | `appearance` · switch · "Load metadata on selection" | metadata off shows an explicit load and loads once on request (workspace-system.spec.mjs) |
|
||||
| Video autoplay, mute, loop | `gallery-settings.js:40–42` | S3 Appearance | covered | `appearance` · switch · "Autoplay videos" | video defaults reach the viewer (workspace-system.spec.mjs) |
|
||||
| Image info on cards; sidebar collapsed | `gallery-settings.js:30,34` | S3 Appearance | covered | `appearance` · switch · "Show image details on cards" | added rows reach their consumers (workspace-system.spec.mjs) |
|
||||
| Reset gallery settings | `gallery.js:1813` | S3 Appearance | covered | `appearance` · button · "Reset" | reset restores defaults without touching other keys (workspace-system.spec.mjs) |
|
||||
| Lazy loading | `gallery-settings.js:25` | Legacy gallery (native always lazy) | kept in legacy | — | a legacy gallery Save keeps the workspace keys and every other stored key (gallery-settings.test.js) |
|
||||
| Results per page / grid columns / paged mode | `gallery-settings.js:28–29,35` | README continuous browsing | approved removal | — | numeric strings from <select> values are accepted and columns are clamped (gallery-settings.test.js) |
|
||||
| Startup thumbnail prompt | `gallery-settings.js:38–39`; `gallery.js:2578` | README; Overview thumbnail row | approved removal | — | thumbnail row reflects the status endpoint (workspace-system.spec.mjs) |
|
||||
| Client debug output `debugMode` | `gallery-settings.js:36` | Legacy gallery (S12-10) | kept in legacy | — | a legacy gallery Save keeps the workspace keys and every other stored key (gallery-settings.test.js) |
|
||||
| API timeout `apiTimeout` | `gallery-settings.js:37` | Legacy gallery (S12-10; read by no request) | kept in legacy | — | defaults are a complete, sane settings object (gallery-settings.test.js) |
|
||||
| config.json-only keys | `py/config.py:212–234,722–739`; `py/api/diagnostics_report.py:185` | S12 report (read-only) | covered | `diagnostics` · button · "Download diagnostic report" | test_report_lists_config_only_keys (test_api_diagnostics_report.py) |
|
||||
|
||||
### Browser cache ownership (#222)
|
||||
|
||||
The legacy gallery's Clear cache used to delete every localStorage key starting `pm_` or
|
||||
@@ -4497,17 +4095,8 @@ impact and command receipts report `image_count` / `images` /
|
||||
`(size, mtime_ns)` equals its verified fingerprint; an in-place rewrite that
|
||||
preserves both keeps the old set matching until the file is next observed. The
|
||||
Images Tags filter (a file matches when every tag is on its linked positive or
|
||||
its current image tag set) and `q` search find their candidates through the tag
|
||||
index (#379), never by streaming every image tag row: `image_tag_ids` reads the
|
||||
tag names that tag at least one image (an `EXISTS` on `idx_media_image_tags_tag`)
|
||||
and keeps a name when it equals a Tags filter tag exactly or contains `q`
|
||||
case-folded; `image_tag_locators` reads the files carrying those ids (`tag_id IN`,
|
||||
500 ids a chunk). Only the listed ones among them are stat'ed, and only their
|
||||
current records count; a linked positive's prompt tags still combine with the
|
||||
image tags under a Tags filter. `test_indexed_candidates_equal_the_full_stream`
|
||||
compares the lookup with the old full stream (kept in the test as the reference)
|
||||
and `test_tag_search_reads_image_tags_only_by_tag_id` traces every statement; the
|
||||
|
||||
its current image tag set) and `q` search stream the tagged records once and
|
||||
stat only the listed files whose records carry a filter tag or a `q` hit; the
|
||||
inspector's `image_tags` facet (`current` / `none` / `unavailable`) uses the
|
||||
inspection's own stable stat (`tests/test_api_workspace_images.py`
|
||||
`TestWorkspaceImageTags`, `tests/test_api_prompt_inspector.py`). The
|
||||
|
||||
+45
-135
@@ -19,11 +19,6 @@ links (``PromptDatabase._move_tag_links``).
|
||||
Cheap reads (spec amendment S1) treat a byte record as *current* only while
|
||||
the file's present ``(size, mtime_ns)`` equals the record's verified
|
||||
fingerprint; a file without a current record shows no image tags.
|
||||
|
||||
A write made from a tagging model's result passes ``mark_autotagged`` (#379):
|
||||
the byte record is then marked auto-tagged (``media_image_autotags``) in the
|
||||
same transaction, so a mark never exists without the write it records. Hand
|
||||
tags never mark.
|
||||
"""
|
||||
|
||||
import re
|
||||
@@ -39,7 +34,6 @@ from .annotation_schema import (
|
||||
MAX_LOCATOR_LENGTH,
|
||||
METADATA_TABLE,
|
||||
)
|
||||
from .autotag_schema import AUTOTAG_TABLE
|
||||
from .library_commands import _UUID, LibraryCommandError, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .vocabulary import validate_vocabulary_tag_name
|
||||
@@ -102,24 +96,12 @@ _TAGS_OF = (
|
||||
"JOIN tags t ON t.id=mit.tag_id WHERE mit.byte_id IN ({marks}) "
|
||||
"ORDER BY mit.byte_id,mit.position"
|
||||
)
|
||||
_MARK = f"INSERT OR IGNORE INTO {AUTOTAG_TABLE}(byte_id) VALUES(?)" # nosec B608
|
||||
_MARKED_OF = (
|
||||
f"SELECT byte_id FROM {AUTOTAG_TABLE} WHERE byte_id IN ({{marks}})" # nosec B608
|
||||
)
|
||||
_MARKED_LOCATORS = (
|
||||
f"SELECT DISTINCT f.locator FROM {AUTOTAG_TABLE} m " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.id=m.byte_id JOIN {FILES_TABLE} f ON f.id=b.file_id"
|
||||
)
|
||||
# Tag search (#379): the names that tag at least one image (the tag index
|
||||
# answers the EXISTS), then the locators carrying chosen names (by tag_id).
|
||||
_IMAGE_TAG_NAMES = (
|
||||
"SELECT t.id,t.name FROM tags t WHERE EXISTS " # nosec B608
|
||||
f"(SELECT 1 FROM {IMAGE_TAGS_TABLE} mit WHERE mit.tag_id=t.id)"
|
||||
)
|
||||
_TAG_LOCATORS = (
|
||||
f"SELECT DISTINCT f.locator FROM {IMAGE_TAGS_TABLE} mit " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.id=mit.byte_id "
|
||||
f"JOIN {FILES_TABLE} f ON f.id=b.file_id WHERE mit.tag_id IN ({{marks}})"
|
||||
_RECORDS = (
|
||||
"SELECT f.locator,b.id,b.size,b.mtime_ns,t.name " # nosec B608
|
||||
f"FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"LEFT JOIN {IMAGE_TAGS_TABLE} mit ON mit.byte_id=b.id "
|
||||
"LEFT JOIN tags t ON t.id=mit.tag_id "
|
||||
"ORDER BY f.locator,b.first_seen_at,b.id,mit.position"
|
||||
)
|
||||
|
||||
|
||||
@@ -319,17 +301,14 @@ class AnnotationOperations:
|
||||
expected_epoch: str,
|
||||
expected_tag_revision: int,
|
||||
tags: list,
|
||||
mark_autotagged: bool = False,
|
||||
) -> dict:
|
||||
"""Replace the set of these exact bytes, guarded by epoch and revision.
|
||||
|
||||
Creates the owner and byte record when the bytes are new (an explicit
|
||||
empty set is a real save). The identical set at the current revision,
|
||||
or one behind it (a lost-response retry), is ``replayed`` and writes
|
||||
no tag; any other revision mismatch is ``tag-conflict`` with the
|
||||
current set and no writes. ``mark_autotagged`` marks the bytes
|
||||
auto-tagged on a saved or replayed receipt (the Tag image dialog's
|
||||
save after Generate suggestions).
|
||||
nothing; any other revision mismatch is ``tag-conflict`` with the
|
||||
current set and no writes.
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
@@ -350,8 +329,6 @@ class AnnotationOperations:
|
||||
):
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
if mark_autotagged:
|
||||
conn.execute(_MARK, (byte_id,))
|
||||
return _receipt(epoch, sha256, current, revision, True)
|
||||
if expected_tag_revision != revision:
|
||||
raise AnnotationCommandError(
|
||||
@@ -360,8 +337,6 @@ class AnnotationOperations:
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
revision = self._write_image_tags(conn, byte_id, tags)
|
||||
if mark_autotagged:
|
||||
conn.execute(_MARK, (byte_id,))
|
||||
return _receipt(epoch, sha256, tags, revision, False)
|
||||
|
||||
def add_image_tags(
|
||||
@@ -373,22 +348,16 @@ class AnnotationOperations:
|
||||
mtime_ns: int,
|
||||
expected_epoch: str,
|
||||
tags: list,
|
||||
mark_autotagged: bool = False,
|
||||
) -> dict:
|
||||
"""Union ``tags`` into the set of these exact bytes, keeping stored spelling.
|
||||
|
||||
``added`` counts the names that were new (case-insensitively). A union
|
||||
over ``MAX_IMAGE_TAGS`` is ``tag-limit`` with the current set and no
|
||||
writes; a union that adds nothing to an existing record is ``replayed``.
|
||||
``mark_autotagged`` (a tagging model's result) also marks the bytes
|
||||
auto-tagged, replayed or not; only such a call may pass no tags (the
|
||||
model found none: the image is marked and its set is unchanged).
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
tags = normalize_image_tags(tags)
|
||||
if not tags and not mark_autotagged:
|
||||
raise ValueError("tags must name at least one tag")
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
now = utc_now_iso()
|
||||
@@ -406,8 +375,6 @@ class AnnotationOperations:
|
||||
epoch,
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
if mark_autotagged:
|
||||
conn.execute(_MARK, (byte_id,))
|
||||
if not created and not added:
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
@@ -436,41 +403,6 @@ class AnnotationOperations:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
@staticmethod
|
||||
def _validated_stats(stats: object) -> dict:
|
||||
if not isinstance(stats, dict):
|
||||
raise ValueError(
|
||||
"stats must map locators to (size, mtime_ns) integer pairs"
|
||||
)
|
||||
for locator, fingerprint in stats.items():
|
||||
_validate_fingerprint(locator, fingerprint)
|
||||
return stats
|
||||
|
||||
@staticmethod
|
||||
def _current_byte_records(conn: sqlite3.Connection, stats: dict) -> dict:
|
||||
"""``{locator: byte_id}`` of each locator's current byte record.
|
||||
|
||||
The one rule every cheap read uses: a record is current while its
|
||||
verified ``(size, mtime_ns)`` equals the file's present fingerprint
|
||||
in ``stats``; the most recently verified wins (ties on ``(verified_at,
|
||||
byte_id)``). A locator without a current record is left out. Lookups
|
||||
go in chunks of 500 locators.
|
||||
"""
|
||||
locators = list(stats)
|
||||
chosen: dict[str, tuple[tuple[str, str], str]] = {}
|
||||
for start in range(0, len(locators), _LOOKUP_CHUNK):
|
||||
chunk = locators[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
|
||||
_FINGERPRINTS.format(marks=marks), chunk
|
||||
):
|
||||
if (size, mtime_ns) != stats[locator]:
|
||||
continue
|
||||
key = (verified_at, byte_id)
|
||||
if locator not in chosen or key > chosen[locator][0]:
|
||||
chosen[locator] = (key, byte_id)
|
||||
return {locator: byte_id for locator, (_key, byte_id) in chosen.items()}
|
||||
|
||||
def current_image_tags(self, stats: dict) -> dict:
|
||||
"""Image tags of the files whose current ``(size, mtime_ns)`` is verified.
|
||||
|
||||
@@ -479,11 +411,29 @@ class AnnotationOperations:
|
||||
exactly that verified fingerprint; the most recently verified record
|
||||
wins when several do. Lookups go in chunks of 500 locators.
|
||||
"""
|
||||
stats = self._validated_stats(stats)
|
||||
if not isinstance(stats, dict):
|
||||
raise ValueError(
|
||||
"stats must map locators to (size, mtime_ns) integer pairs"
|
||||
)
|
||||
for locator, fingerprint in stats.items():
|
||||
_validate_fingerprint(locator, fingerprint)
|
||||
locators = list(stats)
|
||||
chosen: dict[str, tuple[tuple[str, str], str]] = {}
|
||||
result: dict[str, list[str]] = {}
|
||||
with self._annotation_read() as conn:
|
||||
current = self._current_byte_records(conn, stats)
|
||||
owners = {byte_id: locator for locator, byte_id in current.items()}
|
||||
result: dict[str, list[str]] = {locator: [] for locator in current}
|
||||
for start in range(0, len(locators), _LOOKUP_CHUNK):
|
||||
chunk = locators[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
|
||||
_FINGERPRINTS.format(marks=marks), chunk
|
||||
):
|
||||
if (size, mtime_ns) != stats[locator]:
|
||||
continue
|
||||
key = (verified_at, byte_id)
|
||||
if locator not in chosen or key > chosen[locator][0]:
|
||||
chosen[locator] = (key, byte_id)
|
||||
owners = {byte_id: locator for locator, (_key, byte_id) in chosen.items()}
|
||||
result = {locator: [] for locator in chosen}
|
||||
byte_ids = list(owners)
|
||||
for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
|
||||
chunk = byte_ids[start : start + _LOOKUP_CHUNK]
|
||||
@@ -492,60 +442,20 @@ class AnnotationOperations:
|
||||
result[owners[byte_id]].append(name)
|
||||
return result
|
||||
|
||||
def autotag_state(self, stats: dict) -> dict:
|
||||
"""``{locator: bool}``: is each file's current byte record auto-tagged?
|
||||
def iter_image_tag_records(self) -> Iterator[tuple[str, int, int, list[str]]]:
|
||||
"""Stream every byte record as ``(locator, size, mtime_ns, tags)``.
|
||||
|
||||
``stats`` as for :meth:`current_image_tags`, whose rule picks the
|
||||
current record; a locator without one is False. One read snapshot.
|
||||
One SELECT ordered by locator, then by the bytes' first sighting, with
|
||||
each set's names in position order.
|
||||
"""
|
||||
stats = self._validated_stats(stats)
|
||||
with self._annotation_read() as conn:
|
||||
current = self._current_byte_records(conn, stats)
|
||||
byte_ids = list(dict.fromkeys(current.values()))
|
||||
marked: set[str] = set()
|
||||
for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
|
||||
chunk = byte_ids[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
marked.update(
|
||||
row[0]
|
||||
for row in conn.execute(_MARKED_OF.format(marks=marks), chunk)
|
||||
)
|
||||
return {locator: current.get(locator) in marked for locator in stats}
|
||||
|
||||
def autotagged_locators(self) -> set:
|
||||
"""Locators with at least one auto-tagged byte record, current or not.
|
||||
|
||||
No fingerprint is checked (the Tag output images review's estimate).
|
||||
"""
|
||||
with self._annotation_read() as conn:
|
||||
return {row[0] for row in conn.execute(_MARKED_LOCATORS)}
|
||||
|
||||
def image_tag_ids(self, wanted, query: str) -> list[int]:
|
||||
"""Ids of the tag names on at least one image that a search wants.
|
||||
|
||||
A name counts when it is in ``wanted`` exactly (the Tags filter) or
|
||||
contains ``query`` case-folded (the text search; ``query`` is
|
||||
already folded). One read of the names that tag an image.
|
||||
"""
|
||||
wanted = set(wanted)
|
||||
with self._annotation_read() as conn:
|
||||
return [
|
||||
tag_id
|
||||
for tag_id, name in conn.execute(_IMAGE_TAG_NAMES)
|
||||
if name in wanted or (query and query in name.casefold())
|
||||
]
|
||||
|
||||
def image_tag_locators(self, tag_ids) -> set[str]:
|
||||
"""Locators with any byte record (current or not) carrying one of
|
||||
``tag_ids``, through the tag index, in chunks of 500 ids."""
|
||||
ids = list(dict.fromkeys(tag_ids))
|
||||
found: set[str] = set()
|
||||
with self._annotation_read() as conn:
|
||||
for start in range(0, len(ids), _LOOKUP_CHUNK):
|
||||
chunk = ids[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
found.update(
|
||||
row[0]
|
||||
for row in conn.execute(_TAG_LOCATORS.format(marks=marks), chunk)
|
||||
)
|
||||
return found
|
||||
conn = self.model.get_connection()
|
||||
current: tuple | None = None
|
||||
for locator, byte_id, size, mtime_ns, name in conn.execute(_RECORDS):
|
||||
if current is None or current[0] != byte_id:
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
current = (byte_id, (locator, size, mtime_ns, []))
|
||||
if name is not None:
|
||||
current[1][3].append(name)
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
"""Auto-tagged marks (#379): which image bytes a tagging model has handled.
|
||||
|
||||
One table, ``media_image_autotags``: one row per byte record of the image
|
||||
annotation group (``media_annotation_bytes``) means "a tagging model looked at
|
||||
these exact bytes and its result was stored", or that the bytes were found to
|
||||
be untaggable. "Skip already tagged" skips an image whose current byte record
|
||||
is marked; hand tags never mark. Rows are written only by the image tag store
|
||||
(``database/annotations.py``), in the transaction of the tag write they
|
||||
record.
|
||||
|
||||
The group installs like ``image_favorites``: optional in old backups,
|
||||
required once a database is current, and never backfilled. Like the Needs
|
||||
review group it is valid only beside a valid image annotation group, because
|
||||
every mark cites one byte record (``ON DELETE CASCADE``). Validation reads
|
||||
only: the catalog compared after whitespace compaction, the columns and the
|
||||
foreign key through the PRAGMAs, then one bounded value pass and the foreign
|
||||
key check.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
from .annotation_schema import BYTES_TABLE
|
||||
from .annotation_schema import TABLES as ANNOTATION_TABLES
|
||||
from .annotation_schema import _catalog as _annotation_catalog
|
||||
from .annotation_schema import _uuid, annotation_schema_error
|
||||
|
||||
AUTOTAG_TABLE = "media_image_autotags"
|
||||
_DDL = (
|
||||
f"CREATE TABLE {AUTOTAG_TABLE} (byte_id TEXT PRIMARY KEY "
|
||||
f"REFERENCES {BYTES_TABLE}(id) ON DELETE CASCADE) WITHOUT ROWID"
|
||||
)
|
||||
# (cid, name, declared type, not null, default, primary key position): the
|
||||
# key of a WITHOUT ROWID table is implicitly NOT NULL.
|
||||
_COLUMNS = [(0, "byte_id", "TEXT", 1, None, 1)]
|
||||
# (column, parent table, parent column, on delete)
|
||||
_FOREIGN_KEYS = {("byte_id", BYTES_TABLE, "id", "CASCADE")}
|
||||
|
||||
NEEDS_ANNOTATIONS = "Image auto-tag marks need the image annotation tables"
|
||||
INVALID_SCHEMA = "Image auto-tag marks have an invalid schema"
|
||||
INVALID_VALUES = "Image auto-tag marks contain invalid values"
|
||||
MISSING = "Missing image auto-tag marks schema"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql).lower()
|
||||
|
||||
|
||||
def _catalog(conn: sqlite3.Connection) -> list:
|
||||
return conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name=? OR tbl_name=?",
|
||||
(AUTOTAG_TABLE, AUTOTAG_TABLE),
|
||||
).fetchall()
|
||||
|
||||
|
||||
def autotag_without_annotations_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
"""Catalog only: the marks table present without its annotation tables.
|
||||
|
||||
Startup asks this before the annotation group installs, which would make
|
||||
a marks table copied without it look valid. It reads no rows.
|
||||
"""
|
||||
if not _catalog(conn):
|
||||
return None
|
||||
present = {
|
||||
name for kind, name, _sql in _annotation_catalog(conn) if kind == "table"
|
||||
}
|
||||
if present != set(ANNOTATION_TABLES):
|
||||
return NEEDS_ANNOTATIONS
|
||||
return None
|
||||
|
||||
|
||||
def autotag_schema_error(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
allow_missing: bool = True,
|
||||
annotations_valid: bool = False,
|
||||
) -> Optional[str]:
|
||||
"""Reject an unsupported marks table or invalid marks; reads only.
|
||||
|
||||
A database without the table is acceptable for backups that predate it
|
||||
unless ``allow_missing`` is false. A present table requires a present,
|
||||
valid image annotation group; ``annotations_valid`` says the caller has
|
||||
just validated it (startup's annotation install), so only its presence is
|
||||
checked again.
|
||||
"""
|
||||
objects = _catalog(conn)
|
||||
if not objects:
|
||||
return None if allow_missing else MISSING
|
||||
if annotations_valid:
|
||||
if autotag_without_annotations_error(conn):
|
||||
return NEEDS_ANNOTATIONS
|
||||
elif annotation_schema_error(conn, allow_missing=False):
|
||||
return NEEDS_ANNOTATIONS
|
||||
actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
|
||||
if actual != {("table", AUTOTAG_TABLE, _compact(_DDL))}:
|
||||
return INVALID_SCHEMA
|
||||
columns = [
|
||||
(row[0], row[1], row[2].upper(), row[3], row[4], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({AUTOTAG_TABLE})")
|
||||
]
|
||||
if columns != _COLUMNS:
|
||||
return INVALID_SCHEMA
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({AUTOTAG_TABLE})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS:
|
||||
return INVALID_SCHEMA
|
||||
# Only this module's constants are interpolated into the SQL below.
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {AUTOTAG_TABLE} " # nosec B608
|
||||
f"WHERE NOT ({_uuid('byte_id')}) LIMIT 1"
|
||||
).fetchone():
|
||||
return INVALID_VALUES
|
||||
if conn.execute(f"PRAGMA foreign_key_check({AUTOTAG_TABLE})").fetchone():
|
||||
return INVALID_VALUES
|
||||
return None
|
||||
|
||||
|
||||
def migrate_autotag_state(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table atomically; validate before and after creating it.
|
||||
|
||||
Runs after ``migrate_annotation_state`` (which has just validated the
|
||||
annotation group). Creates one empty table: no row is read or copied.
|
||||
"""
|
||||
conn.execute("SAVEPOINT media_autotag_schema")
|
||||
try:
|
||||
error = autotag_schema_error(conn, annotations_valid=True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
error = autotag_schema_error(conn, allow_missing=False, annotations_valid=True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE media_autotag_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO media_autotag_schema")
|
||||
conn.execute("RELEASE media_autotag_schema")
|
||||
raise
|
||||
@@ -23,11 +23,6 @@ from . import restore_stage
|
||||
from .library_schema import library_schema_error, migrate_library_state
|
||||
from .image_favorites import image_favorites_schema_error, migrate_image_favorites
|
||||
from .annotation_schema import annotation_schema_error, migrate_annotation_state
|
||||
from .autotag_schema import (
|
||||
autotag_schema_error,
|
||||
autotag_without_annotations_error,
|
||||
migrate_autotag_state,
|
||||
)
|
||||
from .review_schema import (
|
||||
migrate_review_state,
|
||||
review_schema_error,
|
||||
@@ -343,13 +338,8 @@ class PromptModel:
|
||||
review_error = review_without_annotations_error(conn)
|
||||
if review_error:
|
||||
raise sqlite3.DatabaseError(review_error)
|
||||
# The auto-tag marks cite annotation bytes the same way (#379).
|
||||
autotag_error = autotag_without_annotations_error(conn)
|
||||
if autotag_error:
|
||||
raise sqlite3.DatabaseError(autotag_error)
|
||||
migrate_annotation_state(conn)
|
||||
migrate_review_state(conn)
|
||||
migrate_autotag_state(conn)
|
||||
# Last: the older generated_images rebuilds run only before file_path.
|
||||
migrate_workflow_documents(conn)
|
||||
|
||||
@@ -1235,11 +1225,6 @@ class PromptModel:
|
||||
)
|
||||
if review_error:
|
||||
return False, review_error
|
||||
autotag_error = autotag_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if autotag_error:
|
||||
return False, autotag_error
|
||||
documents_error = workflow_documents_schema_error(
|
||||
conn, allow_missing=not require_library_state, documents=documents
|
||||
)
|
||||
|
||||
@@ -1808,6 +1808,37 @@ class PromptDatabase(
|
||||
reader = DocumentReader(conn)
|
||||
return [self._image_row_to_dict(row, reader) for row in cursor]
|
||||
|
||||
def autotag_batch_candidates(self) -> List[Dict[str, Any]]:
|
||||
"""The images a Tag output images batch looks at (#328).
|
||||
|
||||
``{id, image_path, prompt_id, prompt_tags}`` for every image with a
|
||||
stored path that is linked to a prompt, newest first (the
|
||||
``get_all_images`` order, ties newest row first). Never reads
|
||||
``workflow_data`` or the other documents: a batch needs none of them.
|
||||
"""
|
||||
sql = (
|
||||
"SELECT gi.id, gi.image_path, gi.prompt_id, "
|
||||
"(SELECT GROUP_CONCAT(t.name, '|||') FROM prompt_tags pt "
|
||||
"JOIN tags t ON pt.tag_id = t.id WHERE pt.prompt_id = gi.prompt_id) "
|
||||
"AS prompt_tags "
|
||||
"FROM generated_images gi "
|
||||
"INNER JOIN prompts p ON gi.prompt_id = p.id "
|
||||
"WHERE gi.image_path IS NOT NULL AND gi.image_path != '' "
|
||||
"ORDER BY gi.generation_time DESC, gi.id DESC"
|
||||
)
|
||||
with self.model.get_connection() as conn:
|
||||
return [
|
||||
{
|
||||
"id": row["id"],
|
||||
"image_path": row["image_path"],
|
||||
"prompt_id": row["prompt_id"],
|
||||
"prompt_tags": (
|
||||
row["prompt_tags"].split("|||") if row["prompt_tags"] else []
|
||||
),
|
||||
}
|
||||
for row in conn.execute(sql)
|
||||
]
|
||||
|
||||
def get_all_images(
|
||||
self,
|
||||
limit: int = 0,
|
||||
|
||||
+193
-7
@@ -295,10 +295,6 @@ async def _gzip_middleware(request, handler):
|
||||
if len(compressed) >= len(body):
|
||||
return response
|
||||
|
||||
# A length the handler declared is the uncompressed one: drop it so the
|
||||
# compressed body's own length is sent (#329: a stale length made the
|
||||
# browser wait for bytes that never came).
|
||||
response.headers.pop("Content-Length", None)
|
||||
response.body = compressed
|
||||
response.headers["Content-Encoding"] = "gzip"
|
||||
response.headers["Vary"] = "Accept-Encoding"
|
||||
@@ -363,9 +359,6 @@ class PromptManagerAPI(
|
||||
# Private media captures of the annotation routes (py/api/annotations.py),
|
||||
# created on first use with their restore subscription.
|
||||
self._captures = None
|
||||
# Capture tokens Generate suggestions ran on: a save of one marks the
|
||||
# image auto-tagged (#379).
|
||||
self._suggested_tokens = set()
|
||||
self._captures_unsubscribe = None
|
||||
# Set by close_annotation_captures (shutdown): no store is recreated.
|
||||
self._captures_closed = False
|
||||
@@ -1107,3 +1100,196 @@ class PromptManagerAPI(
|
||||
result["parameters"][field] = metadata[field]
|
||||
|
||||
return result
|
||||
|
||||
def _extract_readable_prompt(self, parsed_data):
|
||||
"""Extract human-readable prompt text from ComfyUI/A1111 data."""
|
||||
|
||||
def safe_to_string(value):
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
elif isinstance(value, list):
|
||||
return " ".join(str(item) for item in value if item)
|
||||
elif value is not None:
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
# First check if we already extracted a positive prompt (A1111 format)
|
||||
if parsed_data.get("positive_prompt"):
|
||||
return parsed_data["positive_prompt"]
|
||||
|
||||
# Check if prompt is already a string
|
||||
if isinstance(parsed_data.get("prompt"), str):
|
||||
return parsed_data["prompt"]
|
||||
|
||||
# Check if prompt is a simple value that can be converted
|
||||
if parsed_data.get("prompt") and not isinstance(
|
||||
parsed_data.get("prompt"), dict
|
||||
):
|
||||
return safe_to_string(parsed_data["prompt"])
|
||||
|
||||
prompt_data = parsed_data.get("prompt")
|
||||
if isinstance(prompt_data, dict):
|
||||
# Use the enhanced logic from parse-metadata.py
|
||||
positive_prompt = self._extract_positive_prompt_from_comfyui_data(
|
||||
prompt_data
|
||||
)
|
||||
if positive_prompt:
|
||||
return positive_prompt
|
||||
|
||||
# Check workflow data if available
|
||||
workflow_data = parsed_data.get("workflow")
|
||||
if isinstance(workflow_data, dict):
|
||||
positive_prompt = self._extract_positive_prompt_from_comfyui_data(
|
||||
workflow_data
|
||||
)
|
||||
if positive_prompt:
|
||||
return positive_prompt
|
||||
|
||||
# Check parameters for positive prompt
|
||||
if parsed_data.get("parameters", {}).get("positive"):
|
||||
return safe_to_string(parsed_data["parameters"]["positive"])
|
||||
|
||||
return None
|
||||
|
||||
def _get_node_inputs(self, node):
|
||||
"""Safely get inputs from a node, handling both dict and list formats.
|
||||
|
||||
Old format: inputs is a dict with direct key-value pairs
|
||||
inputs = {"text": "my prompt", "seed": 123}
|
||||
|
||||
New format: inputs is a list of connection objects
|
||||
inputs = [
|
||||
{"name": "text", "type": "STRING", "link": null,
|
||||
"widget": {"name": "text"}},
|
||||
{"name": "clip", "type": "CLIP", "link": 11}
|
||||
]
|
||||
"""
|
||||
if not isinstance(node, dict):
|
||||
return {}
|
||||
|
||||
inputs = node.get("inputs", {})
|
||||
|
||||
# If inputs is already a dict, return it
|
||||
if isinstance(inputs, dict):
|
||||
return inputs
|
||||
|
||||
# If inputs is a list, convert to dict format
|
||||
if isinstance(inputs, list):
|
||||
inputs_dict = {}
|
||||
for input_item in inputs:
|
||||
if isinstance(input_item, dict) and "name" in input_item:
|
||||
name = input_item["name"]
|
||||
inputs_dict[name] = input_item
|
||||
return inputs_dict
|
||||
|
||||
return {}
|
||||
|
||||
def _find_text_in_node(self, node):
|
||||
"""Try to find text content in a node using various strategies.
|
||||
|
||||
Handles both old and new workflow formats.
|
||||
"""
|
||||
if not isinstance(node, dict):
|
||||
return None
|
||||
|
||||
# Strategy 1: Check normalized inputs for 'text' field
|
||||
inputs = self._get_node_inputs(node)
|
||||
if "text" in inputs and isinstance(inputs["text"], str):
|
||||
return inputs["text"]
|
||||
|
||||
# Strategy 2: For text encoder nodes, check widgets_values
|
||||
class_type = node.get("class_type", node.get("type", ""))
|
||||
text_encoder_types = [
|
||||
"CLIPTextEncode",
|
||||
"CLIPTextEncodeSDXL",
|
||||
"CLIPTextEncodeSDXLRefiner",
|
||||
"CLIPTextEncodeFlux",
|
||||
"PromptManager",
|
||||
"PromptManagerText",
|
||||
"BNK_CLIPTextEncoder",
|
||||
"Text Encoder",
|
||||
"CLIP Text Encode",
|
||||
]
|
||||
|
||||
if any(
|
||||
encoder_type.lower() in class_type.lower()
|
||||
for encoder_type in text_encoder_types
|
||||
):
|
||||
widgets_values = node.get("widgets_values", [])
|
||||
if widgets_values and len(widgets_values) > 0:
|
||||
if isinstance(widgets_values[0], str) and widgets_values[0].strip():
|
||||
return widgets_values[0]
|
||||
|
||||
return None
|
||||
|
||||
def _extract_positive_prompt_from_comfyui_data(self, data):
|
||||
"""Extract positive prompt from ComfyUI data.
|
||||
|
||||
Handles both old and new formats.
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
|
||||
# Build nodes dictionary
|
||||
nodes_by_id = {}
|
||||
if "nodes" in data:
|
||||
# Handle nodes array format
|
||||
for node in data["nodes"]:
|
||||
if isinstance(node, dict):
|
||||
nid = node.get("id")
|
||||
if nid is not None:
|
||||
nodes_by_id[nid] = node
|
||||
else:
|
||||
# Handle flat dictionary format (node_id -> node_data)
|
||||
for nid_str, node in data.items():
|
||||
try:
|
||||
nid = int(nid_str)
|
||||
except (ValueError, TypeError):
|
||||
nid = nid_str
|
||||
if isinstance(node, dict):
|
||||
if "id" in node:
|
||||
nid = node["id"]
|
||||
nodes_by_id[nid] = node
|
||||
|
||||
if not nodes_by_id:
|
||||
return None
|
||||
|
||||
# First, try to find positive/negative connection pattern
|
||||
pos_id = None
|
||||
for node in nodes_by_id.values():
|
||||
if isinstance(node, dict):
|
||||
inputs = self._get_node_inputs(node)
|
||||
if "positive" in inputs and "negative" in inputs:
|
||||
try:
|
||||
pos_input = inputs["positive"]
|
||||
if isinstance(pos_input, list) and len(pos_input) > 0:
|
||||
pos_id = int(pos_input[0])
|
||||
break
|
||||
except (ValueError, TypeError, IndexError):
|
||||
continue
|
||||
|
||||
# Get text from the positive node
|
||||
if pos_id is not None and pos_id in nodes_by_id:
|
||||
text_val = self._find_text_in_node(nodes_by_id[pos_id])
|
||||
if text_val:
|
||||
return text_val
|
||||
|
||||
# Fallback: find any text encoder node with text content
|
||||
text_nodes = []
|
||||
for node in nodes_by_id.values():
|
||||
if isinstance(node, dict):
|
||||
text_val = self._find_text_in_node(node)
|
||||
if text_val:
|
||||
# Try to determine if this is positive or negative
|
||||
node_title = node.get("title", "").lower()
|
||||
if "neg" not in node_title and "negative" not in node_title:
|
||||
# Prioritize non-negative prompts
|
||||
text_nodes.insert(0, text_val)
|
||||
else:
|
||||
text_nodes.append(text_val)
|
||||
|
||||
# Return the first positive-looking prompt
|
||||
if text_nodes:
|
||||
return text_nodes[0]
|
||||
|
||||
return None
|
||||
|
||||
+8
-19
@@ -47,7 +47,7 @@ try:
|
||||
extract_generation_params,
|
||||
parse_page_json,
|
||||
)
|
||||
from ...utils.prompt_pair_resolver import resolve_prompt_pair, sampler_less_prompt
|
||||
from ...utils.prompt_pair_resolver import resolve_prompt_pair
|
||||
from ...utils.review_capture import reviewable
|
||||
except ImportError:
|
||||
from database.pair_ingestion import blank_positive, ingest_resolved_pair
|
||||
@@ -62,7 +62,7 @@ except ImportError:
|
||||
extract_generation_params,
|
||||
parse_page_json,
|
||||
)
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair, sampler_less_prompt
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair
|
||||
from utils.review_capture import reviewable
|
||||
|
||||
|
||||
@@ -1754,24 +1754,13 @@ class AdminRoutesMixin:
|
||||
canonical.get("parameters"), str
|
||||
):
|
||||
return resolution["state"]
|
||||
elif canonical.get("prompt"):
|
||||
# No node takes a positive or negative input (a guider or wrapper
|
||||
# graph): the one wired text, never a guess (#252).
|
||||
found = sampler_less_prompt(canonical["prompt"])
|
||||
prompt_text = found["text"]
|
||||
if prompt_text is None and isinstance(canonical.get("parameters"), str):
|
||||
# A metadata saver also wrote A1111 text: its positive prompt.
|
||||
parsed = self._parse_comfyui_prompt(metadata)["positive_prompt"]
|
||||
prompt_text = parsed.strip() if parsed and parsed.strip() else None
|
||||
if prompt_text is None:
|
||||
linked = self._link_recorded_run(
|
||||
media_file, canonical, detailed, history
|
||||
)
|
||||
if linked is not None:
|
||||
return linked
|
||||
return "unresolved" if found["state"] == "ambiguous" else "found"
|
||||
else:
|
||||
prompt_text = self._parse_comfyui_prompt(metadata)["positive_prompt"]
|
||||
parsed = self._parse_comfyui_prompt(metadata)
|
||||
if not (parsed.get("prompt") or parsed.get("parameters")):
|
||||
return None
|
||||
prompt_text = self._extract_readable_prompt(parsed)
|
||||
if prompt_text is not None and not isinstance(prompt_text, str):
|
||||
prompt_text = str(prompt_text)
|
||||
if not (prompt_text and prompt_text.strip()):
|
||||
return "found"
|
||||
prompt_text = prompt_text.strip()
|
||||
|
||||
@@ -296,7 +296,6 @@ class AnnotationRoutesMixin:
|
||||
self._captures_closed = True
|
||||
unsubscribe, self._captures_unsubscribe = self._captures_unsubscribe, None
|
||||
store, self._captures = self._captures, None
|
||||
self._suggested_tokens = set()
|
||||
if unsubscribe is not None:
|
||||
unsubscribe()
|
||||
if store is not None:
|
||||
@@ -516,7 +515,6 @@ class AnnotationRoutesMixin:
|
||||
store = self._captures
|
||||
if store is not None and _TOKEN.fullmatch(token) is not None:
|
||||
store.release(token)
|
||||
self._suggested_tokens.discard(token)
|
||||
return web.json_response(
|
||||
{"success": True}, headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
@@ -566,9 +564,6 @@ class AnnotationRoutesMixin:
|
||||
return _coded("capture-expired", _OTHER_FILE)
|
||||
if facts.sha256 != capture.sha256:
|
||||
return _coded("file-changed")
|
||||
# A save after Generate suggestions ran on this capture marks the
|
||||
# image auto-tagged (#379), so a Skip batch leaves the review alone.
|
||||
suggested = token in self._suggested_tokens
|
||||
try:
|
||||
receipt = await self._run_in_executor(
|
||||
self.db.replace_image_tags,
|
||||
@@ -579,7 +574,6 @@ class AnnotationRoutesMixin:
|
||||
expected_epoch=expected_epoch,
|
||||
expected_tag_revision=revision,
|
||||
tags=tags,
|
||||
mark_autotagged=suggested,
|
||||
)
|
||||
except AnnotationCommandError as exc:
|
||||
current = exc.current or {}
|
||||
@@ -596,23 +590,10 @@ class AnnotationRoutesMixin:
|
||||
except Exception:
|
||||
self.logger.exception("Image tag save failed")
|
||||
return _error(500, "save-failed", "The tags could not be saved.")
|
||||
# Dropped only after a saved or replayed receipt: a retry after a
|
||||
# tag-conflict still marks.
|
||||
self._suggested_tokens.discard(token)
|
||||
return web.json_response(
|
||||
{"success": True, **receipt}, headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
|
||||
def _remember_suggested(self, store, token: str) -> None:
|
||||
"""Note that Generate ran on *token*; forget tokens whose capture is
|
||||
gone. ``expires_in`` does not reset a capture's idle clock."""
|
||||
self._suggested_tokens = {
|
||||
kept
|
||||
for kept in self._suggested_tokens
|
||||
if store.expires_in(kept) is not None
|
||||
}
|
||||
self._suggested_tokens.add(token)
|
||||
|
||||
# -- suggest ---------------------------------------------------------------
|
||||
|
||||
async def suggest_image_tags(self, request: web.Request) -> web.Response:
|
||||
@@ -694,7 +675,6 @@ class AnnotationRoutesMixin:
|
||||
finally:
|
||||
if not started:
|
||||
store.release_use(token)
|
||||
self._remember_suggested(store, token)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
|
||||
+560
-572
File diff suppressed because it is too large
Load Diff
+33
-464
@@ -2,9 +2,7 @@
|
||||
|
||||
``POST /prompt_manager/jobs/image-tags`` starts one ``image-tags`` job over
|
||||
1..500 listed images, named by the listing identity ``{root_index,
|
||||
root_scope, path}``, or over every image a listing ``filter`` admits (at most
|
||||
10,000, resolved at start in the listing's order; "Select all N matching",
|
||||
#392). The job is record-changing (it writes image tag sets and
|
||||
root_scope, path}``. The job is record-changing (it writes image tag sets and
|
||||
new vocabulary names), so it never runs beside the metadata scan.
|
||||
|
||||
- **Add tags** (``mode: "add"``) hashes each current file and unions a fixed
|
||||
@@ -20,21 +18,13 @@ the suggestion worker and the storage call ``add_image_tags``. Files are
|
||||
handled one at a time and the job stops at a file boundary when cancelled,
|
||||
keeping what it already wrote. Rows and log lines carry root-relative paths
|
||||
and public root labels only.
|
||||
|
||||
Auto-tag marks each image it ran the model on as auto-tagged (#379), with the
|
||||
union of its tags. The ``autotag`` job ("Tag output images", started by
|
||||
``POST /jobs/autotag``) runs the same per-file loop over every still image the
|
||||
output folders list (:meth:`ImageTagJobsMixin._run_autotag_batch`): image
|
||||
tags only, never a prompt tag or prompt text.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import functools
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import NamedTuple
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -55,24 +45,10 @@ from .annotations import (
|
||||
_submit,
|
||||
_target,
|
||||
)
|
||||
from .autotag_routes import BATCH_CANCELLED, MODEL_BUSY_ERROR, model_busy_response
|
||||
from .images import (
|
||||
MAX_LISTING_SELECTION,
|
||||
ListingSourceChanged,
|
||||
_current_fingerprint,
|
||||
_listing_posix,
|
||||
_public_path,
|
||||
parse_listing_filter,
|
||||
)
|
||||
from .autotag_routes import model_busy_response
|
||||
from .images import _public_path
|
||||
from .job_routes import busy_response
|
||||
from .jobs import (
|
||||
AUTOTAG_BATCH_JOB_KIND,
|
||||
AUTOTAG_GROUP_NAMES,
|
||||
IMAGE_TAG_GROUP_NAMES,
|
||||
IMAGE_TAG_JOB_KIND,
|
||||
JobBusy,
|
||||
)
|
||||
from .maintenance import disk_plan, short_space_message
|
||||
from .jobs import IMAGE_TAG_GROUP_NAMES, IMAGE_TAG_JOB_KIND, JobBusy
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
@@ -99,11 +75,6 @@ except ImportError:
|
||||
from utils.video_metadata import is_video_path
|
||||
|
||||
MAX_JOB_TARGETS = 500
|
||||
# A filter start resolves on the server, so the request body does not bound it.
|
||||
MAX_FILTER_TARGETS = MAX_LISTING_SELECTION
|
||||
NO_IMAGES = "No images match this view."
|
||||
FILTER_LISTING_FAILED = "The matching images could not be listed. Try again."
|
||||
TOO_MANY_IMAGES = "Narrow the view to at most {cap:,} images."
|
||||
# 500 maximal targets plus the options: a path of MAX_TARGET_PATH code points
|
||||
# takes at most 6 bytes each on the wire (a \uXXXX escape; raw UTF-8 is at most
|
||||
# 4), and the root index and scope fit in 256. The annotation routes' 64 KiB
|
||||
@@ -117,10 +88,9 @@ ADD_TITLE = "Add tags to images"
|
||||
AUTO_TITLE = "Auto-tag images"
|
||||
GATE_OWNER_KIND = "image-tag-job"
|
||||
|
||||
# Each mode's fields besides the images, named by ``targets`` or ``filter``.
|
||||
_FIELDS = {
|
||||
"add": frozenset({"mode", "tags"}),
|
||||
"auto": frozenset({"mode", "model_type", "options"}),
|
||||
"add": frozenset({"mode", "targets", "tags"}),
|
||||
"auto": frozenset({"mode", "targets", "model_type", "options"}),
|
||||
}
|
||||
_VERBS = {"add": "Add tags", "auto": "Auto-tag"}
|
||||
_LOG_PREFIXES = {"tagged": "+", "unchanged": "=", "skipped": "-", "errors": "!"}
|
||||
@@ -137,14 +107,6 @@ READ_FAILED = "The file could not be read."
|
||||
SOURCE_CHANGED = "The source changed. Refresh the image collection."
|
||||
STORAGE_RESTORED = "Prompt storage was restored; start the job again."
|
||||
BUSY_RETRY = "The file could not be read now. Run the job again for it."
|
||||
# Tag output images (#379): the per-image batch.
|
||||
ALREADY_AUTOTAGGED = "Already auto-tagged"
|
||||
LISTING_FAILED = "The output folders could not be listed. Try again."
|
||||
LISTING_STATUS = "Listing output images…"
|
||||
_INTERNAL_ERROR = "An internal error occurred. Check server logs for details."
|
||||
# The pre-pass stats and looks up this many targets at a time, checking for a
|
||||
# cancel between chunks.
|
||||
PREPASS_CHUNK = 500
|
||||
|
||||
_monotonic = time.monotonic # the job's clock (a test can fake it)
|
||||
|
||||
@@ -179,8 +141,6 @@ class _Spec:
|
||||
tags: tuple = ()
|
||||
model_type: str | None = None
|
||||
options: dict | None = None
|
||||
# A listing filter the start resolves into ``targets`` (#392).
|
||||
filter: dict | None = None
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
@@ -239,35 +199,21 @@ def parse_image_tag_request(data: dict):
|
||||
"""A :class:`_Spec` for a start body, or ``(None, error response)``.
|
||||
|
||||
``{mode:"add", targets, tags}`` or ``{mode:"auto", targets, model_type,
|
||||
options}`` exactly; options follow the Tag dialog's Generate rules. A
|
||||
``filter`` (a listing query object, parse_listing_filter) may stand in for
|
||||
``targets``; the start resolves it.
|
||||
options}`` exactly; options follow the Tag dialog's Generate rules.
|
||||
"""
|
||||
fields = _FIELDS.get(data.get("mode"))
|
||||
if fields is None:
|
||||
return None, bad_request('mode must be "add" or "auto"')
|
||||
fields = fields | {"filter" if "filter" in data else "targets"}
|
||||
if set(data) != fields:
|
||||
return None, bad_request(
|
||||
"Request must contain exactly " + ", ".join(sorted(fields))
|
||||
)
|
||||
listing_filter, targets = None, ()
|
||||
if "filter" in data:
|
||||
try:
|
||||
listing_filter = parse_listing_filter(data["filter"])
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
else:
|
||||
targets, error = _parse_targets(data["targets"])
|
||||
if error is not None:
|
||||
return None, error
|
||||
targets, error = _parse_targets(data["targets"])
|
||||
if error is not None:
|
||||
return None, error
|
||||
if data["mode"] == "add":
|
||||
spec, error = _parse_add(targets, data)
|
||||
else:
|
||||
spec, error = _parse_auto(targets, data)
|
||||
if spec is not None and listing_filter is not None:
|
||||
spec = dataclasses.replace(spec, filter=listing_filter)
|
||||
return spec, error
|
||||
return _parse_add(targets, data)
|
||||
return _parse_auto(targets, data)
|
||||
|
||||
|
||||
def _capture_outcome(exc: CaptureError, root, relative: str):
|
||||
@@ -300,92 +246,6 @@ async def _capacity_retried(job, attempt):
|
||||
return await attempt()
|
||||
|
||||
|
||||
class Refusal(NamedTuple):
|
||||
"""Why a Tag output images start was refused.
|
||||
|
||||
``response`` is the native route's JSON answer; ``message`` the sentence
|
||||
a legacy stream frame carries.
|
||||
"""
|
||||
|
||||
status: int
|
||||
code: str
|
||||
message: str
|
||||
response: web.Response
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AutotagRun:
|
||||
"""What one Tag output images job runs with (validated at its start)."""
|
||||
|
||||
model_type: str
|
||||
existing: str
|
||||
options: dict
|
||||
keep_in_memory: bool
|
||||
backup_on: bool
|
||||
owner: object
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _BatchTarget:
|
||||
"""One still image of the batch: the job target and its private keys."""
|
||||
|
||||
target: tuple # (root_index, root_scope, relative): the image-tags target
|
||||
path: str # the listed file, for the pre-pass stat (never sent out)
|
||||
locator: str # the image tag key: normcase(join(realpath(root), relative))
|
||||
label: str # the root's public label
|
||||
|
||||
|
||||
def _still_targets(api, index, root, files, seen: set) -> list:
|
||||
"""Worker thread: one root's listed still images as _BatchTargets.
|
||||
|
||||
Newest first (the listing's order). A locator an earlier root already
|
||||
produced (overlapping roots) is dropped; ``seen`` collects them.
|
||||
"""
|
||||
scope = api._gallery_root_scope(root, index)
|
||||
label = _public_path(root)
|
||||
real = os.path.realpath(root)
|
||||
prefix = os.path.join(str(root), "")
|
||||
found = []
|
||||
for path, _mtime in files:
|
||||
relative = _listing_posix(root, prefix, path)
|
||||
if is_video_path(relative):
|
||||
continue
|
||||
locator = os.path.normcase(os.path.join(real, *relative.split("/")))
|
||||
if locator in seen:
|
||||
continue
|
||||
seen.add(locator)
|
||||
found.append(_BatchTarget((index, scope, relative), str(path), locator, label))
|
||||
return found
|
||||
|
||||
|
||||
def _prepass_chunk(db, chunk, skip_marked: bool, max_file: int):
|
||||
"""Worker thread: ``(remaining, skips)`` for a chunk of _BatchTargets.
|
||||
|
||||
Stats each file (no open, no hash): gone is "File not found", over the
|
||||
capture limit "Too large to verify"; under "Skip already tagged" a file
|
||||
whose current bytes are marked is "Already auto-tagged". ``skips`` are
|
||||
``(target, detail)`` pairs in target order.
|
||||
"""
|
||||
stats, skips, alive = {}, [], []
|
||||
for item in chunk:
|
||||
fingerprint = _current_fingerprint(item.path)
|
||||
if fingerprint is None:
|
||||
skips.append((item, FILE_NOT_FOUND))
|
||||
elif fingerprint[0] > max_file:
|
||||
skips.append((item, TOO_LARGE))
|
||||
else:
|
||||
stats[item.locator] = fingerprint
|
||||
alive.append(item)
|
||||
marked = db.autotag_state(stats) if skip_marked and stats else {}
|
||||
remaining = []
|
||||
for item in alive:
|
||||
if marked.get(item.locator):
|
||||
skips.append((item, ALREADY_AUTOTAGGED))
|
||||
else:
|
||||
remaining.append(item)
|
||||
return remaining, skips
|
||||
|
||||
|
||||
def _load_model(service, model_type: str, use_gpu: bool) -> None:
|
||||
"""Worker thread: load (or swap to) *model_type*; never downloads."""
|
||||
if service.is_model_loaded() and service.get_loaded_model_type() == model_type:
|
||||
@@ -394,25 +254,6 @@ def _load_model(service, model_type: str, use_gpu: bool) -> None:
|
||||
raise RuntimeError(f"{model_type} could not be loaded")
|
||||
|
||||
|
||||
def _refusal(status: int, code: str, message: str) -> Refusal:
|
||||
response = web.json_response(
|
||||
{"success": False, "code": code, "error": message}, status=status
|
||||
)
|
||||
return Refusal(status, code, message, response)
|
||||
|
||||
|
||||
def _busy_refusal(busy: JobBusy) -> Refusal:
|
||||
title = getattr(busy.running, "title", "") or "Another job"
|
||||
return Refusal(409, "busy", f"{title} already running", busy_response(busy))
|
||||
|
||||
|
||||
def _batch_cancelled(job) -> str:
|
||||
"""Say the batch was cancelled; its end state (the runner ends the job)."""
|
||||
job.update(status_line=BATCH_CANCELLED)
|
||||
job.log_line(BATCH_CANCELLED)
|
||||
return "cancelled"
|
||||
|
||||
|
||||
class ImageTagJobsMixin:
|
||||
"""The ``image-tags`` job kind: start route and runner."""
|
||||
|
||||
@@ -429,10 +270,6 @@ class ImageTagJobsMixin:
|
||||
spec, error = parse_image_tag_request(data)
|
||||
if error is not None:
|
||||
return error
|
||||
if spec.filter is not None:
|
||||
spec, error = await self._resolve_job_filter(spec)
|
||||
if error is not None:
|
||||
return error
|
||||
roots = await self._run_in_executor(self._job_roots, spec.targets)
|
||||
if any(root is None for root, _label in roots.values()):
|
||||
return _coded("source-changed")
|
||||
@@ -464,246 +301,6 @@ class ImageTagJobsMixin:
|
||||
job.task.add_done_callback(lambda _task: model_gate.release(owner))
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
# -- Tag output images (#379) ---------------------------------------------
|
||||
|
||||
async def _autotag_targets(self) -> list:
|
||||
"""Every still image the output folders list, as _BatchTargets.
|
||||
|
||||
Root by root in configured order, each newest first (the shared
|
||||
listing, ``_get_gallery_files``, used as it is even when stale). Videos
|
||||
are not targets; a file under two overlapping roots is one target.
|
||||
"""
|
||||
seen: set = set()
|
||||
found = []
|
||||
for index, root in enumerate(list(self._get_all_output_dirs())):
|
||||
files = await self._get_gallery_files(root)
|
||||
found.extend(
|
||||
await self._run_in_executor(
|
||||
_still_targets, self, index, root, files, seen
|
||||
)
|
||||
)
|
||||
return found
|
||||
|
||||
async def _start_autotag(
|
||||
self, *, model_type, existing, options, keep_in_memory, title, owner_kind
|
||||
):
|
||||
"""Register a Tag output images job: ``(job, None)`` or ``(None, Refusal)``.
|
||||
|
||||
``options`` are already validated (``validate_options``). The checks
|
||||
run in this order: 409 ``model-unavailable``; 507
|
||||
``insufficient-space`` (only when the safety backup is on); 409
|
||||
``busy`` (another records job); 409 ``model-busy``. No await follows
|
||||
the busy check. The gate is taken before the job registers and
|
||||
released when the task ends, also before its runner ever starts.
|
||||
"""
|
||||
from ..autotag import MODELS, get_autotag_service
|
||||
|
||||
backup_on = self._safety_backup_policy() == "on"
|
||||
status = await self._run_in_executor(get_autotag_service().get_models_status)
|
||||
_disk, short, _size = await self._run_in_executor(
|
||||
functools.partial(
|
||||
disk_plan, self.db.model.db_path, backup=backup_on, vacuum=False
|
||||
)
|
||||
)
|
||||
label = MODELS[model_type]["label"]
|
||||
if not (status.get(model_type) or {}).get("downloaded"):
|
||||
message = (
|
||||
f"{label} is not downloaded. Download it in System › Auto-tagging."
|
||||
)
|
||||
return None, _refusal(409, "model-unavailable", message)
|
||||
if backup_on and short is not None:
|
||||
return None, _refusal(507, "insufficient-space", short_space_message(short))
|
||||
busy = self._jobs.blocker(AUTOTAG_BATCH_JOB_KIND, admission="records")
|
||||
if busy is not None:
|
||||
return None, _busy_refusal(JobBusy(busy))
|
||||
owner = model_gate.new_owner(owner_kind)
|
||||
if not model_gate.try_acquire(owner):
|
||||
response = model_busy_response()
|
||||
return None, Refusal(409, "model-busy", MODEL_BUSY_ERROR, response)
|
||||
run = AutotagRun(
|
||||
model_type, existing, options, keep_in_memory, backup_on, owner
|
||||
)
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
AUTOTAG_BATCH_JOB_KIND,
|
||||
title,
|
||||
admission="records",
|
||||
cancellable=True,
|
||||
scope={"model_type": model_type, "existing": existing, "label": label},
|
||||
groups=AUTOTAG_GROUP_NAMES,
|
||||
runner=functools.partial(self._run_autotag_batch, run),
|
||||
)
|
||||
except BaseException as refused:
|
||||
model_gate.release(owner)
|
||||
if isinstance(refused, JobBusy):
|
||||
return None, _busy_refusal(refused)
|
||||
raise
|
||||
job.task.add_done_callback(lambda _task: model_gate.release(owner))
|
||||
return job, None
|
||||
|
||||
async def _run_autotag_batch(self, run: AutotagRun, job) -> None:
|
||||
"""The ``autotag`` job: backup, model, listing, pre-pass, per-image loop.
|
||||
|
||||
``job.cancelled`` is checked between phases (and per pre-pass chunk
|
||||
and per image). Each phase publishes ``{"type": "phase", "phase"}``.
|
||||
With ``keep_in_memory`` false the model is unloaded under the gate.
|
||||
The job ends only after that cleanup and the gate release, so a job
|
||||
that reads as ended never still holds the gate.
|
||||
"""
|
||||
state = "failed"
|
||||
try:
|
||||
state = await self._autotag_batch_phases(run, job)
|
||||
except Exception:
|
||||
self.logger.exception("Tag output images failed")
|
||||
job.update(status_line=_INTERNAL_ERROR)
|
||||
job.log_line(f"Tagging failed · {_INTERNAL_ERROR}")
|
||||
finally:
|
||||
try:
|
||||
if not run.keep_in_memory:
|
||||
await self._unload_batch_model(run.owner)
|
||||
if run.backup_on:
|
||||
await self._prune_backup_pool(job, self.db)
|
||||
finally:
|
||||
model_gate.release(run.owner)
|
||||
job.publish({"type": state, "progress": job.progress})
|
||||
job.finish(state)
|
||||
|
||||
async def _unload_batch_model(self, owner) -> None:
|
||||
from ..autotag import get_autotag_service
|
||||
|
||||
service = get_autotag_service()
|
||||
try:
|
||||
if service.is_model_loaded():
|
||||
await model_gate.run(owner, service.unload_model)
|
||||
except Exception:
|
||||
self.logger.exception("The tagging model could not be unloaded")
|
||||
|
||||
async def _autotag_batch_phases(self, run: AutotagRun, job) -> str:
|
||||
"""Run the phases; the job's end state (the caller ends the job)."""
|
||||
job.publish({"type": "phase", "phase": "backup"})
|
||||
ok, _backup = await self._safety_backup_step(job, self.db, run.backup_on)
|
||||
if not ok:
|
||||
return "failed"
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
job.publish({"type": "phase", "phase": "model"})
|
||||
job.update(status_line="Loading the tagging model")
|
||||
spec = _Spec("auto", (), model_type=run.model_type, options=run.options)
|
||||
service = await self._prepare_tagging_model(job, spec, run.owner)
|
||||
if service is None:
|
||||
return "failed"
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
job.publish({"type": "phase", "phase": "listing"})
|
||||
job.update(status_line=LISTING_STATUS)
|
||||
epoch = await self._run_in_executor(self.db.get_execution_epoch)
|
||||
try:
|
||||
targets = await self._autotag_targets()
|
||||
except Exception:
|
||||
self.logger.exception("Tag output images could not list the outputs")
|
||||
job.update(status_line=LISTING_FAILED)
|
||||
job.log_line(f"! {LISTING_FAILED}")
|
||||
return "failed"
|
||||
total = len(targets)
|
||||
job.scope["images"] = total
|
||||
job.update(total=total)
|
||||
policy = (
|
||||
"skip images already auto-tagged" if run.existing == "skip" else "merge"
|
||||
)
|
||||
job.log_line(
|
||||
f"Tagging started · {job.scope.get('label') or run.model_type} · "
|
||||
f"{_plural(total, 'image')} · {policy}"
|
||||
)
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
remaining = await self._autotag_prepass(job, targets, run.existing == "skip")
|
||||
if remaining is None:
|
||||
return _batch_cancelled(job)
|
||||
done = total - len(remaining)
|
||||
spec = dataclasses.replace(
|
||||
spec, targets=tuple(item.target for item in remaining)
|
||||
)
|
||||
loop_run = _Run(job, spec, epoch, run.owner, service)
|
||||
if not await self._tag_each_file(loop_run, done_before=done):
|
||||
job.update(status_line=STORAGE_RESTORED)
|
||||
job.log_line(f"Tagging stopped · {STORAGE_RESTORED}")
|
||||
return "failed"
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
finished = f"Tagged {_plural(job.counters.get('tagged', 0), 'image')}"
|
||||
job.update(status_line=finished, progress=100)
|
||||
job.log_line(finished)
|
||||
return "finished"
|
||||
|
||||
async def _autotag_prepass(self, job, targets, skip_marked: bool):
|
||||
"""The targets left for the model, or None when cancelled meanwhile.
|
||||
|
||||
Opens and hashes nothing: one stat per file and one indexed read per
|
||||
chunk. Skips are counted with ``record_counted`` (exact counters, rows
|
||||
up to the cap, no log line each).
|
||||
"""
|
||||
max_file = self._capture_store().max_file
|
||||
remaining, skipped = [], 0
|
||||
total = len(targets)
|
||||
for start in range(0, total, PREPASS_CHUNK):
|
||||
if job.cancelled:
|
||||
return None
|
||||
chunk = targets[start : start + PREPASS_CHUNK]
|
||||
left, skips = await self._run_in_executor(
|
||||
_prepass_chunk, self.db, chunk, skip_marked, max_file
|
||||
)
|
||||
remaining.extend(left)
|
||||
if skips:
|
||||
job.record_counted(
|
||||
"skipped",
|
||||
len(skips),
|
||||
[
|
||||
{
|
||||
"path": item.target[2],
|
||||
"root": item.label,
|
||||
"root_index": item.target[0],
|
||||
"detail": detail,
|
||||
}
|
||||
for item, detail in skips
|
||||
],
|
||||
)
|
||||
skipped += len(skips)
|
||||
job.update(processed=skipped, progress=skipped * 100 // total)
|
||||
job.publish({"type": "progress", "progress": job.progress})
|
||||
if skipped:
|
||||
job.log_line(f"Skipped {_plural(skipped, 'image')} before tagging")
|
||||
return remaining
|
||||
|
||||
async def _resolve_job_filter(self, spec: _Spec):
|
||||
"""``spec`` with its filter resolved into targets, or ``(None, error)``.
|
||||
|
||||
The listing's own identities, in its order, kept only where a target
|
||||
could have named them (a media path the job can tag).
|
||||
"""
|
||||
cap = MAX_FILTER_TARGETS
|
||||
try:
|
||||
selection = await self.resolve_listing_filter(spec.filter, cap)
|
||||
except ListingSourceChanged:
|
||||
return None, _coded("source-changed")
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
except Exception:
|
||||
# Never echo it: it can carry SQL or database text.
|
||||
self.logger.error("A tagging filter could not be resolved", exc_info=True)
|
||||
return None, _error(500, "listing-failed", FILTER_LISTING_FAILED)
|
||||
if selection.truncated:
|
||||
return None, _error(400, "too-many-images", TOO_MANY_IMAGES.format(cap=cap))
|
||||
targets = []
|
||||
for index, scope, relative in selection.targets:
|
||||
target, error = _target(
|
||||
{"root_index": index, "root_scope": scope, "path": relative}
|
||||
)
|
||||
if error is None:
|
||||
targets.append(target)
|
||||
if not targets:
|
||||
return None, _error(400, "no-images", NO_IMAGES)
|
||||
return dataclasses.replace(spec, targets=tuple(targets)), None
|
||||
|
||||
# -- runner ----------------------------------------------------------------
|
||||
|
||||
def _job_roots(self, targets) -> dict:
|
||||
@@ -734,39 +331,28 @@ class ImageTagJobsMixin:
|
||||
job.finish("failed")
|
||||
return
|
||||
if not await self._tag_each_file(run):
|
||||
job.update(status_line=STORAGE_RESTORED)
|
||||
job.log_line(f"{verb} stopped · {STORAGE_RESTORED}")
|
||||
job.finish("failed")
|
||||
return
|
||||
state = "cancelled" if job.cancelled else "finished"
|
||||
# An ended job never keeps the loop's "about X min left" (#379).
|
||||
ended = f"{verb} {state} · {_plural(job.processed, 'file')} processed"
|
||||
job.update(status_line=ended)
|
||||
job.log_line(ended)
|
||||
job.log_line(f"{verb} {state} · {_plural(job.processed, 'file')} processed")
|
||||
job.publish({"type": state, "progress": job.progress})
|
||||
job.finish(state)
|
||||
finally:
|
||||
if owner is not None:
|
||||
model_gate.release(owner)
|
||||
|
||||
async def _tag_each_file(self, run: _Run, *, done_before: int = 0) -> bool:
|
||||
"""Tag the targets in order; False when the job had to stop.
|
||||
|
||||
``done_before`` files were already counted (the batch's pre-pass): file
|
||||
n of this loop is "file done_before + n" of the job, and the time left
|
||||
uses this loop's rate only.
|
||||
"""
|
||||
job, count = run.job, len(run.spec.targets)
|
||||
total = done_before + count
|
||||
async def _tag_each_file(self, run: _Run) -> bool:
|
||||
"""Tag the targets in order; False when the job had to stop."""
|
||||
job, total = run.job, len(run.spec.targets)
|
||||
run.roots = await self._run_in_executor(self._job_roots, run.spec.targets)
|
||||
started = _monotonic()
|
||||
for number, target in enumerate(run.spec.targets, 1):
|
||||
if job.cancelled:
|
||||
break
|
||||
# The scan's estimate (D56), with each finished file as one batch.
|
||||
left = _time_left(number - 1, count, number - 1, _monotonic() - started)
|
||||
shown = done_before + number
|
||||
job.update(status_line=f"Processing file {shown:,} of {total:,}{left}")
|
||||
left = _time_left(number - 1, total, number - 1, _monotonic() - started)
|
||||
job.update(status_line=f"Processing file {number:,} of {total:,}{left}")
|
||||
job.publish({"type": "progress", "progress": job.progress})
|
||||
stopped = False
|
||||
try:
|
||||
@@ -777,7 +363,7 @@ class ImageTagJobsMixin:
|
||||
self.logger.exception("Image tagging failed for one file")
|
||||
group, detail = "errors", SAVE_FAILED
|
||||
self._record_tag_outcome(run, target, group, detail)
|
||||
job.update(processed=shown, progress=shown * 100 // total)
|
||||
job.update(processed=number, progress=number * 100 // total)
|
||||
if stopped:
|
||||
return False
|
||||
return True
|
||||
@@ -853,17 +439,9 @@ class ImageTagJobsMixin:
|
||||
store.release(capture.token)
|
||||
|
||||
async def _auto_tag_capture(self, run: _Run, store, capture, root, relative):
|
||||
"""Suggest on the capture, then union only onto the same current bytes.
|
||||
|
||||
The bytes are marked auto-tagged with the union, also when the model
|
||||
returned no names; captured bytes no model can read are marked too
|
||||
(and stay skipped), so a "Skip already tagged" run passes them by.
|
||||
"""
|
||||
"""Suggest on the capture, then union only onto the same current bytes."""
|
||||
if not capture.inferable:
|
||||
group, detail = await self._union_image_tags(
|
||||
capture, [], run.epoch, mark_autotagged=True
|
||||
)
|
||||
return (group, detail) if group == "errors" else ("skipped", NOT_A_STILL)
|
||||
return "skipped", NOT_A_STILL
|
||||
if store.acquire(capture.token) is None:
|
||||
raise _JobStopped # every capture was invalidated (restore)
|
||||
spec = run.spec
|
||||
@@ -889,30 +467,21 @@ class ImageTagJobsMixin:
|
||||
if facts.locator != capture.locator or facts.sha256 != capture.sha256:
|
||||
return "skipped", FILE_CHANGED
|
||||
names = [str(name) for name, _score in pairs]
|
||||
return await self._union_image_tags(
|
||||
facts, names, run.epoch, mark_autotagged=True
|
||||
)
|
||||
if not names:
|
||||
return "unchanged", NO_NEW_TAGS
|
||||
return await self._union_image_tags(facts, names, run.epoch)
|
||||
|
||||
async def _union_image_tags(
|
||||
self, facts, tags: list, epoch: str, *, mark_autotagged: bool = False
|
||||
):
|
||||
"""``add_image_tags`` on the verified bytes, as a (group, detail).
|
||||
|
||||
``mark_autotagged`` (a model's result) marks the bytes auto-tagged in
|
||||
the same write; then ``tags`` may be empty.
|
||||
"""
|
||||
async def _union_image_tags(self, facts, tags: list, epoch: str):
|
||||
"""``add_image_tags`` on the verified bytes, as a (group, detail)."""
|
||||
try:
|
||||
receipt = await self._run_in_executor(
|
||||
functools.partial(
|
||||
self.db.add_image_tags,
|
||||
locator=facts.locator,
|
||||
sha256=facts.sha256,
|
||||
size=facts.size,
|
||||
mtime_ns=facts.mtime_ns,
|
||||
expected_epoch=epoch,
|
||||
tags=tags,
|
||||
mark_autotagged=mark_autotagged,
|
||||
)
|
||||
self.db.add_image_tags,
|
||||
locator=facts.locator,
|
||||
sha256=facts.sha256,
|
||||
size=facts.size,
|
||||
mtime_ns=facts.mtime_ns,
|
||||
expected_epoch=epoch,
|
||||
tags=tags,
|
||||
)
|
||||
except LibraryCommandError as exc:
|
||||
if exc.code == "storage-changed":
|
||||
|
||||
+61
-204
@@ -575,6 +575,12 @@ def _current_fingerprint(path: Path) -> tuple[int, int] | None:
|
||||
return observed.st_size, observed.st_mtime_ns
|
||||
|
||||
|
||||
def _image_tags_relevant(names: list, wanted: set, query: str) -> bool:
|
||||
if wanted and not wanted.isdisjoint(names):
|
||||
return True
|
||||
return bool(query) and any(query in name.casefold() for name in names)
|
||||
|
||||
|
||||
def _workspace_key(entry: tuple, order: str) -> tuple:
|
||||
"""The keyset sort key of *entry* (a _Listed or a 4-tuple entry)."""
|
||||
if not isinstance(entry, _Listed):
|
||||
@@ -1099,94 +1105,37 @@ def _workspace_filters(query: Mapping) -> dict:
|
||||
}
|
||||
|
||||
|
||||
# The listing parameters a bulk request may name as its filter (#392): the
|
||||
# workspace query without mode, limit, cursor and the Activity-only ``path``,
|
||||
# plus the chosen root's ``root_scope`` (a renumbered source is refused).
|
||||
LISTING_FILTER_KEYS = frozenset(
|
||||
{
|
||||
"q",
|
||||
"subfolder",
|
||||
"category",
|
||||
"tags",
|
||||
"root",
|
||||
"root_scope",
|
||||
"media",
|
||||
"sort",
|
||||
"favorites",
|
||||
}
|
||||
)
|
||||
MAX_LISTING_FILTER_VALUE = 2048
|
||||
# The most images "Select all N matching" acts on in one request (#392).
|
||||
MAX_LISTING_SELECTION = 10_000
|
||||
|
||||
|
||||
def parse_listing_filter(raw: object) -> dict:
|
||||
"""A bulk request's listing filter (#392): the listing's own query as an object.
|
||||
|
||||
Keys are a subset of LISTING_FILTER_KEYS, each value a string of at most
|
||||
MAX_LISTING_FILTER_VALUE characters, validated as the listing validates its
|
||||
query. The root's bounds need the current roots, so resolve_listing_filter
|
||||
checks those. Raises ValueError.
|
||||
"""
|
||||
if not isinstance(raw, dict) or not set(raw) <= LISTING_FILTER_KEYS:
|
||||
raise ValueError("filter must be an object of listing parameters")
|
||||
if any(
|
||||
not isinstance(value, str) or len(value) > MAX_LISTING_FILTER_VALUE
|
||||
for value in raw.values()
|
||||
):
|
||||
raise ValueError("filter values must be short strings")
|
||||
if "root_scope" in raw and "root" not in raw:
|
||||
raise ValueError("root_scope needs root")
|
||||
_workspace_filters(raw)
|
||||
return dict(raw)
|
||||
|
||||
|
||||
class ListingSourceChanged(Exception):
|
||||
"""A listing filter's ``root_scope`` no longer names its root (#392)."""
|
||||
|
||||
|
||||
class ListingSelection(NamedTuple):
|
||||
"""Every image a listing filter admits, in the listing's order (#392).
|
||||
|
||||
``targets`` are listing identities ``(root_index, root_scope, path)``;
|
||||
``positive_ids`` the distinct linked positives (first seen), ``unlinked``
|
||||
the admitted images without one, and ``truncated`` that more than the cap
|
||||
matched (the lists then stop at the cap).
|
||||
"""
|
||||
|
||||
targets: list
|
||||
positive_ids: list
|
||||
unlinked: int
|
||||
truncated: bool
|
||||
|
||||
|
||||
# A plain listing's cursor page leaves positive_count out (see
|
||||
# _select_workspace_page); None instead means the count could not be read.
|
||||
_COUNT_OMITTED = object()
|
||||
|
||||
|
||||
def _admitted_entries(
|
||||
def _filtered_page(
|
||||
entries: list,
|
||||
start: int,
|
||||
size: int,
|
||||
filters: dict,
|
||||
links: dict | None,
|
||||
favorites: frozenset | None = None,
|
||||
image_tags: dict | None = None,
|
||||
):
|
||||
"""Yield ``(position, entry, link match)`` for each entry the filters admit.
|
||||
) -> tuple[int, list, int | None]:
|
||||
"""Count the *entries* the filters admit and take *size* from *start* on.
|
||||
|
||||
*links* maps normalized paths to (hit, rating, positive id) for every
|
||||
unambiguously linked image (see PromptDatabase.get_image_link_matches); a
|
||||
category or tag filter admits only those, and ``q`` admits a filename match
|
||||
or a linked-positive hit. *favorites*, when given, admits only those listed
|
||||
paths (``favorites=1``). *image_tags* maps normalized paths to their
|
||||
_ImageTagMatch: a tag filter also admits a file whose current image tags
|
||||
(with its linked positive's tags) carry every tag, and ``q`` also admits an
|
||||
image tag name hit.
|
||||
or a linked-positive hit. Returns the total, the page and the number of
|
||||
distinct linked positives among all admitted entries (None without links).
|
||||
*favorites*, when given, admits only those listed paths (``favorites=1``).
|
||||
*image_tags* maps normalized paths to their _ImageTagMatch: a tag filter
|
||||
also admits a file whose current image tags (with its linked positive's
|
||||
tags) carry every tag, and ``q`` also admits an image tag name hit.
|
||||
"""
|
||||
query, media, folder = filters["query"], filters["media"], filters["folder"]
|
||||
members_only = filters["category"] is not None or bool(filters["tags"])
|
||||
native_folder = folder.replace("/", os.sep)
|
||||
nested = native_folder + os.sep
|
||||
total, page, positives = 0, [], set()
|
||||
exact = filters.get("path", "")
|
||||
for position, entry in enumerate(entries):
|
||||
if exact and entry.folded != exact:
|
||||
@@ -1210,28 +1159,6 @@ def _admitted_entries(
|
||||
and not (image and image.query)
|
||||
):
|
||||
continue
|
||||
yield position, entry, match
|
||||
|
||||
|
||||
def _filtered_page(
|
||||
entries: list,
|
||||
start: int,
|
||||
size: int,
|
||||
filters: dict,
|
||||
links: dict | None,
|
||||
favorites: frozenset | None = None,
|
||||
image_tags: dict | None = None,
|
||||
) -> tuple[int, list, int | None]:
|
||||
"""Count the *entries* the filters admit and take *size* from *start* on.
|
||||
|
||||
Admission is _admitted_entries'. Returns the total, the page and the number
|
||||
of distinct linked positives among all admitted entries (None without
|
||||
links).
|
||||
"""
|
||||
total, page, positives = 0, [], set()
|
||||
for position, entry, match in _admitted_entries(
|
||||
entries, filters, links, favorites, image_tags
|
||||
):
|
||||
total += 1
|
||||
if match is not None:
|
||||
positives.add(match[2])
|
||||
@@ -2164,9 +2091,8 @@ class ImageRoutesMixin:
|
||||
def _image_tag_matches(self, listings: list[tuple], filters: dict) -> dict:
|
||||
"""Normalized path -> _ImageTagMatch for files whose image tags count.
|
||||
|
||||
Looks the relevant tag names up (exact under Tags, a case-folded
|
||||
substring for ``q``) and their files through the tag index, keeping
|
||||
only listed locators; only those candidates are
|
||||
Streams the tagged byte records once and keeps only listed locators
|
||||
carrying a filter tag or a ``q`` hit; only those candidates are
|
||||
stat'ed (never every listed file), and a candidate's tags count only
|
||||
while a byte record is verified at its present ``(size, mtime_ns)``
|
||||
(spec amendment S1). Under a Tags filter the candidates' linked
|
||||
@@ -2175,15 +2101,16 @@ class ImageRoutesMixin:
|
||||
tags, query, category = filters["tags"], filters["query"], filters["category"]
|
||||
wanted = set(tags)
|
||||
located = self._workspace_locators(listings)
|
||||
# The tag index answers which files carry a relevant name (#379); no
|
||||
# read streams every image tag row.
|
||||
tag_ids = self.db.image_tag_ids(wanted, query)
|
||||
candidates = {}
|
||||
if tag_ids:
|
||||
for locator in self.db.image_tag_locators(tag_ids):
|
||||
records = self.db.iter_image_tag_records()
|
||||
try:
|
||||
for locator, _size, _mtime_ns, names in records:
|
||||
entry = located.get(locator)
|
||||
if entry is not None:
|
||||
candidates[locator] = entry
|
||||
if entry is not None and locator not in candidates:
|
||||
if _image_tags_relevant(names, wanted, query):
|
||||
candidates[locator] = entry
|
||||
finally:
|
||||
records.close()
|
||||
stats = {}
|
||||
for locator, entry in candidates.items():
|
||||
fingerprint = _current_fingerprint(entry.path)
|
||||
@@ -2210,23 +2137,35 @@ class ImageRoutesMixin:
|
||||
)
|
||||
return matches
|
||||
|
||||
def _workspace_admission(
|
||||
self, listings: list[tuple], filters: dict, *, read_links: bool, strict=False
|
||||
def _select_workspace_page(
|
||||
self,
|
||||
listings: list[tuple],
|
||||
filters: dict,
|
||||
after: tuple | None,
|
||||
limit: int,
|
||||
scope: str,
|
||||
) -> tuple:
|
||||
"""The ordered entries and what admits them (blocking; run in executor).
|
||||
"""One keyset page of the filtered listing (blocking; run in executor).
|
||||
|
||||
Returns ``(entries, key, links, image_tags, favorites)``: the listing in
|
||||
the filters' order, its keyset key, and the link, image-tag and favorite
|
||||
maps _admitted_entries reads. One SELECT (get_image_link_matches) reads
|
||||
every linked positive when *read_links*. A failed read raises when the
|
||||
filters need it (category, tags, the rating sort) or *strict*; else the
|
||||
listing stays usable without links (a search then matches filenames
|
||||
only). A failed image-tag read raises under a Tags filter; a search then
|
||||
matches without image tags.
|
||||
One SELECT (get_image_link_matches) reads every linked positive of the
|
||||
listing: the filters, the rating sort and the positive count share it.
|
||||
A plain listing (no ``q``, category, tags or rating sort) reads it only
|
||||
for its first page; its cursor pages read no links and the returned
|
||||
count is _COUNT_OMITTED, so the response leaves ``positive_count`` out
|
||||
and the frontend keeps the first page's value. On that plain first
|
||||
page a failed read leaves the listing usable and the count unknown
|
||||
(None); so does a failed read under ``q`` alone, whose search then
|
||||
matches filenames only (as before the linked search existed).
|
||||
"""
|
||||
order, category, tags = filters["order"], filters["category"], filters["tags"]
|
||||
needed = (
|
||||
bool(filters["query"])
|
||||
or category is not None
|
||||
or bool(tags)
|
||||
or order == "rating"
|
||||
)
|
||||
links = None
|
||||
if read_links:
|
||||
if needed or after is None:
|
||||
try:
|
||||
links = self.db.get_image_link_matches(
|
||||
filters["text"],
|
||||
@@ -2236,7 +2175,10 @@ class ImageRoutesMixin:
|
||||
wanted=self._workspace_wanted(listings),
|
||||
)
|
||||
except Exception:
|
||||
if strict or category is not None or tags or order == "rating":
|
||||
# Category, tags and the rating sort cannot be served without
|
||||
# the links. A plain listing and a filename search can: the
|
||||
# search then matches filenames only, and the count is None.
|
||||
if category is not None or tags or order == "rating":
|
||||
raise
|
||||
self.logger.warning(
|
||||
"Workspace prompt links could not be read; %s",
|
||||
@@ -2252,6 +2194,8 @@ class ImageRoutesMixin:
|
||||
try:
|
||||
image_tags = self._image_tag_matches(listings, filters)
|
||||
except Exception:
|
||||
# A Tags filter cannot be served without them (as without the
|
||||
# links); a search then matches without image tags.
|
||||
if tags:
|
||||
raise
|
||||
self.logger.warning(
|
||||
@@ -2290,41 +2234,6 @@ class ImageRoutesMixin:
|
||||
def key(entry):
|
||||
return _workspace_key(entry, order)
|
||||
|
||||
favorites = (
|
||||
self._workspace_favorite_paths(listings) if filters["favorites"] else None
|
||||
)
|
||||
return entries, key, links, image_tags, favorites
|
||||
|
||||
def _select_workspace_page(
|
||||
self,
|
||||
listings: list[tuple],
|
||||
filters: dict,
|
||||
after: tuple | None,
|
||||
limit: int,
|
||||
scope: str,
|
||||
) -> tuple:
|
||||
"""One keyset page of the filtered listing (blocking; run in executor).
|
||||
|
||||
The filters, the rating sort and the positive count share one link
|
||||
read (_workspace_admission). A plain listing (no ``q``, category, tags
|
||||
or rating sort) reads it only for its first page; its cursor pages read
|
||||
no links and the returned count is _COUNT_OMITTED, so the response
|
||||
leaves ``positive_count`` out and the frontend keeps the first page's
|
||||
value. On that plain first page a failed read leaves the listing usable
|
||||
and the count unknown (None); so does a failed read under ``q`` alone,
|
||||
whose search then matches filenames only (as before the linked search
|
||||
existed).
|
||||
"""
|
||||
order = filters["order"]
|
||||
needed = (
|
||||
bool(filters["query"])
|
||||
or filters["category"] is not None
|
||||
or bool(filters["tags"])
|
||||
or order == "rating"
|
||||
)
|
||||
entries, key, links, image_tags, favorites = self._workspace_admission(
|
||||
listings, filters, read_links=needed or after is None
|
||||
)
|
||||
start = 0
|
||||
if after is not None:
|
||||
if order == "name-desc":
|
||||
@@ -2332,6 +2241,9 @@ class ImageRoutesMixin:
|
||||
start = len(entries) - bisect_left(ascending, after, key=key)
|
||||
else:
|
||||
start = bisect_right(entries, after, key=key)
|
||||
favorites = (
|
||||
self._workspace_favorite_paths(listings) if filters["favorites"] else None
|
||||
)
|
||||
total, page, positive_count = _filtered_page(
|
||||
entries, start, limit + 1, filters, links, favorites, image_tags
|
||||
)
|
||||
@@ -2348,61 +2260,6 @@ class ImageRoutesMixin:
|
||||
positive_count,
|
||||
)
|
||||
|
||||
def _select_workspace_matches(
|
||||
self, listings: list[tuple], filters: dict, cap: int
|
||||
) -> ListingSelection:
|
||||
"""Every admitted entry, up to *cap*, as a ListingSelection (blocking).
|
||||
|
||||
The links are read strictly: the positives must be the listing's own.
|
||||
"""
|
||||
entries, _key, links, image_tags, favorites = self._workspace_admission(
|
||||
listings, filters, read_links=True, strict=True
|
||||
)
|
||||
targets, positives, unlinked, scopes = [], {}, 0, {}
|
||||
for _position, entry, match in _admitted_entries(
|
||||
entries, filters, links, favorites, image_tags
|
||||
):
|
||||
if len(targets) == cap:
|
||||
return ListingSelection(targets, list(positives), unlinked, True)
|
||||
scope = scopes.get(entry.index)
|
||||
if scope is None:
|
||||
scope = scopes[entry.index] = self._gallery_root_scope(
|
||||
entry.root, entry.index
|
||||
)
|
||||
relative = entry.path.relative_to(entry.root).as_posix()
|
||||
targets.append((entry.index, scope, relative))
|
||||
if match is None or match[2] is None:
|
||||
unlinked += 1
|
||||
else:
|
||||
positives.setdefault(match[2])
|
||||
return ListingSelection(targets, list(positives), unlinked, False)
|
||||
|
||||
async def resolve_listing_filter(
|
||||
self, params: Mapping, cap: int = MAX_LISTING_SELECTION
|
||||
) -> ListingSelection:
|
||||
"""The images a parse_listing_filter result admits right now (#392).
|
||||
|
||||
Raises ValueError for a root outside the current roots, and
|
||||
ListingSourceChanged when the filter's ``root_scope`` no longer names
|
||||
its root (the sources were renumbered since the listing).
|
||||
"""
|
||||
output_dirs = list(self._get_all_output_dirs())
|
||||
root_index = _workspace_root(params, output_dirs)
|
||||
error = self._gallery_scope_error(params, output_dirs)
|
||||
if error is not None:
|
||||
if error.status == 409:
|
||||
raise ListingSourceChanged()
|
||||
raise ValueError("Invalid gallery root scope")
|
||||
filters = _workspace_filters(params)
|
||||
listings = [
|
||||
(index, root, await self._get_gallery_files(root))
|
||||
for index, root in enumerate(output_dirs)
|
||||
if root_index is None or index == root_index
|
||||
]
|
||||
return await self._run_in_executor(
|
||||
self._select_workspace_matches, listings, filters, cap
|
||||
)
|
||||
|
||||
async def serve_image(self, request):
|
||||
"""Serve the actual image file using streamed FileResponse."""
|
||||
try:
|
||||
|
||||
@@ -129,9 +129,8 @@ def inspect_current_file(
|
||||
profile = (
|
||||
PREVIEW_INSPECTION_PROFILE if _in_host_temp(path) else INSPECTION_PROFILE
|
||||
)
|
||||
# The file name anchors on the saver that wrote it (#381).
|
||||
evidence = inspect_prompt_evidence(
|
||||
inspection.metadata, role_saved, profile=profile, file_name=path.name
|
||||
inspection.metadata, role_saved, profile=profile
|
||||
)
|
||||
return {
|
||||
"file": {
|
||||
|
||||
@@ -1178,8 +1178,6 @@ class PromptRoutesMixin:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Expected a JSON object"}, status=400
|
||||
)
|
||||
if "images_filter" in data:
|
||||
return await self._bulk_category_by_filter(data)
|
||||
if not data.get("prompt_ids"):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "No prompt IDs provided"}, status=400
|
||||
@@ -1216,66 +1214,6 @@ class PromptRoutesMixin:
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def _bulk_category_by_filter(self, data: dict) -> web.Response:
|
||||
"""Set the category on every linked positive a listing filter admits.
|
||||
|
||||
#392 "Select all N matching": ``{images_filter, category}``, the filter
|
||||
being the Images listing's own query (parse_listing_filter), resolved
|
||||
now. The answer adds ``prompt_count`` (the distinct linked positives)
|
||||
and ``skipped`` (matching images without one); none linked changes
|
||||
nothing.
|
||||
"""
|
||||
from .images import (
|
||||
MAX_LISTING_SELECTION,
|
||||
ListingSourceChanged,
|
||||
parse_listing_filter,
|
||||
)
|
||||
|
||||
def refused(message, **extra):
|
||||
return web.json_response(
|
||||
{"success": False, "error": message, **extra}, status=400
|
||||
)
|
||||
|
||||
if set(data) != {"images_filter", "category"}:
|
||||
return refused("Request must contain exactly category, images_filter")
|
||||
category = data["category"]
|
||||
try:
|
||||
params = parse_listing_filter(data["images_filter"])
|
||||
validate_category(category)
|
||||
selection = await self.resolve_listing_filter(params)
|
||||
except ListingSourceChanged:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "source-changed",
|
||||
"error": "The source changed. Refresh the image collection.",
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return refused(str(exc))
|
||||
if selection.truncated:
|
||||
return refused(
|
||||
f"Narrow the view to at most {MAX_LISTING_SELECTION:,} images.",
|
||||
code="too-many-images",
|
||||
)
|
||||
category = (category or "").strip()
|
||||
ids = selection.positive_ids
|
||||
updated = (
|
||||
await self._run_in_executor(self.db.bulk_set_category, ids, category)
|
||||
if ids
|
||||
else 0
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"message": f"Set category for {updated} prompts",
|
||||
"updated_count": updated,
|
||||
"prompt_count": len(ids),
|
||||
"skipped": selection.unlinked,
|
||||
}
|
||||
)
|
||||
|
||||
def _all_prompts_for_export(self):
|
||||
"""Every prompt, fetched in EXPORT_PAGE_SIZE pages (blocking)."""
|
||||
prompts, offset = [], 0
|
||||
|
||||
@@ -756,24 +756,6 @@ class AutoTagService:
|
||||
|
||||
self.logger.info("Model unloaded, memory freed")
|
||||
|
||||
def wd14_device(self) -> str:
|
||||
"""``"gpu"`` when WD14 runs (or would run) on CUDA, else ``"cpu"``.
|
||||
|
||||
A loaded WD14 session answers from its own first provider; otherwise
|
||||
ONNX Runtime's available providers say whether CUDA would be offered
|
||||
(a session can still fall back to the processor). Loads no model.
|
||||
"""
|
||||
tagger = self._tagger
|
||||
if tagger is not None and tagger[0] == "wd14":
|
||||
providers = tagger[1][0].get_providers()
|
||||
return "gpu" if providers[:1] == ["CUDAExecutionProvider"] else "cpu"
|
||||
try:
|
||||
import onnxruntime as ort
|
||||
except ImportError:
|
||||
return "cpu"
|
||||
available = ort.get_available_providers()
|
||||
return "gpu" if "CUDAExecutionProvider" in available else "cpu"
|
||||
|
||||
def is_model_loaded(self) -> bool:
|
||||
"""Check if a model is currently loaded."""
|
||||
return self._tagger is not None
|
||||
|
||||
+1
-17
@@ -13,23 +13,7 @@ LLM_MODEL_TYPES = frozenset({"gguf", "hf"})
|
||||
MODEL_TYPES = WD14_MODEL_TYPES | LLM_MODEL_TYPES
|
||||
MAX_INSTRUCTION_CHARS = 4000
|
||||
MAX_INSTRUCTION_BYTES = 16 * 1024
|
||||
# Tag output images' review estimate (#379): seconds per image for the WD14
|
||||
# engines, by where ONNX Runtime runs them. The graphics card figure was
|
||||
# measured in PR 1's live check (RTX 4090, CUDA: Auto-tag selected on 100
|
||||
# images in 16.7 s with the model load, 0.167 s/image). The processor figure
|
||||
# is an upper bound from the maintainer's cancelled CPU run (19 min / 697
|
||||
# images, backup, model load and skips included), kept as the fallback. The
|
||||
# language-model engines have no figure.
|
||||
WD14_SECONDS_PER_IMAGE = 1.6
|
||||
WD14_GPU_SECONDS_PER_IMAGE = 0.17
|
||||
|
||||
|
||||
def wd14_seconds_per_image(device: str) -> float:
|
||||
"""The estimate's rate for ``"gpu"`` or ``"cpu"`` (anything else)."""
|
||||
return WD14_GPU_SECONDS_PER_IMAGE if device == "gpu" else WD14_SECONDS_PER_IMAGE
|
||||
|
||||
|
||||
# What a batch does with an image that already has tags (#328, #379).
|
||||
# What a batch does with a prompt that already has tags (#328).
|
||||
EXISTING_POLICIES = ("skip", "merge", "replace_after_review")
|
||||
|
||||
# The language-model tag template every page starts from until one is saved.
|
||||
|
||||
@@ -676,73 +676,3 @@ export function measureInspector(selector) {
|
||||
primary:node.classList.contains('ca-primary'),minHeight:getComputedStyle(node).minHeight})),
|
||||
};
|
||||
}
|
||||
|
||||
// #249 System S7 Task 20, shared since S12 (#329): the maintenance plan dialog's measurement and its D100 mapping,
|
||||
// moved unchanged from system-maintenance-concept.spec.mjs so the rotated-log review uses the same ones.
|
||||
/** Runs in the page: the open review dialog (plan or duplicates): its title, body and the concept's tokens. */
|
||||
export function measureReview() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const body = dialog.querySelector('#modal-body');
|
||||
const grid = body.querySelector(':scope > dl.meta-grid');
|
||||
const compare = body.querySelector('.compare-grid');
|
||||
const item = compare?.querySelector('.compare-item');
|
||||
const notes = [...body.querySelectorAll(':scope > p.small')];
|
||||
const note = notes.at(-1);
|
||||
const actions = body.querySelector(':scope > .dialog-actions');
|
||||
return {
|
||||
title: text(dialog.querySelector('#modal-title')),
|
||||
tree: [...body.children].map(name),
|
||||
heading: body.querySelector(':scope > h3') && { text: text(body.querySelector(':scope > h3')),
|
||||
...css(body.querySelector(':scope > h3'), 'fontSize', 'fontWeight', 'color', 'marginBottom') },
|
||||
grid: grid && { rows: [...grid.children].map(row => [text(row.querySelector('dt')), text(row.querySelector('dd'))]),
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
dt: css(grid.querySelector('dt'), 'fontSize', 'color', 'textTransform'), dd: css(grid.querySelector('dd'), 'fontSize', 'color') },
|
||||
details: body.querySelector(':scope > details') && { summary: text(body.querySelector(':scope > details > summary')),
|
||||
tree: [...body.querySelector(':scope > details').children].map(name) },
|
||||
backup: body.querySelector(':scope > .notice') && { tree: [...body.querySelector(':scope > .notice').children].map(name),
|
||||
text: [...body.querySelectorAll(':scope > .notice > div:first-child > *')].map(node => [node.localName, text(node)]),
|
||||
buttons: [...body.querySelectorAll(':scope > .notice button')].map(text) },
|
||||
compare: compare && { tree: [...compare.children].map(name), ...css(compare, 'display', 'gridTemplateColumns', 'columnGap'),
|
||||
items: [...compare.children].map(entry => ({ tree: [...entry.children].map(name), label: text(entry.querySelector('label')),
|
||||
checked: entry.querySelector('input[type=radio]').checked, image: entry.querySelector('img')?.className ?? null })),
|
||||
item: css(item, 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius'),
|
||||
image: css(item.querySelector('img'), 'width', 'objectFit'),
|
||||
label: css(item.querySelector('label'), 'fontSize', 'color'),
|
||||
radio: { name: item.querySelector('input[type=radio]').name.startsWith('keep-') ? 'keep-*' : 'other',
|
||||
type: item.querySelector('input').type } },
|
||||
note: note && { text: text(note), ...css(note, 'fontSize', 'color', 'marginTop') },
|
||||
actions: { tree: [...actions.children].map(name), ...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(item => ({ text: text(item), className: item.className,
|
||||
...css(item, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor') })) },
|
||||
};
|
||||
}
|
||||
|
||||
export const BACKUP_BOX = Object.freeze({ tree: ['div', 'div.row-actions'], text: [['h3', 'Back up the database first (recommended)'],
|
||||
['p', 'Downloads a copy of the prompts database before this operation changes any records.']], buttons: ['Back up now'] });
|
||||
|
||||
/**
|
||||
* D100: the server's plan in the concept's dialog: the operation's row title as its `h3` (the concept prints its
|
||||
* `data-plan` name), the four rows with the plan's values, the listed records (`details`), the hold line in the
|
||||
* concept's `p.small`, the D55 backup box before the actions, and "Apply reviewed operation". No sample note.
|
||||
*/
|
||||
export function mapPlanConcept(reference, title, plan) {
|
||||
const backup = plan.backup.policy === 'off' ? 'No safety backup (the policy is off)'
|
||||
: `Create and verify before changes (${plan.backup.destination_label})`;
|
||||
const tree = ['h3', 'dl.meta-grid', ...(plan.details.length ? ['details'] : []), 'p.small', 'div.notice', 'div.dialog-actions'];
|
||||
const [cancel, apply] = reference.actions.buttons;
|
||||
return {
|
||||
...reference,
|
||||
tree,
|
||||
heading: { ...reference.heading, text: title },
|
||||
grid: { ...reference.grid, rows: [['Scope', plan.scope.label],
|
||||
['Protected', plan.protected.reasons.length ? plan.protected.reasons.join(' · ') : 'Nothing is protected'],
|
||||
['Backup', backup], ['Media on disk', plan.media.label]] },
|
||||
details: plan.details.length ? { summary: `Listed records (${plan.details.length})`, tree: ['summary', 'pre'] } : null,
|
||||
backup: BACKUP_BOX,
|
||||
note: { ...reference.note, text: 'Nothing changes until you apply. This review holds for 15 minutes.' },
|
||||
actions: { ...reference.actions, buttons: [cancel, { ...apply, text: 'Apply reviewed operation' }] },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -95,14 +95,8 @@ async function watchProgress(page) {
|
||||
await page.evaluate(() => {
|
||||
window.progressSeen = [];
|
||||
const node = document.getElementById('autoTagProgressPercent');
|
||||
// Each change from its own record: frames that land in one task share one
|
||||
// callback, and the node's text by then is only the last of them.
|
||||
new MutationObserver(records => {
|
||||
for (const record of records) {
|
||||
if (record.type === 'characterData') window.progressSeen.push(record.target.data);
|
||||
for (const added of record.addedNodes) window.progressSeen.push(added.textContent);
|
||||
}
|
||||
}).observe(node, { childList: true, characterData: true, subtree: true });
|
||||
new MutationObserver(() => window.progressSeen.push(node.textContent))
|
||||
.observe(node, { childList: true, characterData: true, subtree: true });
|
||||
});
|
||||
}
|
||||
|
||||
@@ -114,58 +108,25 @@ async function runGalleryBatch(page) {
|
||||
await page.locator('#startAutoTagBtn').click();
|
||||
await done;
|
||||
await expect(page.locator('#autoTagProgressModal')).toBeHidden();
|
||||
// #379: the batch counts images, never prompts.
|
||||
await expect(page.getByText(/^Auto tagging complete! Tagged \d+ images?\.$/)).toBeVisible();
|
||||
}
|
||||
|
||||
const CLOSE_NOTE = 'Closing this window stops tagging. Tags already added are kept. Prompt Manager\'s Activity page shows the '
|
||||
+ 'time left.';
|
||||
|
||||
async function serverJobs(request) {
|
||||
const response = await request.get('/prompt_manager/jobs');
|
||||
await expect(response).toBeOK();
|
||||
return (await response.json()).jobs;
|
||||
}
|
||||
|
||||
async function runNativeBatch(request) {
|
||||
const started = await request.post('/prompt_manager/jobs/autotag', { data: { model_type: 'wd14-swinv2', existing: 'skip',
|
||||
options: { general_threshold: 0.35, character_threshold: 0.85 } } });
|
||||
expect(started.status()).toBe(202);
|
||||
const { job } = await started.json();
|
||||
await expect.poll(async () => (await serverJobs(request)).find(item => item.id === job.id)?.state).toBe('finished');
|
||||
await expect(page.getByText(/^Auto tagging complete! Applied tags to \d+ prompts\.$/)).toBeVisible();
|
||||
}
|
||||
|
||||
test('gallery batch modal counts progress and closes on complete', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await runGalleryBatch(page);
|
||||
const seen = await page.evaluate(() => window.progressSeen);
|
||||
// The phase frames (model 5%, listing 8%) then the per-image frames (from 10%) move the bar.
|
||||
expect(seen).toContain('8%');
|
||||
expect(seen.some(value => parseInt(value, 10) >= 10)).toBe(true);
|
||||
await expect(page.locator('#autoTagCloseNote')).toHaveText(CLOSE_NOTE);
|
||||
// The batch ran as the autotag job of the legacy page.
|
||||
expect((await serverJobs(request)).filter(job => job.kind === 'autotag').map(job => [job.title, job.state]))
|
||||
.toEqual([['Tag output images (legacy page)', 'finished']]);
|
||||
// The fixture's prompts are already tagged, so the batch skips them; its progress frames still move the bar.
|
||||
expect(seen).toContain('95%');
|
||||
expect((await downloads(request)).length).toBe(0);
|
||||
});
|
||||
|
||||
test('admin batch copy names images, not prompts (#379)', async ({ page }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await expect(page.locator('input[name="autoTagMode"][value="skip"] + span')).toHaveText('Skip images already auto-tagged');
|
||||
await expect(page.getByText('"auto-scanned" is not counted as a real tag')).toHaveCount(0);
|
||||
await expect(page.locator('#autoTagCloseNote')).toHaveText(CLOSE_NOTE);
|
||||
await expect(page.locator('#retagConfirmModal h3')).toHaveText('Image Already Auto-tagged');
|
||||
await expect(page.locator('#retagConfirmModal')).toContainText('A tagging model already tagged this image. Generate new tags?');
|
||||
});
|
||||
|
||||
test('admin review opens the first image', async ({ page }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check();
|
||||
await page.locator('#startReviewBtn').click();
|
||||
// #379: no image is auto-tagged yet (prompt tags no longer count), so the review opens on the first image.
|
||||
await expect(page.locator('#retagConfirmModal')).toBeHidden();
|
||||
// The first image's prompt is already tagged: confirm a new pass, then the review opens on that image.
|
||||
await page.locator('#retagConfirmBtn').click();
|
||||
await expect(page.locator('#autoTagReviewModal')).toBeVisible();
|
||||
await expect(page.locator('#reviewImageName')).not.toHaveText(/^(image_name\.png|Unknown|)$/);
|
||||
await expect(page.getByText('Failed to start review mode')).toHaveCount(0);
|
||||
@@ -189,70 +150,3 @@ test('gallery unload works and is refused while the gate is held', async ({ page
|
||||
await unload.click();
|
||||
await expect(page.getByText('WD14-SWINV2 model unloaded successfully')).toBeVisible();
|
||||
});
|
||||
|
||||
/** The admin review lists every row, videos too: Skip any image that is not one until the confirmation shows. */
|
||||
async function untilAutoTagged(page, confirm) {
|
||||
// Each image's skip check answers before its confirmation or its error shows.
|
||||
const failed = page.locator('#reviewTagsVisible .text-pm-error');
|
||||
for (let tries = 0; tries < 5; tries++) {
|
||||
await expect.poll(async () => (await confirm.isVisible()) || (await failed.count()) > 0).toBe(true);
|
||||
if (await confirm.isVisible()) return;
|
||||
const answered = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/single'));
|
||||
await page.locator('#skipReviewBtn').click();
|
||||
await answered;
|
||||
}
|
||||
await expect(confirm).toBeVisible();
|
||||
}
|
||||
|
||||
// #379: the admin review asks before re-tagging an image a model already tagged, and Apply & Next writes that image's
|
||||
// tags (its bytes, never its prompt).
|
||||
test('admin review: an auto-tagged image asks first; Re-tag then Apply & Next tags the image', async ({ page, request }) => {
|
||||
await runNativeBatch(request);
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check();
|
||||
await page.locator('#startReviewBtn').click();
|
||||
const confirm = page.locator('#retagConfirmModal');
|
||||
await untilAutoTagged(page, confirm);
|
||||
await expect(confirm.locator('#retagExistingTags span').first()).toHaveText('sandstone');
|
||||
await page.locator('#retagConfirmBtn').click();
|
||||
await expect(page.locator('#autoTagReviewModal')).toBeVisible();
|
||||
await expect(page.locator('#reviewTagsVisible .tag-chip').first()).toContainText('sandstone');
|
||||
const applied = page.waitForRequest(item => item.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
const answer = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
await page.locator('#applyReviewBtn').click();
|
||||
const body = (await applied).postDataJSON();
|
||||
expect(body.prompt_id).toBeUndefined();
|
||||
expect(body.sha256).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect((await answer).status()).toBe(200);
|
||||
});
|
||||
|
||||
test('admin review: Skip All Tagged passes every auto-tagged image', async ({ page, request }) => {
|
||||
await runNativeBatch(request);
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check();
|
||||
await page.locator('#startReviewBtn').click();
|
||||
const confirm = page.locator('#retagConfirmModal');
|
||||
await untilAutoTagged(page, confirm);
|
||||
await page.locator('#retagSkipAllBtn').click();
|
||||
await expect(page.locator('#retagConfirmModal')).toBeHidden();
|
||||
await expect(page.getByText('Review complete!')).toBeVisible();
|
||||
});
|
||||
|
||||
test('gallery review: Apply & Next tags the image by its path and bytes', async ({ page }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await page.locator('input[name="autoTagModel"][value="wd14-swinv2"]').check();
|
||||
const suggested = page.waitForRequest(item => item.url().endsWith('/prompt_manager/autotag/single'));
|
||||
await page.locator('#startReviewBtn').click();
|
||||
expect((await suggested).postDataJSON().existing).toBe('merge');
|
||||
await expect(page.locator('#reviewTagsContainer .tag-chip').first()).toContainText('sandstone');
|
||||
const applied = page.waitForRequest(item => item.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
const answer = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
await page.locator('#applyReviewBtn').click();
|
||||
const body = (await applied).postDataJSON();
|
||||
expect(Object.keys(body).sort()).toEqual(['path', 'root_index', 'sha256', 'tags'].filter(key => key in body).sort());
|
||||
expect(body.prompt_id).toBeUndefined();
|
||||
expect((await answer).status()).toBe(200);
|
||||
});
|
||||
|
||||
@@ -47,10 +47,6 @@ from tests.browser.workspace_library_editor_fixture import (
|
||||
from tests.browser.workspace_vocabulary_fixture import register_vocabulary_commands
|
||||
from tests.browser.workspace_tagging_fixture import register_tagging_commands
|
||||
from tests.browser.workspace_review_fixture import register_review_commands
|
||||
from tests.browser.workspace_diagnostics_fixture import (
|
||||
register_diagnostics_commands,
|
||||
repoint_logs,
|
||||
)
|
||||
|
||||
CANONICAL_ROOT = Path(__file__).resolve().parents[2]
|
||||
CANONICAL_ASSETS = {
|
||||
@@ -205,8 +201,6 @@ def create_app(root: Path) -> web.Application:
|
||||
directory.mkdir()
|
||||
setattr(folder_paths, f"get_{name}_directory", lambda path=directory: str(path))
|
||||
sys.modules["folder_paths"] = folder_paths
|
||||
# S12: the logger writes into this server's root before any record is written.
|
||||
repoint_logs(root)
|
||||
|
||||
from py.config import GalleryConfig, PromptManagerConfig
|
||||
|
||||
@@ -231,7 +225,6 @@ def create_app(root: Path) -> web.Application:
|
||||
register_activity_commands(app, routes, api)
|
||||
register_system_commands(app, routes, api, root)
|
||||
register_integrations_commands(app, routes, api, root)
|
||||
reset_logs = register_diagnostics_commands(app, routes, api, root)
|
||||
request_holds = register_request_holds(app, routes)
|
||||
slow_reset_watch.install(app, api)
|
||||
# Resets seed a scratch copy and publish it in one commit (#384).
|
||||
@@ -432,7 +425,6 @@ def create_app(root: Path) -> web.Application:
|
||||
# No preview queued by the last test may land in the new fixture.
|
||||
await api._run_in_executor(api.close_auto_thumbnails)
|
||||
library_editor_commands.reset()
|
||||
await api._run_in_executor(reset_logs)
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
if not isinstance(data, dict):
|
||||
raise web.HTTPBadRequest()
|
||||
|
||||
@@ -1,396 +0,0 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate, measureReview, mapPlanConcept }
|
||||
from './concept-inventory.mjs';
|
||||
|
||||
// #249 System S12 Task 7 (#329): System › Diagnostics & logs (app.js renderSystem 694–695) and its log cleanup review
|
||||
// (the concept's `maintenancePlan("rotated logs")`, 953–958) against the immutable concept, on the concept-shaped
|
||||
// fixture with the logger writing into this server's own root and seeded with the concept's sample (four lines, a
|
||||
// 1.4 MB active log, three rotated copies, console logging off). Each concept measurement is mapped only by the recorded
|
||||
// deviations (D109, D110, and D100 for the review), each commented where it applies; after that the difference must be
|
||||
// empty, and a removed term, a renamed button, a reordered option, a primary Refresh, a squeezed select or a swapped
|
||||
// token must be noticed.
|
||||
test.use({ locale: 'en-US', timezoneId: 'UTC' });
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(await request.post('/__test__/logs/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/logs/seed', { data: { mode: 'concept' } })).toBeOK();
|
||||
page.conceptErrors = [];
|
||||
page.on('pageerror', error => page.conceptErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/logs/reset');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.conceptErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const REPORT_URL = '/prompt_manager/system/diagnostics';
|
||||
const PREVIEW_URL = '/prompt_manager/maintenance/preview';
|
||||
const LINE_FORMAT = /^\d\d:\d\d:\d\d [A-Z]+ +\S/;
|
||||
|
||||
/** Runs in the page: Diagnostics & logs under `mainSelector`: every surface's text, controls, geometry and tokens. */
|
||||
function measureDiagnostics([mainSelector, lineFormat]) {
|
||||
const format = new RegExp(lineFormat);
|
||||
const layout = document.querySelector(`${mainSelector} .settings-layout`);
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const tokens = (node, ...names) => Object.fromEntries(names.map(key => [key, getComputedStyle(node).getPropertyValue(key).trim()]));
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const button = node => ({ text: text(node), className: node.className });
|
||||
const control = row => {
|
||||
const select = row.querySelector(':scope > select');
|
||||
if (select) return { kind: 'select', id: select.id, options: [...select.options].map(option => option.textContent),
|
||||
selected: select.selectedOptions[0]?.textContent ?? null, disabled: select.disabled };
|
||||
const toggle = row.querySelector(':scope > * input[type="checkbox"]');
|
||||
return toggle ? { kind: 'switch', id: toggle.id, wrapper: name(toggle.parentElement), checked: toggle.checked,
|
||||
disabled: toggle.disabled } : null;
|
||||
};
|
||||
const operations = parent => [...parent.querySelectorAll(':scope > .operation')].map(row => ({ tree: [...row.children].map(name),
|
||||
title: text(row.querySelector('h3')), description: text(row.querySelector('p')),
|
||||
actions: [...row.querySelectorAll('.row-actions > button')].map(button) }));
|
||||
const toolbarItem = node => ({ name: name(node), text: node.localName === 'select' || node.localName === 'input' ? null : text(node),
|
||||
for: node.localName === 'label' ? node.htmlFor : null, id: node.id || null,
|
||||
options: node.localName === 'select' ? [...node.options].map(option => option.textContent) : null,
|
||||
// The chosen option, so a wrong default level is noticed.
|
||||
selected: node.localName === 'select' ? node.selectedOptions[0]?.textContent ?? null : null,
|
||||
input: node.localName === 'input' ? [node.type, node.min, node.max, node.value] : null,
|
||||
className: node.localName === 'button' ? node.className : null });
|
||||
const surfaces = [...layout.querySelectorAll(':scope > section.surface')];
|
||||
const pre = layout.querySelector('pre');
|
||||
const lines = (pre?.textContent ?? '').split('\n').filter(Boolean);
|
||||
const details = layout.querySelector('details');
|
||||
const firstRow = layout.querySelector('.setting-row');
|
||||
const toolbar = layout.querySelector('.toolbar');
|
||||
return {
|
||||
layout: [...layout.children].map(name),
|
||||
surfaces: surfaces.map(surface => {
|
||||
const heading = surface.querySelector(':scope > .panel-heading');
|
||||
const grid = surface.querySelector(':scope > .meta-grid');
|
||||
return {
|
||||
tree: [...surface.children].map(name),
|
||||
heading: text(surface.querySelector(':scope > h2, :scope > .panel-heading > h2')),
|
||||
panelHeading: heading && [...heading.children].map(node => [name(node), text(node)]),
|
||||
grid: grid && [...grid.children].map(entry => [name(entry), text(entry.querySelector('dt')), text(entry.querySelector('dd'))]),
|
||||
buttons: [...surface.querySelectorAll(':scope > button')].map(button),
|
||||
toolbar: surface.querySelector(':scope > .toolbar') && [...surface.querySelector(':scope > .toolbar').children].map(toolbarItem),
|
||||
pre: surface.querySelector(':scope > pre') && { tabIndex: pre.tabIndex, tree: [...pre.children].map(name), lines,
|
||||
format: lines.every(line => format.test(line)) },
|
||||
notes: [...surface.querySelectorAll(':scope > p')].map(node => [name(node), text(node), node.getAttribute('role'),
|
||||
node.getAttribute('aria-label')]),
|
||||
rows: [...surface.querySelectorAll(':scope > .setting-row')].map(row => ({ tree: [...row.children].map(name),
|
||||
text: [...row.firstElementChild.children].map(node => [name(node), text(node), node.localName === 'label' ? node.htmlFor : null]),
|
||||
control: control(row) })),
|
||||
details: surface.querySelector(':scope > details') && {
|
||||
summary: text(details.querySelector(':scope > summary')), tree: [...details.children].map(name),
|
||||
notes: [...details.querySelectorAll(':scope > p')].map(node => [name(node), text(node)]),
|
||||
buttons: [...details.querySelectorAll(':scope > button')].map(button),
|
||||
files: [...details.querySelectorAll(':scope > .operation-list > .operation')].map(row => ({
|
||||
title: text(row.querySelector('h3')), detail: text(row.querySelector('p')), actions: [...row.querySelectorAll('.row-actions > button')].map(item =>
|
||||
[text(item), item.getAttribute('aria-label')]) })),
|
||||
operations: operations(details) },
|
||||
operations: operations(surface),
|
||||
};
|
||||
}),
|
||||
tokens: tokens(surfaces[0], '--line', '--panel', '--accent'),
|
||||
surface: css(surfaces[1], 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius',
|
||||
'marginTop'),
|
||||
grid: css(layout.querySelector('.meta-grid'), 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
toolbar: css(toolbar, 'display', 'flexWrap', 'alignItems', 'columnGap', 'rowGap', 'marginBottom'),
|
||||
pre: css(pre, 'fontFamily', 'fontSize', 'backgroundColor', 'borderTopColor', 'borderTopLeftRadius', 'paddingTop', 'overflowX'),
|
||||
row: css(firstRow, 'display', 'alignItems', 'justifyContent', 'columnGap', 'paddingTop', 'paddingBottom', 'borderTopWidth',
|
||||
'borderTopColor'),
|
||||
label: css(firstRow.querySelector('label'), 'fontSize', 'fontWeight', 'color'),
|
||||
description: css(firstRow.querySelector('p'), 'fontSize', 'color', 'marginTop'),
|
||||
select: css(firstRow.querySelector('select'), 'fontSize', 'minWidth', 'maxWidth', 'backgroundColor', 'color', 'borderTopColor'),
|
||||
summary: css(details.querySelector('summary'), 'fontSize', 'fontWeight', 'color', 'cursor'),
|
||||
small: css(details.querySelector(':scope > p.small'), 'fontSize', 'color'),
|
||||
// Whether each select is as wide as the browser lays out a copy holding only its chosen option, i.e. it shows
|
||||
// that option whole (the browser's own select sizing, arrow included).
|
||||
selectsFit: [...layout.querySelectorAll('select')].map(select => {
|
||||
const copy = select.cloneNode(false);
|
||||
copy.removeAttribute('id');
|
||||
copy.append(new Option(select.selectedOptions[0]?.textContent ?? ''));
|
||||
Object.assign(copy.style, { position: 'absolute', visibility: 'hidden', width: 'max-content', minWidth: '0',
|
||||
maxWidth: 'none', flex: 'none' });
|
||||
select.parentElement.append(copy);
|
||||
const needed = copy.getBoundingClientRect().width;
|
||||
copy.remove();
|
||||
return select.getBoundingClientRect().width + 0.5 >= needed;
|
||||
}),
|
||||
pageScroll: document.documentElement.scrollWidth > document.documentElement.clientWidth,
|
||||
};
|
||||
}
|
||||
|
||||
const plural = (count, one, many = `${one}s`) => `${count.toLocaleString('en-US')} ${count === 1 ? one : many}`;
|
||||
/** The server's size label (1 KB = 1024 bytes), as the section prints it. */
|
||||
function sizeText(bytes) {
|
||||
if (bytes < 1024) return plural(bytes, 'byte');
|
||||
if (bytes < 2 ** 20) return `${(bytes / 2 ** 10).toFixed(1)} KB`;
|
||||
if (bytes < 2 ** 30) return `${(bytes / 2 ** 20).toFixed(1)} MB`;
|
||||
return `${(bytes / 2 ** 30).toFixed(1)} GB`;
|
||||
}
|
||||
|
||||
const KINDS = Object.freeze({ active: 'Active log', rotated: 'Rotated copy', other: 'Other file' });
|
||||
const RENDERER_NONE = 'No ComfyUI window is open in this browser, so its renderer is not known.';
|
||||
const COPY = Object.freeze({
|
||||
testLink: 'Write a placeholder image record to check that linking works.',
|
||||
runtimeLevel: 'How much the server writes to the log. Applied until the server restarts.',
|
||||
console: 'Print Prompt Manager’s warnings and errors in the ComfyUI console (everything at DEBUG). Applied until the server restarts.',
|
||||
clear: 'Empty the active log and delete its rotated copies.',
|
||||
retention: 'Rotates at 10 MB · keeps 5 rotated copies · file logging on · memory buffer 1,000 entries',
|
||||
});
|
||||
|
||||
/**
|
||||
* The concept's Diagnostics & logs with the recorded deviations applied, for the server's `report` and the seeded
|
||||
* `files` (`/logs/files`). The concept's own sample values the fixture reproduces stay as they are: the four log lines,
|
||||
* the Viewer.js line, "prompt_manager.log · 1.4 MB · rotated copies: 3", INFO and console logging off.
|
||||
*/
|
||||
function mapDiagnosticsConcept(reference, { report, files, stats }) {
|
||||
const [diagnostics, logs] = reference.surfaces;
|
||||
const [levelLabel, level, refresh, download] = logs.toolbar;
|
||||
const [runtime, consoleRow] = logs.rows;
|
||||
const database = report.checks.database;
|
||||
const { configured, watching, unavailable } = report.sources;
|
||||
const sources = configured === 0 ? `ComfyUI output folder · ${watching ? 'watching' : 'manual scan'}`
|
||||
: unavailable > 0 ? `${plural(configured, 'source')} · ${unavailable} not available`
|
||||
: `${plural(configured, 'source')} · ${watching} watching`;
|
||||
const errors = (stats.level_counts.ERROR ?? 0) + (stats.level_counts.CRITICAL ?? 0);
|
||||
return {
|
||||
...reference,
|
||||
surfaces: [
|
||||
{ ...diagnostics,
|
||||
// D110: the Test image link operation follows the concept's Image-link diagnostics.
|
||||
tree: [...diagnostics.tree, 'div.operation'],
|
||||
// D110: live values. Database and Sources from `GET /system/diagnostics`, Host renderers is D106's
|
||||
// renderer line (no ComfyUI window in this browser); the pinned Viewer.js line is the concept's own.
|
||||
grid: diagnostics.grid.map(([entry, term, value]) => [entry, term, {
|
||||
Database: `Readable · ${sizeText(report.database_info.database_size_bytes)} · ${plural(database.image_count, 'image')}`,
|
||||
Sources: sources,
|
||||
'Host renderers': RENDERER_NONE }[term] ?? value]),
|
||||
// D110: the buttons drop the concept's "sample".
|
||||
buttons: diagnostics.buttons.map(item => ({ ...item, text: item.text.replace('Download sample diagnostic report',
|
||||
'Download diagnostic report') })),
|
||||
operations: [
|
||||
// D110: "Inspect" opens the read-only Inspect a file page.
|
||||
...diagnostics.operations.map(row => ({ ...row, actions: row.actions.map(item => ({ ...item, text: 'Inspect' })) })),
|
||||
// D110: the legacy synthetic link (S12-11), enabled: the fixture has a most recent prompt.
|
||||
{ ...diagnostics.operations[0], title: 'Test image link', description: COPY.testLink,
|
||||
actions: [{ ...diagnostics.operations[0].actions[0], text: 'Create test link…' }] },
|
||||
] },
|
||||
{ ...logs,
|
||||
// D109: the statistics line follows the viewer.
|
||||
tree: [logs.tree[0], logs.tree[1], logs.tree[2], 'p.small', ...logs.tree.slice(3)],
|
||||
toolbar: [
|
||||
levelLabel,
|
||||
// D109: all six minimum levels (the concept has three).
|
||||
{ ...level, options: ['All levels', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] },
|
||||
// D109: the added Lines control, 1–500, default 100.
|
||||
{ name: 'label.small', text: 'Lines', for: 'log-limit', id: null, options: null, selected: null, input: null,
|
||||
className: null },
|
||||
{ name: 'input', text: null, for: null, id: 'log-limit', options: null, selected: null,
|
||||
input: ['number', '1', '500', '100'], className: null },
|
||||
refresh,
|
||||
// D109: the active log, not a sample.
|
||||
{ ...download, text: 'Download log' },
|
||||
],
|
||||
// D109: "Log statistics" (errors are ERROR + CRITICAL).
|
||||
notes: [['p.small', `${plural(stats.buffer_count, 'entry', 'entries')} in memory · ${plural(errors, 'error')} · `
|
||||
+ `${plural(stats.level_counts.WARNING ?? 0, 'warning')} · ${plural(stats.log_files_count, 'file')} · `
|
||||
+ `${sizeText(stats.total_log_size)} on disk`, 'status', 'Log statistics']],
|
||||
rows: [
|
||||
// D109: CRITICAL after the concept's levels; the value lasts until the server restarts.
|
||||
{ ...runtime, text: [runtime.text[0], ['p', COPY.runtimeLevel, null]],
|
||||
control: { ...runtime.control, options: [...runtime.control.options, 'CRITICAL'] } },
|
||||
// D109: what the console setting does and that it lasts until the server restarts (off, as the
|
||||
// concept's sample, by the fixture's seed).
|
||||
{ ...consoleRow, text: [consoleRow.text[0], ['p', COPY.console, null]] },
|
||||
],
|
||||
details: { ...logs.details,
|
||||
// D109: the retention line, the files and Clear all logs follow the concept's line and review.
|
||||
tree: [...logs.details.tree, 'p.small', 'div.operation-list', 'div.operation'],
|
||||
notes: [...logs.details.notes, ['p.small', COPY.retention]],
|
||||
files: files.map(row => ({ title: row.filename, detail: `${sizeText(row.size)} · ${row.modified.slice(0, 16)
|
||||
.replace('T', ' ')} · ${KINDS[row.kind]}`, actions: [['Download', `Download ${row.filename}`]] })),
|
||||
operations: [{ tree: ['div', 'div.row-actions'], title: 'Clear all logs', description: COPY.clear,
|
||||
actions: [{ text: 'Clear all logs…', className: 'danger' }] }] } },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptDiagnostics(page, theme) {
|
||||
await openConcept(page, 'system', theme);
|
||||
await page.locator('#main .section-nav').getByRole('button', { name: 'Diagnostics & logs', exact: true }).click();
|
||||
await expect(page.locator('#main .settings-layout > section.surface')).toHaveCount(2);
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureDiagnostics, ['#main', LINE_FORMAT.source]);
|
||||
}
|
||||
|
||||
/** Production's section once every read answered, with the server's report, files and statistics it shows. */
|
||||
async function productionDiagnostics(page, request, theme) {
|
||||
await openProduction(page, 'system/diagnostics', theme);
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(main.locator('.settings-layout > section.surface')).toHaveCount(2);
|
||||
await expect(main.locator('.meta-grid dd')).not.toContainText(['Loading…']);
|
||||
await expect(main.locator('.meta-grid dd').last()).toHaveText(RENDERER_NONE);
|
||||
await expect(main.locator('#log-lines')).not.toHaveText('Loading…');
|
||||
await expect(main.locator('#log-stats')).not.toHaveText('Loading…');
|
||||
await expect(main.locator('details > p.small')).not.toContainText(['Loading…']);
|
||||
await expect(main.locator('#runtime-level')).toBeEnabled();
|
||||
await expect(main.locator('.operation').filter({ hasText: 'Create test link…' }).locator('button')).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await page.mouse.move(0, 0);
|
||||
const production = await page.evaluate(measureDiagnostics, ['#workspace-main', LINE_FORMAT.source]);
|
||||
const read = async url => (await request.get(url)).json();
|
||||
const report = await read(REPORT_URL);
|
||||
const { files } = await read('/prompt_manager/logs/files');
|
||||
const { stats } = await read('/prompt_manager/logs/stats');
|
||||
return { production, report, files, stats };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`diagnostics and logs match the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const concept = await conceptDiagnostics(page, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-diagnostics-${theme}-${width}.png`), fullPage: true });
|
||||
const { production, ...live } = await productionDiagnostics(page, request, theme);
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-diagnostics-${theme}-${width}.png`), fullPage: true });
|
||||
expect(diffInventory(mapDiagnosticsConcept(concept, live), production)).toEqual([]);
|
||||
expect(production.surfaces[1].pre.format).toBe(true);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
}
|
||||
|
||||
// Below the concept's smallest size: on a 320px phone the Level and Runtime log level selects still show their whole
|
||||
// option, without page scroll.
|
||||
for (const theme of THEMES) test(`diagnostics selects fit at 320 wide ${theme}`, async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 320, height: 700 });
|
||||
const { production } = await productionDiagnostics(page, request, theme);
|
||||
expect(production.selectsFit).toEqual([true, true]);
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
|
||||
/**
|
||||
* D100 with D109's rule for this plan: the server's `rotated_logs` plan in the concept's dialog, "Rotated log copies"
|
||||
* as its `h3`, the plan's grid values (its Backup row says none is needed), "Listed files (N)", the hold line, no D55
|
||||
* backup box (nothing in the library changes) and "Apply reviewed operation".
|
||||
*/
|
||||
function mapCleanupConcept(reference, plan) {
|
||||
const mapped = mapPlanConcept(reference, 'Rotated log copies', plan);
|
||||
return {
|
||||
...mapped,
|
||||
// D109: no backup box for a plan that changes no library record.
|
||||
tree: mapped.tree.filter(item => item !== 'div.notice'),
|
||||
grid: { ...mapped.grid, rows: mapped.grid.rows.map(([term, value]) =>
|
||||
[term, term === 'Backup' ? 'Not needed: no library record changes' : value]) },
|
||||
backup: null,
|
||||
// D109: the plan lists files, not records.
|
||||
details: { ...mapped.details, summary: `Listed files (${plan.details.length})` },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptCleanup(page, theme) {
|
||||
await conceptDiagnostics(page, theme);
|
||||
await page.locator('#main details > summary').click();
|
||||
await page.locator('#main [data-plan="rotated logs"]').click();
|
||||
await expect(page.locator('dialog#modal[open] #modal-title')).toHaveText('Review maintenance plan');
|
||||
await page.mouse.move(0, 0);
|
||||
return page.evaluate(measureReview);
|
||||
}
|
||||
|
||||
async function productionCleanup(page, request, theme) {
|
||||
await productionDiagnostics(page, request, theme);
|
||||
await page.locator('#workspace-main details > summary').click();
|
||||
const answer = page.waitForResponse(response => response.url().endsWith(PREVIEW_URL));
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Review log cleanup', exact: true }).click();
|
||||
const plan = await (await answer).json();
|
||||
await expect(page.locator('dialog#modal[open] #modal-body > h3')).toHaveText('Rotated log copies');
|
||||
await page.mouse.move(0, 0);
|
||||
return { plan, production: await page.evaluate(measureReview) };
|
||||
}
|
||||
|
||||
for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
test(`log cleanup review matches the concept ${theme} ${width}x${height}`, async ({ page, request }, testInfo) => {
|
||||
await page.setViewportSize({ width, height });
|
||||
const concept = await conceptCleanup(page, theme);
|
||||
expect(concept.heading.text).toBe('rotated logs');
|
||||
expect(concept.tree).toEqual(['p.demo-note', 'h3', 'dl.meta-grid', 'p.small', 'div.dialog-actions']);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-log-cleanup-${theme}-${width}.png`) });
|
||||
const { plan, production } = await productionCleanup(page, request, theme);
|
||||
expect(plan.operation).toBe('rotated_logs');
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-log-cleanup-${theme}-${width}.png`) });
|
||||
expect(diffInventory(mapCleanupConcept(concept, plan), production)).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
// Every path where production differs from the unmapped concept: D110's live values (Viewer.js is the concept's own),
|
||||
// renamed buttons and added operation, and D109's toolbar, statistics line, runtime rows and log files.
|
||||
const DEVIATED_PATHS = [
|
||||
'surfaces.0.buttons.0', 'surfaces.0.grid.0', 'surfaces.0.grid.1', 'surfaces.0.grid.3', 'surfaces.0.operations.0',
|
||||
'surfaces.0.operations.1', 'surfaces.0.operations.length', 'surfaces.0.tree.4', 'surfaces.0.tree.length',
|
||||
'surfaces.1.details.files', 'surfaces.1.details.notes', 'surfaces.1.details.operations', 'surfaces.1.details.tree',
|
||||
'surfaces.1.notes.0', 'surfaces.1.notes.length', 'surfaces.1.rows.0', 'surfaces.1.rows.1', 'surfaces.1.toolbar.1',
|
||||
'surfaces.1.toolbar.2', 'surfaces.1.toolbar.3', 'surfaces.1.toolbar.4', 'surfaces.1.toolbar.5', 'surfaces.1.toolbar.length',
|
||||
'surfaces.1.tree.3', 'surfaces.1.tree.5', 'surfaces.1.tree.6', 'surfaces.1.tree.length'].sort();
|
||||
|
||||
test('diagnostics comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const concept = await conceptDiagnostics(page, 'dark');
|
||||
const { production, ...live } = await productionDiagnostics(page, request, 'dark');
|
||||
const expected = mapDiagnosticsConcept(concept, live);
|
||||
expect(diffInventory(expected, production)).toEqual([]);
|
||||
// Unmapped, the section differs only where D109 and D110 are recorded.
|
||||
expect(diffInventory(concept, production).map(item => item.path.split('.').slice(0, 4).join('.'))
|
||||
.filter((path, index, all) => all.indexOf(path) === index).sort()).toEqual(DEVIATED_PATHS);
|
||||
const measure = () => page.evaluate(measureDiagnostics, ['#workspace-main', LINE_FORMAT.source]);
|
||||
const paths = async () => diffInventory(expected, await measure()).map(item => item.path);
|
||||
// A removed term.
|
||||
const undoTerm = await mutate(page, { remove: ['#workspace-main .meta-grid > div:nth-child(2)'] });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.0.grid.length']));
|
||||
await undoTerm();
|
||||
// A renamed button.
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main section.surface > button').textContent = 'Download report'; });
|
||||
expect(await paths()).toEqual(['surfaces.0.buttons.0.text']);
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main section.surface > button').textContent = 'Download diagnostic report'; });
|
||||
// A reordered level option.
|
||||
await page.evaluate(() => { const select = document.querySelector('#workspace-main #log-level'); select.append(select.options[1]); });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.1.toolbar.1.options.1']));
|
||||
await page.evaluate(() => { const select = document.querySelector('#workspace-main #log-level'); select.insertBefore(select.options[5], select.options[1]); });
|
||||
// A wrong default level.
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main #log-level').value = 'INFO'; });
|
||||
expect(await paths()).toEqual(['surfaces.1.toolbar.1.selected']);
|
||||
await page.evaluate(() => { document.querySelector('#workspace-main #log-level').value = ''; });
|
||||
// A primary Refresh.
|
||||
await page.evaluate(() => { document.querySelectorAll('#workspace-main .toolbar > button')[0].classList.add('ca-primary'); });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['surfaces.1.toolbar.4.className']));
|
||||
await page.evaluate(() => { document.querySelectorAll('#workspace-main .toolbar > button')[0].classList.remove('ca-primary'); });
|
||||
// A select squeezed until its chosen option is cut.
|
||||
const undoSqueeze = await mutate(page, { css: '#workspace-main #runtime-level { max-width: 30px !important; min-width: 0 !important; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['selectsFit.1']));
|
||||
await undoSqueeze();
|
||||
// A swapped token.
|
||||
const undoToken = await mutate(page, { css: '#workspace-main .settings-layout { --line: rgb(255, 0, 0); }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['tokens.--line', 'row.borderTopColor']));
|
||||
await undoToken();
|
||||
expect(diffInventory(expected, await measure())).toEqual([]);
|
||||
|
||||
// The log cleanup review: a changed grid term, a backup box that came back and a grid token are noticed.
|
||||
const conceptReview = await conceptCleanup(page, 'dark');
|
||||
const { plan } = await productionCleanup(page, request, 'dark');
|
||||
const expectedReview = mapCleanupConcept(conceptReview, plan);
|
||||
const reviewPaths = async () => diffInventory(expectedReview, await page.evaluate(measureReview)).map(item => item.path);
|
||||
expect(await reviewPaths()).toEqual([]);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[2].textContent = 'Safety'; });
|
||||
expect(await reviewPaths()).toEqual(['grid.rows.2.0']);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[2].textContent = 'Backup'; });
|
||||
const undoGrid = await mutate(page, { css: 'dialog#modal dl.meta-grid { column-gap: 3px; }' });
|
||||
expect(await reviewPaths()).toEqual(expect.arrayContaining(['grid.columnGap']));
|
||||
await undoGrid();
|
||||
expect(await reviewPaths()).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate, measureReview, mapPlanConcept }
|
||||
from './concept-inventory.mjs';
|
||||
import { SIZES, THEMES, seedConcept, openConcept, openProduction, diffInventory, mutate } from './concept-inventory.mjs';
|
||||
|
||||
// #249 System S7 Task 20: System › Maintenance (app.js renderSystem 688–690), its plan dialog (`maintenancePlan`,
|
||||
// 953–958) for each operation and its duplicates dialog (`duplicates`, 1457–1461) against the immutable concept, on
|
||||
@@ -104,6 +103,47 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
});
|
||||
}
|
||||
|
||||
/** Runs in the page: the open review dialog (plan or duplicates): its title, body and the concept's tokens. */
|
||||
function measureReview() {
|
||||
const dialog = document.querySelector('dialog#modal[open]');
|
||||
const css = (node, ...keys) => node ? Object.fromEntries(keys.map(key => [key, getComputedStyle(node)[key]])) : null;
|
||||
const name = node => [node.localName, ...[...node.classList].sort()].join('.');
|
||||
const text = node => node?.textContent.trim() ?? null;
|
||||
const body = dialog.querySelector('#modal-body');
|
||||
const grid = body.querySelector(':scope > dl.meta-grid');
|
||||
const compare = body.querySelector('.compare-grid');
|
||||
const item = compare?.querySelector('.compare-item');
|
||||
const notes = [...body.querySelectorAll(':scope > p.small')];
|
||||
const note = notes.at(-1);
|
||||
const actions = body.querySelector(':scope > .dialog-actions');
|
||||
return {
|
||||
title: text(dialog.querySelector('#modal-title')),
|
||||
tree: [...body.children].map(name),
|
||||
heading: body.querySelector(':scope > h3') && { text: text(body.querySelector(':scope > h3')),
|
||||
...css(body.querySelector(':scope > h3'), 'fontSize', 'fontWeight', 'color', 'marginBottom') },
|
||||
grid: grid && { rows: [...grid.children].map(row => [text(row.querySelector('dt')), text(row.querySelector('dd'))]),
|
||||
...css(grid, 'display', 'gridTemplateColumns', 'columnGap', 'rowGap'),
|
||||
dt: css(grid.querySelector('dt'), 'fontSize', 'color', 'textTransform'), dd: css(grid.querySelector('dd'), 'fontSize', 'color') },
|
||||
details: body.querySelector(':scope > details') && { summary: text(body.querySelector(':scope > details > summary')),
|
||||
tree: [...body.querySelector(':scope > details').children].map(name) },
|
||||
backup: body.querySelector(':scope > .notice') && { tree: [...body.querySelector(':scope > .notice').children].map(name),
|
||||
text: [...body.querySelectorAll(':scope > .notice > div:first-child > *')].map(node => [node.localName, text(node)]),
|
||||
buttons: [...body.querySelectorAll(':scope > .notice button')].map(text) },
|
||||
compare: compare && { tree: [...compare.children].map(name), ...css(compare, 'display', 'gridTemplateColumns', 'columnGap'),
|
||||
items: [...compare.children].map(entry => ({ tree: [...entry.children].map(name), label: text(entry.querySelector('label')),
|
||||
checked: entry.querySelector('input[type=radio]').checked, image: entry.querySelector('img')?.className ?? null })),
|
||||
item: css(item, 'paddingTop', 'paddingLeft', 'backgroundColor', 'borderTopWidth', 'borderTopColor', 'borderTopLeftRadius'),
|
||||
image: css(item.querySelector('img'), 'width', 'objectFit'),
|
||||
label: css(item.querySelector('label'), 'fontSize', 'color'),
|
||||
radio: { name: item.querySelector('input[type=radio]').name.startsWith('keep-') ? 'keep-*' : 'other',
|
||||
type: item.querySelector('input').type } },
|
||||
note: note && { text: text(note), ...css(note, 'fontSize', 'color', 'marginTop') },
|
||||
actions: { tree: [...actions.children].map(name), ...css(actions, 'display', 'justifyContent', 'columnGap', 'marginTop'),
|
||||
buttons: [...actions.children].map(item => ({ text: text(item), className: item.className,
|
||||
...css(item, 'backgroundColor', 'color', 'fontSize', 'paddingTop', 'paddingLeft', 'borderTopColor') })) },
|
||||
};
|
||||
}
|
||||
|
||||
// The concept's plan names (`data-plan`) and production's operation, row and button for each.
|
||||
const PLANS = Object.freeze([
|
||||
['duplicate prompts', 'duplicate_prompts', 'Duplicate prompt text'],
|
||||
@@ -111,6 +151,32 @@ const PLANS = Object.freeze([
|
||||
['unused prompt components', 'unused_components', 'Unused prompt components'],
|
||||
['missing-file links', 'missing_links', 'Missing-file links'],
|
||||
]);
|
||||
const BACKUP_BOX = Object.freeze({ tree: ['div', 'div.row-actions'], text: [['h3', 'Back up the database first (recommended)'],
|
||||
['p', 'Downloads a copy of the prompts database before this operation changes any records.']], buttons: ['Back up now'] });
|
||||
|
||||
/**
|
||||
* D100: the server's plan in the concept's dialog: the operation's row title as its `h3` (the concept prints its
|
||||
* `data-plan` name), the four rows with the plan's values, the listed records (`details`), the hold line in the
|
||||
* concept's `p.small`, the D55 backup box before the actions, and "Apply reviewed operation". No sample note.
|
||||
*/
|
||||
function mapPlanConcept(reference, title, plan) {
|
||||
const backup = plan.backup.policy === 'off' ? 'No safety backup (the policy is off)'
|
||||
: `Create and verify before changes (${plan.backup.destination_label})`;
|
||||
const tree = ['h3', 'dl.meta-grid', ...(plan.details.length ? ['details'] : []), 'p.small', 'div.notice', 'div.dialog-actions'];
|
||||
const [cancel, apply] = reference.actions.buttons;
|
||||
return {
|
||||
...reference,
|
||||
tree,
|
||||
heading: { ...reference.heading, text: title },
|
||||
grid: { ...reference.grid, rows: [['Scope', plan.scope.label],
|
||||
['Protected', plan.protected.reasons.length ? plan.protected.reasons.join(' · ') : 'Nothing is protected'],
|
||||
['Backup', backup], ['Media on disk', plan.media.label]] },
|
||||
details: plan.details.length ? { summary: `Listed records (${plan.details.length})`, tree: ['summary', 'pre'] } : null,
|
||||
backup: BACKUP_BOX,
|
||||
note: { ...reference.note, text: 'Nothing changes until you apply. This review holds for 15 minutes.' },
|
||||
actions: { ...reference.actions, buttons: [cancel, { ...apply, text: 'Apply reviewed operation' }] },
|
||||
};
|
||||
}
|
||||
|
||||
async function conceptPlan(page, theme, name) {
|
||||
await conceptMaintenance(page, theme);
|
||||
|
||||
@@ -33,7 +33,7 @@ test.afterEach(async ({ page, request, browserName }) => {
|
||||
const PREVIEW_URL = '/prompt_manager/autotag/batch/preview';
|
||||
const MODEL_OPTIONS = ['WD14 SwinV2 · ~400 MB', 'WD14 ViT · ~400 MB', 'GGUF JoyCaption · ~4.5 GB', 'HuggingFace JoyCaption · ~16 GB'];
|
||||
const SUGGESTIONS = 'From 1 to 100: the most suggestions Generate shows for one image.';
|
||||
const BATCH = 'Tag each output image from its own pixels with the model above. Prompts are not changed. Review the scope first.';
|
||||
const BATCH = 'Add tags to each prompt from its linked output images with the model above. Review the scope first.';
|
||||
|
||||
/** Runs in the page: Auto-tagging's surface under `mainSelector`: rows, field, actions, operation, geometry and tokens. */
|
||||
function measureTagging(mainSelector) {
|
||||
@@ -127,9 +127,9 @@ function mapTaggingConcept(reference, { template, phone }) {
|
||||
actions: { ...reference.actions, buttons: [preview, ['Download again', ''], ['Unload', '']] },
|
||||
// D134: the prompt-tag batch, reviewed first.
|
||||
operations: [{ tree: ['div', 'div.row-actions'], title: 'Tag output images', description: BATCH, actions: [['Review', '']] }],
|
||||
// D132 (#398): every select shows its whole chosen option at every width (at 390 the concept's 120px cap cuts
|
||||
// them; production lets each grow past it).
|
||||
selectsFit: reference.selectsFit.map(() => true),
|
||||
// D132: the engine select shows its whole chosen option at every width (at 390 the concept's 120px cap cuts it;
|
||||
// `#tag-model` alone may grow there). Existing tags keeps the concept's own fit, cut or not.
|
||||
selectsFit: reference.selectsFit.map((fit, index) => (index === 0 ? true : fit)),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -168,28 +168,16 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
});
|
||||
}
|
||||
|
||||
// D132 (#398) at phone width and below the concept's smallest size: every option of every select, once chosen, shows
|
||||
// whole, with no page scroll.
|
||||
for (const theme of THEMES) for (const width of [390, 320]) {
|
||||
test(`auto-tagging selects fit every option at ${width} wide ${theme}`, async ({ page }, testInfo) => {
|
||||
await page.setViewportSize({ width, height: 844 });
|
||||
await productionTagging(page, theme);
|
||||
await page.locator('#workspace-main #tag-existing').selectOption('replace_after_review');
|
||||
await page.locator('#workspace-main section.surface').screenshot({ path: testInfo.outputPath(`production-tagging-selects-${theme}-${width}.png`) });
|
||||
const selects = page.locator('#workspace-main section.surface > .setting-row > select');
|
||||
await expect(selects).toHaveCount(2);
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
const select = selects.nth(index);
|
||||
const values = await select.evaluate(node => [...node.options].map(option => option.value));
|
||||
for (const value of values) {
|
||||
await select.selectOption(value);
|
||||
const measured = await page.evaluate(measureTagging, '#workspace-main');
|
||||
expect(measured.selectsFit, `${await select.getAttribute('id')}=${value}`).toEqual([true, true]);
|
||||
expect(measured.pageScroll, value).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
// D132 below the concept's smallest size: on a 320px phone the engine select still shows its whole option.
|
||||
for (const theme of THEMES) test(`auto-tagging engine select fits at 320 wide ${theme}`, async ({ page }) => {
|
||||
await page.setViewportSize({ width: 320, height: 700 });
|
||||
const production = await productionTagging(page, theme);
|
||||
for (const model of ['hf', 'gguf', 'wd14-swinv2']) {
|
||||
await page.locator('#workspace-main #tag-model').selectOption(model);
|
||||
expect((await page.evaluate(measureTagging, '#workspace-main')).selectsFit[0], model).toBe(true);
|
||||
}
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
|
||||
test('auto-tagging comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
@@ -234,9 +222,6 @@ test('auto-tagging comparison detects mutations', async ({ page, request }) => {
|
||||
const undoSqueeze = await mutate(page, { css: '#workspace-main #tag-model { max-width: 60px !important; min-width: 0 !important; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['selectsFit.0']));
|
||||
await undoSqueeze();
|
||||
const undoSqueezeExisting = await mutate(page, { css: '#workspace-main #tag-existing { max-width: 60px !important; min-width: 0 !important; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['selectsFit.1']));
|
||||
await undoSqueezeExisting();
|
||||
// A row that wraps where the concept's does not.
|
||||
const undoWrap = await mutate(page, { css: '#workspace-main .setting-row:has(> #tag-existing) { flex-wrap: wrap; }' });
|
||||
expect(await paths()).toEqual(['rows.1.flexWrap']);
|
||||
@@ -248,14 +233,14 @@ test('auto-tagging comparison detects mutations', async ({ page, request }) => {
|
||||
expect(diffInventory(expected, await measure())).toEqual([]);
|
||||
});
|
||||
|
||||
// D132 at phone width: only the engine row wraps; Existing tags grows beside its text instead (#398).
|
||||
// D132 at phone width, unmapped: only the engine row wraps; Existing tags keeps the concept's cut select.
|
||||
test('auto-tagging at 390 wraps only the engine row', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
const concept = await conceptTagging(page, 'dark');
|
||||
const production = await productionTagging(page, 'dark');
|
||||
expect(diffInventory(mapTaggingConcept(concept, await savedTemplate(request, page)), production)).toEqual([]);
|
||||
expect(production.rows.map(row => row.flexWrap)).toEqual(['wrap', 'nowrap', 'nowrap', 'nowrap', 'nowrap']);
|
||||
expect(production.selectsFit).toEqual([true, true]);
|
||||
expect(production.selectsFit[1]).toBe(concept.selectsFit[1]);
|
||||
});
|
||||
|
||||
/** Runs in the page: the open review dialog: its title, body and the concept's tokens. */
|
||||
@@ -287,19 +272,11 @@ function measureReview() {
|
||||
}
|
||||
|
||||
const number = value => value.toLocaleString('en-US');
|
||||
/** DEC11 (#379): the Time row the preview's estimate reads as (WD14 on the test server's processor). */
|
||||
function timeRow(plan) {
|
||||
const minutes = Math.max(1, Math.ceil(plan.estimate_seconds / 60));
|
||||
const text = minutes < 90 ? `about ${minutes} min`
|
||||
: `about ${Math.floor(minutes / 60)} h${minutes % 60 ? ` ${minutes % 60} min` : ''}`;
|
||||
return `${text} on a processor; a graphics card can be faster.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* D134 (#379): the server's batch preview in the concept's review: "Review tag output images" with "Tag output images"
|
||||
* as its `h3`, the model, policy, image counts, time estimate and settings in the concept's `dl.meta-grid` (Backup in
|
||||
* the concept's wording), the estimate sentence in the concept's `p.small`, the D55 backup box and Cancel / "Start
|
||||
* tagging".
|
||||
* D134: the server's batch preview in the concept's review: "Review tag output images" with "Tag output images" as its
|
||||
* `h3`, the model, policy, counts and settings in the concept's `dl.meta-grid` (Backup in the concept's wording), the
|
||||
* upper-bound sentence in the concept's `p.small`, the D55 backup box and Cancel / "Start tagging".
|
||||
*/
|
||||
function mapBatchConcept(reference, plan) {
|
||||
const [cancel, start] = reference.actions.buttons;
|
||||
@@ -309,14 +286,13 @@ function mapBatchConcept(reference, plan) {
|
||||
tree: ['h3', 'dl.meta-grid', 'p.small', 'div.notice', 'div.dialog-actions'],
|
||||
heading: { ...reference.heading, text: 'Tag output images' },
|
||||
grid: { ...reference.grid, rows: [['Model', 'WD14 SwinV2 · ~400 MB'], ['Existing tags', 'Merge with existing'],
|
||||
['Images', number(plan.images)], ['Already auto-tagged', number(plan.already_tagged)],
|
||||
['Images to tag', `about ${number(plan.to_tag)}`], ['Time', timeRow(plan)],
|
||||
['Settings', `General 0.35 · Character 0.85 · up to ${number(plan.suggestion_limit)} tags per image`],
|
||||
['Images linked', number(plan.images)], ['Prompts', number(plan.prompts)], ['Already tagged', number(plan.already_tagged)],
|
||||
['Prompts to tag', `up to ${number(plan.to_tag)}`], ['Settings', 'General 0.35 · Character 0.85'],
|
||||
['Backup', 'Create and verify before changes']] },
|
||||
backup: { tree: ['div', 'div.row-actions'], text: [['h3', 'Back up the database first (recommended)'],
|
||||
['p', 'Downloads a copy of the prompts database before this operation changes any records.']], buttons: ['Back up now'] },
|
||||
note: { ...reference.note, text: 'Nothing is written until you start. The number to tag is an estimate: images changed '
|
||||
+ 'since they were tagged are tagged again, and missing files are skipped. Activity reports what was tagged.' },
|
||||
note: { ...reference.note, text: 'Nothing is written until you start. Missing files and files outside the output folders '
|
||||
+ 'are counted, so the prompts to tag are an upper bound; Activity reports what was tagged.' },
|
||||
actions: { ...reference.actions, buttons: [cancel, { ...start, text: 'Start tagging' }] },
|
||||
};
|
||||
}
|
||||
@@ -332,8 +308,8 @@ async function conceptReview(page, theme) {
|
||||
|
||||
async function productionReview(page, theme) {
|
||||
await productionTagging(page, theme);
|
||||
// Merge reviews every listed still image (#379): a batch with work, the full review the concept's shell is compared
|
||||
// with.
|
||||
// Every concept-fixture prompt with an image is already tagged, so under the saved Skip there is nothing to start
|
||||
// (the Cancel-only review). Merge reviews a batch with work: the full review the concept's shell is compared with.
|
||||
await page.locator('#workspace-main #tag-existing').selectOption('merge');
|
||||
const answer = page.waitForResponse(response => response.url().endsWith(PREVIEW_URL));
|
||||
await page.locator('#workspace-main .operation').getByRole('button', { name: 'Review', exact: true }).click();
|
||||
@@ -350,9 +326,8 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
expect(concept.tree).toEqual(['p.demo-note', 'h3', 'dl.meta-grid', 'p.small', 'div.dialog-actions']);
|
||||
await page.screenshot({ path: testInfo.outputPath(`concept-plan-review-${theme}-${width}.png`) });
|
||||
const { plan, production } = await productionReview(page, theme);
|
||||
// Under Merge every listed still image is tagged again: the batch has work to review.
|
||||
// Under Merge every prompt with a linked image is tagged again: the batch has work to review.
|
||||
expect(plan.to_tag).toBeGreaterThan(0);
|
||||
expect(plan.estimate_device).toBe('cpu');
|
||||
await page.screenshot({ path: testInfo.outputPath(`production-tagging-review-${theme}-${width}.png`) });
|
||||
expect(diffInventory(mapBatchConcept(concept, plan), production)).toEqual([]);
|
||||
});
|
||||
@@ -397,17 +372,9 @@ test('tag output images review comparison detects mutations', async ({ page }) =
|
||||
const expected = mapBatchConcept(concept, plan);
|
||||
const paths = async () => diffInventory(expected, await page.evaluate(measureReview)).map(item => item.path);
|
||||
expect(await paths()).toEqual([]);
|
||||
// Every label of the grid is compared (#379's rows included): a changed one is named by its row.
|
||||
for (const [index, [term]] of expected.grid.rows.entries()) {
|
||||
await page.evaluate(([at, label]) => {
|
||||
document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[at].textContent = label;
|
||||
}, [index, `${term}!`]);
|
||||
expect(await paths()).toEqual([`grid.rows.${index}.0`]);
|
||||
await page.evaluate(([at, label]) => {
|
||||
document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[at].textContent = label;
|
||||
}, [index, term]);
|
||||
}
|
||||
expect(await paths()).toEqual([]);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[5].textContent = 'To tag'; });
|
||||
expect(await paths()).toEqual(['grid.rows.5.0']);
|
||||
await page.evaluate(() => { document.querySelectorAll('dialog#modal[open] dl.meta-grid dt')[5].textContent = 'Prompts to tag'; });
|
||||
const undoGrid = await mutate(page, { css: 'dialog#modal dl.meta-grid { column-gap: 3px; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['grid.columnGap']));
|
||||
await undoGrid();
|
||||
|
||||
@@ -42,13 +42,8 @@ test('actual gallery form saves a prompt that the admin page can edit and reload
|
||||
await page.locator(`[data-action="edit"][data-prompt-id="${id}"]`).click();
|
||||
const editor = page.locator(`.prompt-text[data-id="${id}"]`);
|
||||
await editor.fill('Browser edited portrait');
|
||||
// The page shows the edit at once and saves it after: reload only once the save answered (a reload that wins the
|
||||
// race abandons the PUT, as seen on WebKit in CI).
|
||||
const savedEdit = page.waitForResponse(item => item.request().method() === 'PUT'
|
||||
&& item.url().endsWith(`/prompt_manager/prompts/${id}`));
|
||||
await editor.press('Enter');
|
||||
await expect(page.locator('.prompt-text').filter({ hasText: 'Browser edited portrait' })).toBeVisible();
|
||||
expect((await savedEdit).ok()).toBe(true);
|
||||
await page.reload();
|
||||
await expect(page.locator('.prompt-text').filter({ hasText: 'Browser edited portrait' })).toBeVisible();
|
||||
const saved = await request.post('/prompt_manager/save', { data: { text: 'Browser saved landscape' } });
|
||||
@@ -132,8 +127,7 @@ test('the gallery cache clear keeps preferences and other tools caches', async (
|
||||
}))).toEqual({ theme: 'light', other: 'keep', prefixed: 'keep', notice: '3.2.0', caches: ['other-tool-gallery-thumbnails'] });
|
||||
});
|
||||
|
||||
// #379: the reviewed tags are that image's own tags; no prompt changes.
|
||||
test('actual gallery review generates and persists tags for the selected root\'s image', async ({ page, request }) => {
|
||||
test('actual gallery review generates and persists tags for the selected root', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2);
|
||||
await page.locator('#autoTagBtn').click();
|
||||
@@ -141,15 +135,13 @@ test('actual gallery review generates and persists tags for the selected root\'s
|
||||
await page.locator('#startReviewBtn').click();
|
||||
const generation = await generated;
|
||||
expect(generation.request().postDataJSON().root_index).toBe(0);
|
||||
await expect(page.locator('#reviewTagsContainer')).toContainText('sandstone');
|
||||
await expect(page.locator('#reviewTagsContainer')).toContainText('generated');
|
||||
const result = page.waitForResponse(response => response.url().endsWith('/prompt_manager/autotag/apply'));
|
||||
await page.locator('#applyReviewBtn').click();
|
||||
expect((await (await result).json()).success).toBe(true);
|
||||
const data = await (await request.get('/prompt_manager/recent?limit=20')).json();
|
||||
expect(data.results.find(prompt => prompt.text === 'Cat portrait').tags).toEqual(['cats']);
|
||||
expect(data.results.find(prompt => prompt.text === 'Dog landscape').tags).toEqual(['dogs']);
|
||||
const tagged = await (await request.get('/prompt_manager/images/output?mode=workspace&tags=sandstone')).json();
|
||||
expect(tagged.images.map(image => `${image.root_index}:${image.path}`)).toEqual(['0:shared/same.png']);
|
||||
expect(data.results.find(prompt => prompt.text === 'Cat portrait').tags).toContain('generated');
|
||||
expect(data.results.find(prompt => prompt.text === 'Dog landscape').tags).not.toContain('generated');
|
||||
});
|
||||
|
||||
test('actual Tags page reloads the selected filter and opens its image viewer', async ({ page }) => {
|
||||
|
||||
@@ -2189,248 +2189,3 @@ test('scan for metadata opens the review dialog from Import', async ({ page, req
|
||||
await expect.poll(async () => (await (await request.get('/prompt_manager/jobs')).json()).jobs
|
||||
.map(job => [job.kind, job.state])).toEqual([['backup', 'finished']]);
|
||||
});
|
||||
|
||||
// #392 (D135): file-manager selection beyond the concept's checkbox.
|
||||
const selectedCount = page => bulkbar(page).locator('.bulkbar > strong');
|
||||
const cardAt = (page, index) => page.locator(`article[data-image-key][data-index="${index}"]`);
|
||||
const boxAt = (page, index) => cardAt(page, index).getByRole('checkbox', { name: /^Select / });
|
||||
const checkedIndexes = page => page.locator('article[data-image-key]').evaluateAll(nodes => nodes
|
||||
.filter(node => node.querySelector('.image-overlays input[type="checkbox"]').checked)
|
||||
.map(node => Number(node.dataset.index)).sort((a, b) => a - b));
|
||||
const range = (from, to) => Array.from({ length: to - from + 1 }, (_, index) => from + index);
|
||||
async function bulkReset(page, request) {
|
||||
await page.addInitScript(() => { window.IntersectionObserver = undefined; });
|
||||
await expect(await request.post('/__test__/reset', { data: { extra_images: 200 } })).toBeOK();
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(40);
|
||||
}
|
||||
/** A touch press on `locator` the way a finger makes one: pointerdown, (held), pointerup. */
|
||||
const touchDown = locator => locator.evaluate(node => {
|
||||
const { left, top } = node.getBoundingClientRect();
|
||||
node.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true, isPrimary: true, pointerType: 'touch', clientX: left + 5, clientY: top + 5 }));
|
||||
});
|
||||
const touchUpAndClick = locator => locator.evaluate(node => {
|
||||
node.dispatchEvent(new PointerEvent('pointerup', { bubbles: true, isPrimary: true, pointerType: 'touch' }));
|
||||
// A tap's click counts one press (`detail` 1); a keyboard click (and `HTMLElement.click()`) counts none.
|
||||
node.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, detail: 1 }));
|
||||
});
|
||||
async function touchTap(locator) {
|
||||
await touchDown(locator);
|
||||
await touchUpAndClick(locator);
|
||||
}
|
||||
|
||||
test('Shift+click selects a range across an appended batch in Grid, List and Masonry', async ({ page, request }) => {
|
||||
await bulkReset(page, request);
|
||||
await page.getByRole('button', { name: 'Load more images', exact: true }).click();
|
||||
await expect(caption(page)).toContainText('80 of 202 images');
|
||||
for (const mode of ['Grid', 'List', 'Masonry']) {
|
||||
await page.getByRole('group', { name: 'Image layout', exact: true }).getByRole('button', { name: mode, exact: true }).click();
|
||||
// The anchor is the last image toggled on its own (List hides the concept's checkbox: the card carries it).
|
||||
await cardAt(page, 3).locator('.media-info h3').click({ modifiers: ['ControlOrMeta'] });
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
// The range runs into the second batch (index 40 on), and backwards from the same anchor.
|
||||
await cardAt(page, 45).locator('.media-info h3').click({ modifiers: ['Shift'] });
|
||||
await expect(selectedCount(page)).toHaveText('43 selected');
|
||||
expect(await checkedIndexes(page)).toEqual(range(3, 45));
|
||||
await cardAt(page, 0).locator('.media-info h3').click({ modifiers: ['Shift'] });
|
||||
await expect(selectedCount(page)).toHaveText('46 selected');
|
||||
expect(await checkedIndexes(page)).toEqual(range(0, 45));
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
if (mode !== 'List') {
|
||||
// A Shift+click on a checkbox gives its range the box's new state: here, unchecked.
|
||||
await boxAt(page, 3).uncheck();
|
||||
await boxAt(page, 10).click({ modifiers: ['Shift'] });
|
||||
await expect(boxAt(page, 10)).not.toBeChecked();
|
||||
await expect(selectedCount(page)).toHaveText('38 selected');
|
||||
expect(await checkedIndexes(page)).toEqual([0, 1, 2, ...range(11, 45)]);
|
||||
}
|
||||
await bulkbar(page).getByRole('button', { name: 'Clear', exact: true }).click();
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
}
|
||||
});
|
||||
|
||||
test('Ctrl/Cmd+click toggles a card without opening it, and a plain click still opens the viewer', async ({ page }) => {
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
const first = cards(page).first();
|
||||
const inspectorBefore = await details(page).innerText();
|
||||
await first.getByRole('button', { name: 'Open same.png', exact: true }).click({ modifiers: ['ControlOrMeta'] });
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
await expect(checkbox(first, 'same.png')).toBeChecked();
|
||||
await cards(page).last().getByRole('button', { name: 'Inspect same.png', exact: true }).click({ modifiers: ['ControlOrMeta'] });
|
||||
await expect(selectedCount(page)).toHaveText('2 selected');
|
||||
await star(first, 'same.png').click({ modifiers: ['ControlOrMeta'] });
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
await expect(star(first, 'same.png')).toHaveAttribute('aria-pressed', 'false');
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
expect(await details(page).innerText()).toBe(inspectorBefore);
|
||||
// A plain click keeps its meaning: the image opens the viewer.
|
||||
await first.getByRole('button', { name: 'Open same.png', exact: true }).click();
|
||||
await expect(page.locator('#viewer-dialog[open]')).toHaveCount(1);
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
});
|
||||
|
||||
test('Ctrl/Cmd+A, Select all N matching and Add tags tag every matching image on the server', async ({ page, request }) => {
|
||||
await bulkReset(page, request);
|
||||
// Ctrl/Cmd+A in the search box stays the field's own.
|
||||
await search(page).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await expect(selectedCount(page)).toHaveText('40 selected');
|
||||
const offer = bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true });
|
||||
await expect(offer).toBeVisible();
|
||||
await expect(bulkbar(page).locator('.bulkbar > *').nth(1)).toHaveText('Select all 202 matching');
|
||||
await offer.click();
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
await expect(offer).toHaveCount(0);
|
||||
await expect(bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true })).toHaveAttribute('aria-disabled', 'true');
|
||||
// A later batch arrives checked.
|
||||
await page.getByRole('button', { name: 'Load more images', exact: true }).click();
|
||||
await expect(caption(page)).toContainText('80 of 202 images');
|
||||
await expect(boxAt(page, 70)).toBeChecked();
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// Unchecking one image leaves "all matching": the other loaded images stay selected.
|
||||
await boxAt(page, 0).uncheck();
|
||||
await expect(selectedCount(page)).toHaveText('79 selected');
|
||||
await expect(bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true })).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await cardAt(page, 1).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }).click();
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// Add tags sends the listing filter, not 202 identities, and the server tags every match.
|
||||
await bulkbar(page).getByRole('button', { name: 'Add tags', exact: true }).click();
|
||||
const dialog = page.getByRole('dialog', { name: 'Tag selected images', exact: true });
|
||||
await dialog.getByRole('textbox', { name: 'Tags, separated by commas', exact: true }).fill('picked392');
|
||||
const started = page.waitForRequest(item => item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/jobs/image-tags');
|
||||
await dialog.getByRole('button', { name: 'Apply', exact: true }).click();
|
||||
expect((await started).postDataJSON()).toEqual({ mode: 'add', filter: { sort: 'newest' }, tags: ['picked392'] });
|
||||
await expect.poll(async () => (await (await request.get('/prompt_manager/images/output?mode=workspace&limit=1&tags=picked392')).json()).total,
|
||||
{ timeout: 20000 }).toBe(202);
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// Escape in the grid clears the selection.
|
||||
await cardAt(page, 2).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
await expect(boxAt(page, 2)).not.toBeChecked();
|
||||
});
|
||||
|
||||
test('Shift+arrow keys extend the selection from the focused card, and Escape in the bar clears it', async ({ page, request }) => {
|
||||
await bulkReset(page, request);
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('Shift+ArrowRight');
|
||||
await page.keyboard.press('Shift+ArrowRight');
|
||||
await expect(cardAt(page, 2).getByRole('button', { name: /^Open / })).toBeFocused();
|
||||
await expect(selectedCount(page)).toHaveText('3 selected');
|
||||
expect(await checkedIndexes(page)).toEqual([0, 1, 2]);
|
||||
// A plain arrow moves without selecting; Shift+End reaches the last loaded card from the same anchor.
|
||||
await page.keyboard.press('ArrowRight');
|
||||
await expect(selectedCount(page)).toHaveText('3 selected');
|
||||
await page.keyboard.press('Shift+End');
|
||||
await expect(selectedCount(page)).toHaveText('40 selected');
|
||||
await bulkbar(page).getByRole('button', { name: 'Clear', exact: true }).focus();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
await expect(page.locator('#workspace-main h1')).toBeFocused();
|
||||
});
|
||||
|
||||
test('a touch long-press starts selection mode, taps then toggle, and an empty selection ends it', async ({ page }, testInfo) => {
|
||||
await page.goto(workspace);
|
||||
await expect(cards(page)).toHaveCount(2);
|
||||
const [first, second] = [cards(page).first(), cards(page).last()];
|
||||
const open = card => card.getByRole('button', { name: 'Open same.png', exact: true });
|
||||
await touchDown(open(first));
|
||||
// The context menu of a held press belongs to the long press.
|
||||
expect(await open(first).evaluate(node => !node.dispatchEvent(new MouseEvent('contextmenu', { bubbles: true, cancelable: true })))).toBe(true);
|
||||
// Held: the long press selects the card once its delay passed.
|
||||
await expect(selectedCount(page)).toHaveText('1 selected');
|
||||
await touchUpAndClick(open(first));
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
await expect(checkbox(first, 'same.png')).toBeChecked();
|
||||
// Taps now toggle instead of opening.
|
||||
if (testInfo.project.name === 'touch') await open(second).tap();
|
||||
else await touchTap(open(second));
|
||||
await expect(selectedCount(page)).toHaveText('2 selected');
|
||||
// A keyboard Enter keeps its meaning in touch selection mode: the image opens.
|
||||
await open(first).focus();
|
||||
await page.keyboard.press('Enter');
|
||||
await expect(page.locator('#viewer-dialog[open]')).toHaveCount(1);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(page.locator('#viewer-dialog[open]')).toHaveCount(0);
|
||||
await expect(selectedCount(page)).toHaveText('2 selected');
|
||||
await touchTap(open(first));
|
||||
await touchTap(open(second));
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
// The selection is empty, so touch selection mode ended: a tap opens the image again.
|
||||
await touchTap(open(first));
|
||||
await expect(page.locator('#viewer-dialog[open]')).toHaveCount(1);
|
||||
});
|
||||
|
||||
test('all matching sets the category through the listing filter and survives a return visit as the loaded images', async ({ page, request }) => {
|
||||
await bulkReset(page, request);
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }).click();
|
||||
// The offer left with its press: focus moves to the bar's first action.
|
||||
await expect(bulkbar(page).getByRole('button', { name: 'Add tags', exact: true })).toBeFocused();
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// Export metadata waits for loaded images: a press does nothing.
|
||||
const exportButton = bulkbar(page).getByRole('button', { name: 'Export metadata', exact: true });
|
||||
await expect(exportButton).toHaveAttribute('title', 'Export metadata reads each image in the browser, so it works on loaded images only.');
|
||||
// No match linked (a stubbed answer): the nothing-linked sentence, and the selection stays.
|
||||
await page.route('**/prompt_manager/bulk/category', route => route.fulfill({ json: { success: true, updated_count: 0, prompt_count: 0, skipped: 202 } }), { times: 1 });
|
||||
const categorize = page.getByRole('dialog', { name: 'Categorize selected images', exact: true });
|
||||
await bulkbar(page).getByRole('button', { name: 'Set category', exact: true }).click();
|
||||
await categorize.getByRole('textbox', { name: 'Category', exact: true }).fill('Bulk392');
|
||||
await categorize.getByRole('button', { name: 'Apply', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText("None of the selected images is linked to a saved prompt, so there is no prompt category to set.");
|
||||
await expect(selectedCount(page)).toHaveText('All 202 matching selected');
|
||||
// The real write: one request with the filter, every matching image's positive changes.
|
||||
await bulkbar(page).getByRole('button', { name: 'Set category', exact: true }).click();
|
||||
await categorize.getByRole('textbox', { name: 'Category', exact: true }).fill('Bulk392');
|
||||
const posted = page.waitForRequest(item => item.method() === 'POST' && new URL(item.url()).pathname === '/prompt_manager/bulk/category');
|
||||
await categorize.getByRole('button', { name: 'Apply', exact: true }).click();
|
||||
expect((await posted).postDataJSON()).toEqual({ images_filter: { sort: 'newest' }, category: 'Bulk392' });
|
||||
await expect(toast(page)).toHaveText(/^Category set on \d+ prompts?\.$/);
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
const filtered = await (await request.get('/prompt_manager/images/output?mode=workspace&limit=1&category=Bulk392')).json();
|
||||
expect(filtered.total).toBe(202);
|
||||
// A return visit keeps "all matching" as the loaded images, and the offer shows again.
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true }).click();
|
||||
await page.evaluate(() => { window.location.hash = '/overview'; });
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Overview');
|
||||
await page.evaluate(() => { window.location.hash = '/images'; });
|
||||
await expect(selectedCount(page)).toHaveText('40 selected');
|
||||
await expect(bulkbar(page).getByRole('button', { name: 'Select all 202 matching', exact: true })).toBeVisible();
|
||||
// Escape in the bar with nothing selected does nothing; with a selection it clears.
|
||||
await bulkbar(page).getByRole('button', { name: 'Clear', exact: true }).focus();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(bulkbar(page)).toBeEmpty();
|
||||
});
|
||||
|
||||
test('all matching over 10,000 images says so before any dialog or request', async ({ page, request }) => {
|
||||
await page.route(url => url.pathname === '/prompt_manager/images/output' && url.searchParams.get('limit') !== '1', async route => {
|
||||
const response = await route.fetch();
|
||||
const body = await response.json();
|
||||
await route.fulfill({ response, json: { ...body, total: 20002 } });
|
||||
});
|
||||
await bulkReset(page, request);
|
||||
await cardAt(page, 0).getByRole('button', { name: /^Open / }).focus();
|
||||
await page.keyboard.press('ControlOrMeta+a');
|
||||
await bulkbar(page).getByRole('button', { name: 'Select all 20,002 matching', exact: true }).click();
|
||||
await expect(selectedCount(page)).toHaveText('All 20,002 matching selected');
|
||||
let posted = 0;
|
||||
page.on('request', item => { if (item.method() === 'POST') posted += 1; });
|
||||
for (const action of ['Set category', 'Add tags', 'Auto-tag selected']) {
|
||||
await bulkbar(page).getByRole('button', { name: action, exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Narrow the view to at most 10,000 images.');
|
||||
await expect(page.locator('dialog[open]')).toHaveCount(0);
|
||||
}
|
||||
expect(posted).toBe(0);
|
||||
});
|
||||
|
||||
@@ -1,590 +0,0 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
|
||||
// System › Diagnostics & logs (#249 System S12, #329) on the concept-shaped fixture, with the logger writing into this
|
||||
// server's own root (workspace_diagnostics_fixture.py): the meta-grid's live values, Run checks, the report, Inspect and
|
||||
// the test link (D110); the log viewer, statistics, downloads and runtime drafts (D109); and the reviewed rotated-log
|
||||
// cleanup (D100) and Clear all logs, which delete only inside that temporary log folder.
|
||||
const workspace = '/prompt_manager/workspace';
|
||||
const REPORT_URL = '/prompt_manager/system/diagnostics';
|
||||
const LOGS_URL = '/prompt_manager/logs';
|
||||
const CONFIG_URL = '/prompt_manager/logs/config';
|
||||
const PREVIEW_URL = '/prompt_manager/maintenance/preview';
|
||||
const APPLY_URL = '/prompt_manager/maintenance/apply';
|
||||
const TEST_LINK_URL = '/prompt_manager/diagnostics/test-link';
|
||||
const CHANGED = 'The log files changed since the review; nothing was deleted. Review again.';
|
||||
const KEY = 'sk-s12-browser-SECRET-0123456789';
|
||||
const LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'];
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await seedConcept(request);
|
||||
await expect(await request.post('/__test__/system/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/integrations/reset')).toBeOK();
|
||||
await expect(await request.post('/__test__/jobs/restart')).toBeOK();
|
||||
await expect(await request.post('/__test__/logs/reset')).toBeOK();
|
||||
page.systemErrors = [];
|
||||
page.on('pageerror', error => page.systemErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await request.post('/__test__/release');
|
||||
await request.post('/__test__/jobs/restart');
|
||||
await request.post('/__test__/logs/reset');
|
||||
await request.post('/__test__/integrations/reset');
|
||||
await request.post('/__test__/system/reset');
|
||||
expect(page.systemErrors).toEqual([]);
|
||||
});
|
||||
|
||||
const toast = page => page.locator('#workspace-toast');
|
||||
const layoutOf = page => page.locator('#workspace-main .settings-layout');
|
||||
const dialog = page => page.locator('dialog#modal[open]');
|
||||
const surfaceOf = (page, title) => layoutOf(page).locator(':scope > section.surface').filter({
|
||||
has: page.getByRole('heading', { level: 2, name: title, exact: true }) });
|
||||
const cell = (page, term) => surfaceOf(page, 'Diagnostics').locator('.meta-grid > div').filter({
|
||||
has: page.locator(`dt:text-is("${term}")`) }).locator('dd');
|
||||
const lines = page => page.locator('#log-lines > code');
|
||||
const operation = (page, title) => layoutOf(page).locator('.operation').filter({ has: page.locator(`h3:text-is("${title}")`) });
|
||||
const settingOf = (page, label) => layoutOf(page).locator('.setting-row').filter({ has: page.locator(`label:text-is("${label}")`) });
|
||||
const status = page => page.locator('#settings-status');
|
||||
const save = page => page.getByRole('button', { name: 'Save preferences', exact: true });
|
||||
|
||||
async function serverJson(request, url) {
|
||||
const response = await request.get(url);
|
||||
await expect(response).toBeOK();
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function seedLogs(request, data = {}) {
|
||||
await expect(await request.post('/__test__/logs/seed', { data })).toBeOK();
|
||||
}
|
||||
|
||||
const names = async request => (await serverJson(request, '/__test__/logs/state')).entries.map(entry => entry.name);
|
||||
|
||||
async function openDiagnostics(page) {
|
||||
if (page.url().endsWith('#/system/diagnostics')) await page.reload();
|
||||
else await page.goto(`${workspace}#/system/diagnostics`);
|
||||
await expect(layoutOf(page).locator(':scope > section.surface')).toHaveCount(2);
|
||||
await expect(lines(page)).not.toHaveText('Loading…');
|
||||
await expect(cell(page, 'Database')).not.toHaveText('Loading…');
|
||||
// The renderer line settles first: its text can move the rows below.
|
||||
await expect(cell(page, 'Host renderers')).not.toHaveText('Loading…');
|
||||
await expect(page.locator('#log-stats')).not.toHaveText('Loading…');
|
||||
return layoutOf(page);
|
||||
}
|
||||
|
||||
async function openFiles(page) {
|
||||
const details = surfaceOf(page, 'Logs').locator('details');
|
||||
if (!(await details.evaluate(node => node.open))) await details.locator('summary').click();
|
||||
await expect(details.locator(':scope > p.small').first()).not.toHaveText('Loading…');
|
||||
return details;
|
||||
}
|
||||
|
||||
/** The levels of the viewer's lines, in order. */
|
||||
const shownLevels = async page => (await lines(page).textContent()).split('\n').filter(Boolean)
|
||||
.map(line => line.split(/ +/)[1]);
|
||||
|
||||
async function ended(request, id) {
|
||||
await expect.poll(async () => (await serverJson(request, `/prompt_manager/jobs/${id}`)).job.state).not.toBe('running');
|
||||
return (await serverJson(request, `/prompt_manager/jobs/${id}`)).job;
|
||||
}
|
||||
|
||||
/** A second page of this origin that serves a fake ComfyUI window on the send bridge. */
|
||||
async function fakeComfyWindow(context, renderer) {
|
||||
const host = await context.newPage();
|
||||
await host.goto(`${workspace}#/overview`);
|
||||
await host.evaluate(async drawn => {
|
||||
const { createBridgeHost } = await import('/prompt_manager/js/shared/toolkit/prompt-bridge.js');
|
||||
window.fakeHost = createBridgeHost({ namespace: '/', origin: location.origin, label: 'PromptManager — ComfyUI',
|
||||
adapter: { listTargets: () => [{ id: '7', label: 'Prompt Manager', type: 'PromptManager', roles: ['positive'],
|
||||
renderer: drawn }], preview() { throw new Error('unused'); }, apply() { throw new Error('unused'); } } });
|
||||
}, renderer);
|
||||
return host;
|
||||
}
|
||||
|
||||
async function readBlob(download) {
|
||||
const chunks = await (await download.createReadStream()).toArray();
|
||||
return Buffer.concat(chunks).toString('utf8');
|
||||
}
|
||||
|
||||
// ── Task 4: the Diagnostics surface ─────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test('meta-grid shows real database and source status, the pinned Viewer.js version and the host renderers',
|
||||
async ({ page, request, context }) => {
|
||||
const mutations = [];
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) mutations.push(item.url()); });
|
||||
await openDiagnostics(page);
|
||||
const report = await serverJson(request, REPORT_URL);
|
||||
const surface = surfaceOf(page, 'Diagnostics');
|
||||
await expect(surface.locator(':scope > .panel-heading > h2')).toHaveText('Diagnostics');
|
||||
await expect(surface.locator(':scope > .panel-heading > button')).toHaveText('Run checks');
|
||||
await expect(surface.locator('.meta-grid dt')).toHaveText(['Database', 'Sources', 'Viewer.js', 'Host renderers']);
|
||||
const images = report.checks.database.image_count;
|
||||
await expect(cell(page, 'Database')).toHaveText(new RegExp(`^Readable · [\\d.]+ (bytes|KB|MB|GB) · ${images.toLocaleString('en-US')} images?$`));
|
||||
// The size is the database's real size (`database_size_bytes`), never a missing key read as 0.
|
||||
expect(report.database_info.database_size_bytes).toBeGreaterThan(0);
|
||||
await expect(cell(page, 'Database')).not.toContainText('· 0 bytes ·');
|
||||
const { configured, watching } = report.sources;
|
||||
await expect(cell(page, 'Sources')).toHaveText(configured
|
||||
? `${configured} source${configured === 1 ? '' : 's'} · ${watching} watching`
|
||||
: `ComfyUI output folder · ${watching ? 'watching' : 'manual scan'}`);
|
||||
await expect(cell(page, 'Viewer.js')).toHaveText('1.15.2 · pinned vendor');
|
||||
await expect(cell(page, 'Host renderers')).toHaveText('No ComfyUI window is open in this browser, so its renderer is not known.');
|
||||
// A ComfyUI window with a Classic node: the line Integrations shows (D106).
|
||||
const host = await fakeComfyWindow(context, 'classic');
|
||||
await openDiagnostics(page);
|
||||
await expect(cell(page, 'Host renderers')).toHaveText('Detected: Classic (1 ComfyUI window).');
|
||||
await host.close();
|
||||
// A refused read names the running job (D57's naming).
|
||||
await page.route(`**${REPORT_URL}`, route => route.fulfill({ status: 409, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, code: 'busy', error: 'busy', running: { id: 'a'.repeat(32), title: 'Restore' } }) }));
|
||||
await openDiagnostics(page);
|
||||
await expect(cell(page, 'Database')).toHaveText('Busy: Restore is running.');
|
||||
await page.unroute(`**${REPORT_URL}`);
|
||||
await expect(await request.post('/__test__/system/fail', { data: { path: REPORT_URL, times: 1 } })).toBeOK();
|
||||
await openDiagnostics(page);
|
||||
await expect(cell(page, 'Sources')).toHaveText('Could not be read.');
|
||||
expect(mutations).toEqual([]);
|
||||
});
|
||||
|
||||
test('Run checks opens the shared Health report', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
await openDiagnostics(page);
|
||||
const run = surfaceOf(page, 'Diagnostics').getByRole('button', { name: 'Run checks', exact: true });
|
||||
await run.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Health report');
|
||||
await expect(run).toHaveText('Run checks');
|
||||
await dialog(page).getByRole('button', { name: 'Close', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(run).toBeFocused();
|
||||
// The same report Overview reads: Overview's metric names it.
|
||||
expect((await serverJson(request, '/prompt_manager/system/health')).report).toBeTruthy();
|
||||
expect((await serverJson(request, '/__test__/system/digest')).rows).toEqual(before.rows);
|
||||
});
|
||||
|
||||
test('report download goes through the api client and holds no secret or absolute path', async ({ page, request }) => {
|
||||
const { revision } = await serverJson(request, '/prompt_manager/settings');
|
||||
await expect(await request.post('/prompt_manager/lora/enable', { data: { enabled: false, trigger_words_enabled: false, path: '',
|
||||
civitai_api_key: KEY, expected_revision: revision } })).toBeOK();
|
||||
await openDiagnostics(page);
|
||||
const reads = [];
|
||||
page.on('request', item => { if (item.url().endsWith(REPORT_URL)) reads.push(item.url()); });
|
||||
const download = page.waitForEvent('download');
|
||||
await surfaceOf(page, 'Diagnostics').getByRole('button', { name: 'Download diagnostic report', exact: true }).click();
|
||||
const file = await download;
|
||||
expect(file.suggestedFilename()).toMatch(/^prompt-manager-diagnostics-\d{8}-\d{4}\.json$/);
|
||||
await expect(toast(page)).toHaveText('Diagnostic report downloaded.');
|
||||
expect(reads).toHaveLength(1);
|
||||
const text = await readBlob(file);
|
||||
const report = JSON.parse(text);
|
||||
expect(report.host_renderers).toBe('No ComfyUI window is open in this browser, so its renderer is not known.');
|
||||
expect(report.config.integrations.has_civitai_api_key).toBe(true);
|
||||
expect(report.redacted).toBe(0);
|
||||
expect(report.viewerjs).toBe('1.15.2');
|
||||
expect(Object.keys(report.checks)).toEqual(['database', 'dependencies', 'comfyui_output', 'image_monitor']);
|
||||
expect(text).not.toContain(KEY);
|
||||
expect(text).not.toContain('pm-browser-');
|
||||
const strings = [];
|
||||
const walk = value => {
|
||||
if (typeof value === 'string') strings.push(value);
|
||||
else if (value && typeof value === 'object') Object.values(value).forEach(walk);
|
||||
};
|
||||
walk(report);
|
||||
expect(strings.filter(value => /^(\/|[A-Za-z]:[\\/]|\\\\)/.test(value))).toEqual([]);
|
||||
// A refused report names the job and saves nothing.
|
||||
await page.route(`**${REPORT_URL}`, route => route.fulfill({ status: 409, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, code: 'busy', error: 'busy', running: { id: 'a'.repeat(32), title: 'Restore' } }) }));
|
||||
let saved = false;
|
||||
page.on('download', () => { saved = true; });
|
||||
await surfaceOf(page, 'Diagnostics').getByRole('button', { name: 'Download diagnostic report', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Busy: Restore is running. Try again when it ends.');
|
||||
await page.unroute(`**${REPORT_URL}`);
|
||||
await page.route(`**${REPORT_URL}`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'The diagnostic report could not be built.' }) }));
|
||||
await surfaceOf(page, 'Diagnostics').getByRole('button', { name: 'Download diagnostic report', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The diagnostic report could not be downloaded. Nothing was saved to this device.');
|
||||
expect(saved).toBe(false);
|
||||
});
|
||||
|
||||
test('Inspect opens the Inspect a file page and writes nothing', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/__test__/system/digest');
|
||||
const mutations = [];
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) mutations.push(item.url()); });
|
||||
await openDiagnostics(page);
|
||||
const row = operation(page, 'Image-link diagnostics');
|
||||
await expect(row.locator('p')).toHaveText('Check a file-to-prompt association without changing it.');
|
||||
await row.getByRole('button', { name: 'Inspect', exact: true }).click();
|
||||
await expect(page).toHaveURL(`${workspace}#/inspect`);
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Inspect a file');
|
||||
expect(mutations).toEqual([]);
|
||||
expect(await serverJson(request, '/__test__/system/digest')).toEqual(before);
|
||||
});
|
||||
|
||||
test('Create test link confirms first and names the prompt', async ({ page, request }) => {
|
||||
const [prompt] = (await serverJson(request, '/prompt_manager/recent?limit=1')).results;
|
||||
const posts = [];
|
||||
page.on('request', item => { if (item.url().endsWith(TEST_LINK_URL)) posts.push(item.postDataJSON()); });
|
||||
await openDiagnostics(page);
|
||||
const row = operation(page, 'Test image link');
|
||||
await expect(row.locator('p')).toHaveText('Write a placeholder image record to check that linking works.');
|
||||
const create = row.getByRole('button', { name: 'Create test link…', exact: true });
|
||||
await expect(create).not.toHaveAttribute('aria-disabled', 'true');
|
||||
await create.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Create test link');
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText(new RegExp('^Writes a placeholder image record '
|
||||
+ `\\(prompt_manager_test_link\\.png\\) linked to prompt #${prompt.id} “.+”\\. Maintenance › Missing-file links removes it later\\.$`));
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(posts).toEqual([]);
|
||||
await create.click();
|
||||
await dialog(page).getByRole('button', { name: 'Create', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText(/^Test link created \(image record \d+\)\.$/);
|
||||
expect(posts).toEqual([{ prompt_id: prompt.id }]);
|
||||
// A failed write says nothing was changed.
|
||||
await page.route(`**${TEST_LINK_URL}`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'Test link failed' }) }));
|
||||
await create.click();
|
||||
await dialog(page).getByRole('button', { name: 'Create', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The test link could not be created. Nothing was changed.');
|
||||
// With no prompt to link, the button is unavailable and says why.
|
||||
await page.route('**/prompt_manager/recent?limit=1', route => route.fulfill({ status: 200, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: true, results: [] }) }));
|
||||
await openDiagnostics(page);
|
||||
await expect(operation(page, 'Test image link').locator('p'))
|
||||
.toHaveText('Write a placeholder image record to check that linking works. Add a prompt first.');
|
||||
await expect(operation(page, 'Test image link').getByRole('button')).toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
|
||||
// ── Task 5: the log viewer, statistics and runtime settings ─────────────────────────────────────────────────────────
|
||||
|
||||
test('level filter is minimum-level over all six choices', async ({ page, request }) => {
|
||||
await seedLogs(request);
|
||||
const urls = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === LOGS_URL) urls.push(new URL(item.url()).search); });
|
||||
await openDiagnostics(page);
|
||||
const level = page.locator('#log-level');
|
||||
await expect(level.locator('option')).toHaveText(['All levels', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']);
|
||||
await expect(page.locator('label[for="log-level"]')).toHaveText('Level');
|
||||
// All levels: the concept's four lines first, newest first, in its format.
|
||||
await expect(lines(page)).toContainText('09:42:18 INFO Scan completed: 12 enriched, 0 duplicate links\n'
|
||||
+ '09:42:18 WARNING 3 images have ambiguous prompt metadata\n09:39:04 INFO Database backup verified\n'
|
||||
+ '09:38:52 INFO Source monitor started: output');
|
||||
for (const line of (await lines(page).textContent()).split('\n')) expect(line).toMatch(/^\d\d:\d\d:\d\d [A-Z]+ +\S/);
|
||||
expect(urls.at(-1)).toBe('?limit=100');
|
||||
for (const minimum of LEVELS) {
|
||||
await level.selectOption(minimum);
|
||||
await expect.poll(() => urls.at(-1)).toBe(`?limit=100&level=${minimum}`);
|
||||
await expect(lines(page)).toContainText('CRITICAL The database could not be opened');
|
||||
const shown = await shownLevels(page);
|
||||
expect(shown.every(name => LEVELS.indexOf(name) >= LEVELS.indexOf(minimum)), `${minimum}: ${shown}`).toBe(true);
|
||||
if (minimum === 'DEBUG') await expect(lines(page)).toContainText('DEBUG Thumbnail cache lookup');
|
||||
if (minimum === 'WARNING') {
|
||||
await expect(lines(page)).not.toContainText('Database backup verified');
|
||||
await expect(lines(page)).toContainText('ERROR A source folder could not be read');
|
||||
}
|
||||
}
|
||||
await level.selectOption('');
|
||||
await expect.poll(() => urls.at(-1)).toBe('?limit=100');
|
||||
});
|
||||
|
||||
test('Lines accepts 1 to 500 and sends nothing when invalid', async ({ page, request }) => {
|
||||
await seedLogs(request, { filler: 30 });
|
||||
const urls = [];
|
||||
page.on('request', item => { if (new URL(item.url()).pathname === LOGS_URL) urls.push(new URL(item.url()).search); });
|
||||
await openDiagnostics(page);
|
||||
const limit = page.locator('#log-limit');
|
||||
await expect(page.locator('label[for="log-limit"]')).toHaveText('Lines');
|
||||
await expect(limit).toHaveValue('100');
|
||||
expect(await limit.evaluate(node => [node.type, node.min, node.max, node.title]))
|
||||
.toEqual(['number', '1', '500', 'Lines to show: a whole number from 1 to 500.']);
|
||||
await limit.fill('20');
|
||||
await limit.dispatchEvent('change');
|
||||
await expect.poll(() => urls.at(-1)).toBe('?limit=20');
|
||||
await expect.poll(async () => (await lines(page).textContent()).split('\n').length).toBe(20);
|
||||
const sent = urls.length;
|
||||
for (const value of ['0', '501', '2.5', '']) {
|
||||
await limit.fill(value);
|
||||
await limit.dispatchEvent('change');
|
||||
await expect(limit).toHaveAttribute('aria-invalid', 'true');
|
||||
}
|
||||
// A Level change while Lines is invalid sends nothing either.
|
||||
await page.locator('#log-level').selectOption('WARNING');
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
expect(urls.length).toBe(sent);
|
||||
await limit.fill('500');
|
||||
await limit.dispatchEvent('change');
|
||||
await expect(limit).not.toHaveAttribute('aria-invalid');
|
||||
await expect.poll(() => urls.at(-1)).toBe('?limit=500&level=WARNING');
|
||||
});
|
||||
|
||||
test('logs refresh handles empty, error and stale answers', async ({ page, request }) => {
|
||||
await seedLogs(request);
|
||||
await openDiagnostics(page);
|
||||
const level = page.locator('#log-level');
|
||||
const refresh = surfaceOf(page, 'Logs').getByRole('button', { name: 'Refresh', exact: true });
|
||||
// Empty: nothing at this level once the buffer is cleared.
|
||||
await expect(await request.post('/__test__/logs/reset')).toBeOK();
|
||||
await level.selectOption('CRITICAL');
|
||||
await expect(lines(page)).toHaveText('No log entries at this level.');
|
||||
// Error.
|
||||
await page.route(`**${LOGS_URL}?*`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'boom', logs: [] }) }));
|
||||
await refresh.click();
|
||||
await expect(lines(page)).toHaveText('The log could not be read.');
|
||||
await page.unroute(`**${LOGS_URL}?*`);
|
||||
// Stale: an older answer that arrives after a newer request never paints.
|
||||
await seedLogs(request);
|
||||
await expect(await request.post('/__test__/hold', { data: { path: LOGS_URL } })).toBeOK();
|
||||
await level.selectOption('DEBUG');
|
||||
await expect.poll(async () => (await serverJson(request, '/__test__/hold')).held[LOGS_URL] ?? 0).toBe(1);
|
||||
await level.selectOption('ERROR');
|
||||
await expect.poll(async () => (await serverJson(request, '/__test__/hold')).held[LOGS_URL] ?? 0).toBe(2);
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await expect(lines(page)).toContainText('CRITICAL The database could not be opened');
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
expect((await shownLevels(page)).every(name => ['ERROR', 'CRITICAL'].includes(name))).toBe(true);
|
||||
});
|
||||
|
||||
test('log statistics show entries, errors, warnings, files and size', async ({ page, request }) => {
|
||||
await seedLogs(request, { rotated: [2048, 1024], other: true, link: true, active: 512 });
|
||||
await openDiagnostics(page);
|
||||
const stats = page.getByRole('status', { name: 'Log statistics', exact: true });
|
||||
await expect(stats).toHaveAttribute('id', 'log-stats');
|
||||
const text = await stats.textContent();
|
||||
const match = /^([\d,]+) entr(?:y|ies) in memory · (\d+) errors? · (\d+) warnings? · (\d+) files? · (.+) on disk$/.exec(text);
|
||||
expect(match, text).not.toBeNull();
|
||||
const [, entries, errors, warnings, files, size] = match;
|
||||
expect(Number(entries.replace(/,/g, ''))).toBeGreaterThanOrEqual(7);
|
||||
expect(Number(errors)).toBeGreaterThanOrEqual(2);
|
||||
expect(Number(warnings)).toBeGreaterThanOrEqual(1);
|
||||
// The active log, two rotated copies and the .bak; the link is not counted (Rule R).
|
||||
expect(Number(files)).toBe(4);
|
||||
expect(size).toMatch(/^[\d.]+ (bytes|KB|MB)$/);
|
||||
});
|
||||
|
||||
test('Download log saves the active log', async ({ page, request }) => {
|
||||
await seedLogs(request, { active: 300 });
|
||||
await openDiagnostics(page);
|
||||
const download = page.waitForEvent('download');
|
||||
await surfaceOf(page, 'Logs').getByRole('button', { name: 'Download log', exact: true }).click();
|
||||
const file = await download;
|
||||
expect(file.suggestedFilename()).toBe('prompt_manager.log');
|
||||
expect((await readBlob(file)).length).toBeGreaterThanOrEqual(300);
|
||||
await page.route('**/prompt_manager/logs/download/**', route => route.fulfill({ status: 404, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'Log file not found' }) }));
|
||||
await surfaceOf(page, 'Logs').getByRole('button', { name: 'Download log', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('The log could not be downloaded.');
|
||||
});
|
||||
|
||||
test('runtime level (including CRITICAL) and console logging save and say they apply until the server restarts',
|
||||
async ({ page, request }) => {
|
||||
const posts = [];
|
||||
page.on('request', item => { if (item.method() === 'POST' && item.url().endsWith(CONFIG_URL)) posts.push(item.postDataJSON()); });
|
||||
await openDiagnostics(page);
|
||||
const level = settingOf(page, 'Runtime log level');
|
||||
await expect(level.locator('p')).toHaveText('How much the server writes to the log. Applied until the server restarts.');
|
||||
await expect(level.locator('option')).toHaveText(['INFO', 'WARNING', 'DEBUG', 'ERROR', 'CRITICAL']);
|
||||
await expect(level.locator('select')).toHaveValue('INFO');
|
||||
const consoleRow = settingOf(page, 'Console logging');
|
||||
await expect(consoleRow.locator('p')).toHaveText('Print Prompt Manager’s warnings and errors in the ComfyUI console '
|
||||
+ '(everything at DEBUG). Applied until the server restarts.');
|
||||
const toggle = consoleRow.locator('input[type="checkbox"]');
|
||||
await expect(toggle).toBeChecked();
|
||||
await level.locator('select').selectOption('CRITICAL');
|
||||
await toggle.uncheck();
|
||||
await expect(status(page)).toHaveText('Unsaved changes.');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Applied until the server restarts.');
|
||||
expect(posts).toEqual([{ level: 'CRITICAL', console_logging: false }]);
|
||||
const { config } = await serverJson(request, CONFIG_URL);
|
||||
expect([config.level, config.console_logging]).toEqual(['CRITICAL', false]);
|
||||
// Read again on a fresh page: the server's values.
|
||||
await openDiagnostics(page);
|
||||
await expect(settingOf(page, 'Runtime log level').locator('select')).toHaveValue('CRITICAL');
|
||||
await expect(settingOf(page, 'Console logging').locator('input[type="checkbox"]')).not.toBeChecked();
|
||||
// Only the changed key is sent.
|
||||
await settingOf(page, 'Runtime log level').locator('select').selectOption('DEBUG');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Applied until the server restarts.');
|
||||
expect(posts.at(-1)).toEqual({ level: 'DEBUG' });
|
||||
});
|
||||
|
||||
test('a refused runtime value is marked invalid', async ({ page }) => {
|
||||
await page.route(`**${CONFIG_URL}`, route => (route.request().method() === 'POST'
|
||||
? route.fulfill({ status: 400, contentType: 'application/json', body: JSON.stringify({ success: false, error: 'Invalid log level' }) })
|
||||
: route.continue()));
|
||||
await openDiagnostics(page);
|
||||
const select = settingOf(page, 'Runtime log level').locator('select');
|
||||
await select.selectOption('WARNING');
|
||||
await save(page).click();
|
||||
await expect(status(page)).toHaveText('Some changes were not saved. Review the highlighted settings.');
|
||||
await expect(select).toHaveAttribute('aria-invalid', 'true');
|
||||
await expect(settingOf(page, 'Console logging').locator('input[type="checkbox"]')).not.toHaveAttribute('aria-invalid');
|
||||
// The draft stays: an edit clears the mark.
|
||||
await expect(select).toHaveValue('WARNING');
|
||||
await select.selectOption('ERROR');
|
||||
await expect(select).not.toHaveAttribute('aria-invalid');
|
||||
});
|
||||
|
||||
test('runtime values that cannot be read stay disabled and say so', async ({ page }) => {
|
||||
await page.route(`**${CONFIG_URL}`, route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'boom' }) }));
|
||||
await openDiagnostics(page);
|
||||
for (const label of ['Runtime log level', 'Console logging']) {
|
||||
const row = settingOf(page, label);
|
||||
await expect(row.locator('p.small')).toHaveText('The current value could not be read.');
|
||||
await expect(row.locator('select, input[type="checkbox"]')).toBeDisabled();
|
||||
}
|
||||
});
|
||||
|
||||
// ── Task 6: log files, the reviewed cleanup and Clear all logs ─────────────────────────────────────────────────────
|
||||
|
||||
const LOG_FOLDER = { rotated: [3000, 2048, 1024], other: true, link: true, active: 100 };
|
||||
const LINKS = process.platform === 'win32' ? 0 : 1;
|
||||
|
||||
test('Review log cleanup lists only rotated copies and keeps the rest', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
const before = await names(request);
|
||||
const previews = [];
|
||||
page.on('request', item => { if (item.url().endsWith(PREVIEW_URL)) previews.push(item.postDataJSON()); });
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/^prompt_manager\.log · [\d.]+ (bytes|KB) · rotated copies: 3$/);
|
||||
await details.getByRole('button', { name: 'Review log cleanup', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review maintenance plan');
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Rotated log copies');
|
||||
expect(previews).toEqual([{ operation: 'rotated_logs', options: {} }]);
|
||||
const grid = await dialog(page).locator('dl.meta-grid > div').evaluateAll(items => items.map(item =>
|
||||
[item.querySelector('dt').textContent, item.querySelector('dd').textContent]));
|
||||
expect(grid.map(([term]) => term)).toEqual(['Scope', 'Protected', 'Backup', 'Media on disk']);
|
||||
expect(grid[0][1]).toMatch(/^3 rotated log copies · 5\.9 KB$/);
|
||||
expect(grid[1][1]).toBe(['Kept: the active log', ...(LINKS ? ['Kept: 1 link'] : []), 'Kept: 1 other file'].join(' · '));
|
||||
expect(grid[2][1]).toBe('Not needed: no library record changes');
|
||||
expect(grid[3][1]).toBe('Deletes only the listed log files. No image, video or library record changes.');
|
||||
await expect(dialog(page).locator('details > summary')).toHaveText('Listed files (3)');
|
||||
await expect(dialog(page).locator('details pre')).toHaveText(/^prompt_manager\.log\.1 · 2\.9 KB · .+\nprompt_manager\.log\.2 · 2\.0 KB · .+\nprompt_manager\.log\.3 · 1\.0 KB · .+$/);
|
||||
// No D55 backup box: nothing in the library changes.
|
||||
await expect(dialog(page).locator('.notice')).toHaveCount(0);
|
||||
await expect(dialog(page).getByRole('button', { name: 'Back up now' })).toHaveCount(0);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(await names(request)).toEqual(before);
|
||||
});
|
||||
|
||||
test('Apply deletes exactly the listed files and Activity lists them', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await details.getByRole('button', { name: 'Review log cleanup', exact: true }).click();
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
const job = await ended(request, page.url().split('/').pop());
|
||||
expect([job.kind, job.state, job.title, job.counters.changed]).toEqual(['maintenance', 'finished', 'Rotated log copies', 3]);
|
||||
await expect(toast(page)).toHaveText(/Rotated log copies (started|finished · 3 changed)/);
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(main.locator('h1')).toHaveText('Rotated log copies');
|
||||
for (const name of ['prompt_manager.log.1', 'prompt_manager.log.2', 'prompt_manager.log.3']) await expect(main).toContainText(name);
|
||||
// Only those three are gone: the active log, the user's .bak and the link are kept.
|
||||
expect(await names(request)).toEqual(['prompt_manager.log', ...(LINKS ? ['prompt_manager.log.4'] : []), 'prompt_manager.log.bak']);
|
||||
});
|
||||
|
||||
test('a rollover after the review deletes nothing and Activity offers Review again', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await details.getByRole('button', { name: 'Review log cleanup', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Rotated log copies');
|
||||
await expect(await request.post('/__test__/logs/rotate')).toBeOK();
|
||||
const rolled = await serverJson(request, '/__test__/logs/state');
|
||||
await dialog(page).getByRole('button', { name: 'Apply reviewed operation', exact: true }).click();
|
||||
await expect(page).toHaveURL(/#\/activity\/[0-9a-f]{32}$/);
|
||||
const job = await ended(request, page.url().split('/').pop());
|
||||
expect([job.state, job.scope.failure]).toEqual(['failed', 'plan-changed']);
|
||||
await expect(page.locator('#workspace-main')).toContainText(CHANGED);
|
||||
expect((await serverJson(request, '/__test__/logs/state')).entries.map(entry => entry.name))
|
||||
.toEqual(rolled.entries.map(entry => entry.name));
|
||||
const previews = [];
|
||||
page.on('request', item => { if (item.url().endsWith(PREVIEW_URL)) previews.push(item.postDataJSON()); });
|
||||
await page.locator('#workspace-main').getByRole('button', { name: 'Review again', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Rotated log copies');
|
||||
expect(previews).toEqual([{ operation: 'rotated_logs', options: {} }]);
|
||||
});
|
||||
|
||||
test('Clear all logs states both counts, confirms first and reads everything again', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
const truncates = [];
|
||||
page.on('request', item => { if (item.url().endsWith('/prompt_manager/logs/truncate')) truncates.push(item.url()); });
|
||||
await openDiagnostics(page);
|
||||
await expect(lines(page)).toContainText('Database backup verified');
|
||||
const details = await openFiles(page);
|
||||
const clearRow = operation(page, 'Clear all logs');
|
||||
await expect(clearRow.locator('p')).toHaveText('Empty the active log and delete its rotated copies.');
|
||||
const clear = clearRow.getByRole('button', { name: 'Clear all logs…', exact: true });
|
||||
await expect(clear).toHaveClass(/\bdanger\b/);
|
||||
await clear.click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Clear all logs');
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText('Empties prompt_manager.log, deletes 3 rotated copies (5.9 KB) '
|
||||
+ 'and clears the lines shown here. The other files in the log folder are kept. This cannot be undone.');
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
expect(truncates).toEqual([]);
|
||||
await clear.click();
|
||||
await dialog(page).getByRole('button', { name: 'Clear', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Logs cleared · 3 rotated copies deleted.');
|
||||
expect(truncates).toHaveLength(1);
|
||||
// Read again: the viewer, the file line and the list.
|
||||
await expect(lines(page)).not.toContainText('Database backup verified');
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/rotated copies: 0$/);
|
||||
const kept = ['prompt_manager.log', 'prompt_manager.log.bak', ...(LINKS ? ['prompt_manager.log.4'] : [])];
|
||||
// The server lists newest first: compare as a set.
|
||||
await expect.poll(async () => (await details.locator('.operation-list h3').allTextContents()).sort()).toEqual([...kept].sort());
|
||||
expect(await names(request)).toEqual(['prompt_manager.log', ...(LINKS ? ['prompt_manager.log.4'] : []), 'prompt_manager.log.bak']);
|
||||
});
|
||||
|
||||
test('Clear all logs reads the files again before it states what it deletes', async ({ page, request }) => {
|
||||
await seedLogs(request, { rotated: [1024], active: 100 });
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/rotated copies: 1$/);
|
||||
// The log rolls over while the page stays open: Clear names what it deletes now, not the count shown.
|
||||
await seedLogs(request, { records: false, rotated: [1024, 1024, 1024] });
|
||||
await operation(page, 'Clear all logs').getByRole('button', { name: 'Clear all logs…', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText(/^Empties prompt_manager\.log, deletes 3 rotated copies \(3\.0 KB\) /);
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/rotated copies: 3$/);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
// A list that cannot be read gives the general sentence, never an old count.
|
||||
await page.route('**/prompt_manager/logs/files', route => route.fulfill({ status: 500, contentType: 'application/json',
|
||||
body: JSON.stringify({ success: false, error: 'boom', files: [] }) }));
|
||||
await operation(page, 'Clear all logs').getByRole('button', { name: 'Clear all logs…', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText(/^Empties prompt_manager\.log, deletes its rotated copies and clears/);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
expect(await names(request)).toEqual(['prompt_manager.log', 'prompt_manager.log.1', 'prompt_manager.log.2', 'prompt_manager.log.3']);
|
||||
});
|
||||
|
||||
test('each listed file downloads by its returned name and a link has no Download', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
await openDiagnostics(page);
|
||||
const details = await openFiles(page);
|
||||
await expect(details.locator(':scope > p.small').nth(1))
|
||||
.toHaveText('Rotates at 10 MB · keeps 5 rotated copies · file logging on · memory buffer 1,000 entries');
|
||||
const list = details.locator('.operation-list');
|
||||
expect((await list.locator('h3').allTextContents()).sort()).toEqual(['prompt_manager.log', 'prompt_manager.log.1',
|
||||
'prompt_manager.log.2', 'prompt_manager.log.3', ...(LINKS ? ['prompt_manager.log.4'] : []), 'prompt_manager.log.bak'].sort());
|
||||
const item = name => list.locator('.operation').filter({ has: page.locator(`h3:text-is("${name}")`) });
|
||||
await expect(item('prompt_manager.log.1').locator('p')).toHaveText(/^2\.9 KB · \d{4}-\d\d-\d\d \d\d:\d\d · Rotated copy$/);
|
||||
await expect(item('prompt_manager.log.bak').locator('p')).toHaveText(/^0 bytes · .+ · Other file$/);
|
||||
if (LINKS) {
|
||||
await expect(item('prompt_manager.log.4').locator('p')).toHaveText(/ · Link, not downloaded$/);
|
||||
await expect(item('prompt_manager.log.4').getByRole('button')).toHaveCount(0);
|
||||
}
|
||||
for (const name of ['prompt_manager.log.1', 'prompt_manager.log.bak']) {
|
||||
const download = page.waitForEvent('download');
|
||||
await item(name).getByRole('button', { name: `Download ${name}`, exact: true }).click();
|
||||
const file = await download;
|
||||
expect(file.suggestedFilename()).toBe(name);
|
||||
if (name === 'prompt_manager.log.1') expect((await readBlob(file)).length).toBe(3000);
|
||||
}
|
||||
});
|
||||
@@ -315,28 +315,22 @@ test('Preview tagging review without a selection toasts the D115 sentence', asyn
|
||||
await expect(tagDialog(page)).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('Tag output images reviews the images, tags each one and never a prompt (#379)', async ({ page, request }) => {
|
||||
test('Tag output images reviews the scope and starts a batch job', async ({ page, request }) => {
|
||||
const before = await serverJson(request, '/__test__/tagging/snapshot');
|
||||
await openTagging(page);
|
||||
// The saved policy is Skip: no image is auto-tagged yet, so every listed still image is to tag.
|
||||
// The fixture's one prompt is already tagged: under Merge it is tagged again with what its images give.
|
||||
await page.locator('#tag-existing').selectOption('merge');
|
||||
const previewed = page.waitForResponse(item => new URL(item.url()).pathname === PREVIEW_URL);
|
||||
await reviewButton(page).click();
|
||||
const answer = await previewed;
|
||||
expect(answer.request().postDataJSON()).toEqual({ model_type: 'wd14-swinv2', existing: 'skip' });
|
||||
expect(answer.request().postDataJSON()).toEqual({ model_type: 'wd14-swinv2', existing: 'merge' });
|
||||
const plan = await answer.json();
|
||||
// Four still images (two linked, a copy, an unlinked one); the video is not a target.
|
||||
expect([plan.images, plan.already_tagged, plan.to_tag]).toEqual([4, 0, 4]);
|
||||
expect(plan.prompts).toBeUndefined();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review tag output images');
|
||||
await expect(dialog(page).locator('#modal-body > h3')).toHaveText('Tag output images');
|
||||
expect(await gridOf(page)).toEqual([['Model', 'WD14 SwinV2 · ~400 MB'], ['Existing tags', 'Skip already tagged'],
|
||||
['Images', '4'], ['Already auto-tagged', '0'], ['Images to tag', 'about 4'],
|
||||
['Time', 'about 1 min on a processor; a graphics card can be faster.'],
|
||||
['Settings', `General 0.35 · Character 0.85 · up to ${plan.suggestion_limit} tags per image`],
|
||||
expect(await gridOf(page)).toEqual([['Model', 'WD14 SwinV2 · ~400 MB'], ['Existing tags', 'Merge with existing'],
|
||||
['Images linked', String(plan.images)], ['Prompts', String(plan.prompts)], ['Already tagged', String(plan.already_tagged)],
|
||||
['Prompts to tag', `up to ${plan.to_tag}`], ['Settings', 'General 0.35 · Character 0.85'],
|
||||
['Backup', 'Create and verify before changes']]);
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Nothing is written until you start. The number to '
|
||||
+ 'tag is an estimate: images changed since they were tagged are tagged again, and missing files are skipped. '
|
||||
+ 'Activity reports what was tagged.');
|
||||
await expect(dialog(page).locator('#modal-body > .notice h3')).toHaveText('Back up the database first (recommended)');
|
||||
// The review wrote nothing.
|
||||
expect(await serverJson(request, '/__test__/tagging/snapshot')).toEqual(before);
|
||||
@@ -344,23 +338,15 @@ test('Tag output images reviews the images, tags each one and never a prompt (#3
|
||||
const started = page.waitForResponse(item => new URL(item.url()).pathname === '/prompt_manager/jobs/autotag');
|
||||
await dialog(page).getByRole('button', { name: 'Start tagging', exact: true }).click();
|
||||
const start = await started;
|
||||
expect(start.request().postDataJSON()).toEqual({ model_type: 'wd14-swinv2', existing: 'skip',
|
||||
expect(start.request().postDataJSON()).toEqual({ model_type: 'wd14-swinv2', existing: 'merge',
|
||||
options: { general_threshold: 0.35, character_threshold: 0.85 } });
|
||||
const { job } = await start.json();
|
||||
await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`);
|
||||
await expect(page.locator('#workspace-main h1')).toHaveText('Tag output images');
|
||||
const finished = await ended(request, job.id);
|
||||
expect(finished.state).toBe('finished');
|
||||
expect(finished.status_line).toBe('Tagged 4 images');
|
||||
// Each image got its own tags; no prompt, pair or link row changed.
|
||||
expect(await serverJson(request, '/__test__/tagging/snapshot')).toEqual(before);
|
||||
|
||||
// Under Skip every image is now auto-tagged: nothing to start.
|
||||
await openTagging(page);
|
||||
await reviewButton(page).click();
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Every output image is already auto-tagged.');
|
||||
expect((await gridOf(page)).slice(2, 5)).toEqual([['Images', '4'], ['Already auto-tagged', '4'], ['Images to tag', 'about 0']]);
|
||||
await expect(dialog(page).locator('.dialog-actions button')).toHaveText(['Cancel']);
|
||||
const after = await serverJson(request, '/__test__/tagging/snapshot');
|
||||
expect(after.prompt_tags.length).toBeGreaterThan(before.prompt_tags.length);
|
||||
});
|
||||
|
||||
test('Replace after review shows only Cancel with the reason', async ({ page, request }) => {
|
||||
@@ -369,13 +355,18 @@ test('Replace after review shows only Cancel with the reason', async ({ page, re
|
||||
await page.locator('#tag-existing').selectOption('replace_after_review');
|
||||
await reviewButton(page).click();
|
||||
await expect(dialog(page).locator('#modal-title')).toHaveText('Review tag output images');
|
||||
expect((await gridOf(page)).map(([term]) => term)).toEqual(['Model', 'Existing tags', 'Images', 'Already auto-tagged',
|
||||
'Images to tag', 'Time', 'Settings', 'Backup']);
|
||||
expect((await gridOf(page)).map(([term]) => term)).toEqual(['Model', 'Existing tags', 'Images linked', 'Prompts', 'Already tagged',
|
||||
'Prompts to tag', 'Settings', 'Backup']);
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Replacing tags needs a review of each image. '
|
||||
+ 'Use Preview tagging review, or choose Skip or Merge for a batch run.');
|
||||
await expect(dialog(page).locator('.dialog-actions button')).toHaveText(['Cancel']);
|
||||
await dialog(page).getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog(page)).toHaveCount(0);
|
||||
// Under Skip the fixture's only prompt is already tagged: nothing to start either.
|
||||
await page.locator('#tag-existing').selectOption('skip');
|
||||
await reviewButton(page).click();
|
||||
await expect(dialog(page).locator('#modal-body > p.small')).toHaveText('Every prompt with an output image is already tagged.');
|
||||
await expect(dialog(page).locator('.dialog-actions button')).toHaveText(['Cancel']);
|
||||
expect(await serverJson(request, '/__test__/tagging/snapshot')).toEqual(before);
|
||||
expect((await serverJson(request, '/prompt_manager/jobs')).jobs.filter(item => item.kind === 'autotag')).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
import { seedConcept } from './concept-inventory.mjs';
|
||||
import { readCoverageTable, parseControl, CONTROL_EXEMPTIONS } from '../../tools/vendor/legacy-coverage.mjs';
|
||||
|
||||
// System › Overview (#249 System S2 Task 6) on the concept-shaped real fixture: the metrics, rows, Statistics dialog and
|
||||
// Health report read the server, never sample values (D93, D95, D96). The hooks are workspace_system_fixture.py's.
|
||||
@@ -413,16 +412,15 @@ test('out-of-order overview responses never repaint a newer section', async ({ p
|
||||
await nav.getByRole('button', { name: 'Appearance', exact: true }).click();
|
||||
await nav.getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect.poll(held).toBe(2);
|
||||
// Diagnostics & logs is native (S12): its body is its two surfaces.
|
||||
// Diagnostics still hands off (S12), so its body is one known panel.
|
||||
await nav.getByRole('button', { name: 'Diagnostics & logs', exact: true }).click();
|
||||
await expect(layout.locator(':scope > section.surface')).toHaveCount(2);
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
await expect.poll(held).toBe(0);
|
||||
// Both late answers reached a section that had left: Diagnostics keeps its own body.
|
||||
await expect(layout.locator(':scope > section.handoff')).toBeVisible();
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
await expect(layout.locator('.summary-metrics')).toHaveCount(0);
|
||||
expect(await layout.evaluate(node => [...node.children].map(child => `${child.localName}.${child.className}`)))
|
||||
.toEqual(['section.surface', 'section.surface', 'div.save-row']);
|
||||
expect(await layout.evaluate(node => [...node.children].map(child => child.className))).toEqual(['surface handoff', 'save-row']);
|
||||
// Back on Overview, one fresh read paints one set of metrics.
|
||||
await nav.getByRole('button', { name: 'Overview', exact: true }).click();
|
||||
await expect(layout.locator('.summary-metrics')).toHaveCount(1);
|
||||
@@ -795,45 +793,3 @@ test('reset restores defaults without touching other keys', async ({ page }) =>
|
||||
expect(stored).toEqual({ gridColumns: 5, defaultLimit: 200, thumbnailPromptDismissed: 'kept', workspaceLayout: 'grid',
|
||||
videoMute: true, showImageInfo: true });
|
||||
});
|
||||
|
||||
// TESTING.md "Legacy System capability coverage" (#249 System S12, #329): each `covered` row's native control is on its
|
||||
// System section, found by role and accessible name (a `term` by its text, a disclosure by its summary). Collapsed
|
||||
// `details` are opened first, so a control inside one counts once it is shown. tests/vendor/legacy-coverage.test.mjs
|
||||
// checks the rest of each row.
|
||||
test('every covered legacy System capability has its native control', async ({ page, request }) => {
|
||||
// Download is offered for a model that is not on disk ("Download again" otherwise): the selected default's file is
|
||||
// missing here.
|
||||
await expect(await request.post('/__test__/tagging/models', { data: { unavailable: ['wd14-swinv2'] } })).toBeOK();
|
||||
try {
|
||||
await findControls(page);
|
||||
} finally {
|
||||
await request.post('/__test__/tagging/reset', { data: {} });
|
||||
}
|
||||
});
|
||||
|
||||
async function findControls(page) {
|
||||
const covered = readCoverageTable().filter(row => row.Status === 'covered')
|
||||
.map(row => ({ capability: row.Capability, ...parseControl(row['Native control']) }));
|
||||
// A covered row without a control is one of the recorded exemptions, never silently skipped.
|
||||
expect(covered.filter(row => !row.section).map(row => row.capability)).toEqual(Object.keys(CONTROL_EXEMPTIONS));
|
||||
const controls = covered.filter(row => row.section);
|
||||
expect(controls.length).toBeGreaterThan(40);
|
||||
const sections = [...new Set(controls.map(row => row.section))];
|
||||
for (const section of sections) {
|
||||
await page.goto(`${workspace}#/system/${section}`);
|
||||
const main = page.locator('#workspace-main');
|
||||
await expect(layoutOf(page).locator(':scope > *').first()).toBeVisible();
|
||||
for (const details of await main.locator('details:not([open]) > summary').all()) await details.click();
|
||||
for (const { capability, role, name, row } of controls.filter(item => item.section === section)) {
|
||||
// A name several rows share is looked up inside the row the table names.
|
||||
const within = row ? main.locator('.operation').filter({ has: page.locator(`h3:text-is("${row}")`) }) : main;
|
||||
const exact = new RegExp(`^${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`);
|
||||
// A `dt` (term) and a `details` disclosure (group) have no name from content: the term's text, the summary's.
|
||||
const control = role === 'term' ? within.getByRole('term').filter({ hasText: exact })
|
||||
: role === 'group' ? within.locator('details').filter({ has: page.locator(':scope > summary', { hasText: exact }) })
|
||||
.or(within.getByRole('group', { name, exact: true }))
|
||||
: within.getByRole(role, { name, exact: true });
|
||||
await expect(control.first(), `${capability}: ${section} · ${role} · "${name}"${row ? ` in "${row}"` : ''}`).toBeVisible();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -513,41 +513,6 @@ test('search finds an image by its image tag', async ({ page, request }) => {
|
||||
await expect(card(page, UNLINKED)).toBeVisible();
|
||||
});
|
||||
|
||||
// #379 CUJ 8: the Tags filter finds images by their own image tags through the tag index; no prompt carries them, and
|
||||
// selecting a match shows its prompt. A linked prompt's tag still combines with the image tags.
|
||||
test('the Tags filter finds images by image tags only, and selecting one shows its prompt (#379)', async ({ page, request }) => {
|
||||
await saveImageTags(request, FIRST, ['beach', '1girl', 'waves']);
|
||||
await saveImageTags(request, SECOND, ['beach', 'waves']);
|
||||
await saveImageTags(request, UNLINKED, ['beach', '1girl', 'waves', 'gulls']);
|
||||
await page.goto(images);
|
||||
await expect(page.locator('.gallery-body article[data-image-key]')).toHaveCount(manifest.files.length);
|
||||
const filters = page.locator('#workspace-main search.toolbar').getByRole('button', { name: /^Filters(?: · \d+)?$/ });
|
||||
const refine = page.getByRole('dialog', { name: 'Refine this view', exact: true });
|
||||
const applyTags = async value => {
|
||||
await filters.click();
|
||||
await refine.getByLabel('Tags', { exact: true }).fill(value);
|
||||
const listing = page.waitForRequest(item => item.url().includes('/images/output?')
|
||||
&& new URL(item.url()).searchParams.get('tags') !== null);
|
||||
await refine.getByRole('button', { name: 'Apply filters', exact: true }).click();
|
||||
await listing;
|
||||
};
|
||||
await applyTags('beach, 1girl, waves');
|
||||
// Every listed image whose image tags hold all three; SECOND lacks 1girl.
|
||||
await expect(page.locator('.gallery-body article[data-image-key]')).toHaveCount(2);
|
||||
await expect(card(page, FIRST)).toBeVisible();
|
||||
await expect(card(page, UNLINKED)).toBeVisible();
|
||||
await expect(card(page, SECOND)).toHaveCount(0);
|
||||
// The newest match is selected; the inspector shows its linked prompt.
|
||||
await infoButton(page, FIRST).click();
|
||||
await expect(card(page, FIRST)).toHaveClass(/\bselected\b/);
|
||||
await expect(panel(page).locator('.panel-heading h2')).toHaveText(manifest.title);
|
||||
// The linked positive's own tag combines: SECOND has beach and waves itself and minimal through its prompt.
|
||||
await applyTags(`beach, waves, ${manifest.prompt_tags[0]}`);
|
||||
await expect(page.locator('.gallery-body article[data-image-key]')).toHaveCount(2);
|
||||
await expect(card(page, FIRST)).toBeVisible();
|
||||
await expect(card(page, SECOND)).toBeVisible();
|
||||
});
|
||||
|
||||
// Task C4 rulings: a save re-reads the selected file's details without a loading flash, and under a Tags filter or a
|
||||
// search the loaded listing is re-read in place (scroll and selection kept while the file still matches).
|
||||
const cards = page => page.locator('.gallery-body article[data-image-key]');
|
||||
|
||||
@@ -18,7 +18,7 @@ const routes = [
|
||||
['tags', 'Tags & categories', null],
|
||||
// #249 local Inspect: the native Inspect a file page; Overview's tool card still links the legacy metadata page.
|
||||
['inspect', 'Inspect a file', null],
|
||||
// #249 System S2: native System; every section native since S12 (D94 retired, asserted below).
|
||||
// #249 System S2: native System; its undelivered sections hand off per section (D94, asserted below).
|
||||
['system', 'System', null],
|
||||
// #249 D51: SAVED VIEWS' Favorites, Images filtered to favorites.
|
||||
['favorites', 'Favorites', null],
|
||||
@@ -395,7 +395,7 @@ const SURFACE = {
|
||||
intro: { fontSize: '12px', marginTop: '7px' },
|
||||
};
|
||||
|
||||
for (const width of [1600, 390]) test(`overview renders with concept surfaces and no System section hands off ${width}`, async ({ page }) => {
|
||||
for (const width of [1600, 390]) test(`overview and handoff render with concept surfaces ${width}`, async ({ page }) => {
|
||||
await page.setViewportSize({ width, height: 900 });
|
||||
await page.goto(`${workspace}#/overview`);
|
||||
const main = page.locator('#workspace-main');
|
||||
@@ -412,23 +412,23 @@ for (const width of [1600, 390]) test(`overview renders with concept surfaces an
|
||||
expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true);
|
||||
|
||||
await sidebarLink(page, 'System').click();
|
||||
// Every System section is native (S12 retired D94): the last one, Diagnostics & logs, shows concept surfaces.
|
||||
// System's Overview is native (S2 Task 6); a section not delivered yet still hands off (D94).
|
||||
await main.locator('.section-nav').getByRole('button', { name: 'Diagnostics & logs', exact: true }).click();
|
||||
const logs = main.locator('.settings-layout > section.surface').last();
|
||||
await expect(logs.locator(':scope > h2')).toHaveText('Logs');
|
||||
expect(await logs.evaluate(surfaceStyle)).toMatchObject(SURFACE);
|
||||
await expect(main.locator('.handoff')).toHaveCount(0);
|
||||
const handoff = main.locator('section.handoff');
|
||||
await expect(handoff).toHaveClass(/\bsurface\b/);
|
||||
expect(await handoff.evaluate(surfaceStyle)).toMatchObject(SURFACE);
|
||||
await expect(handoff.locator('a[href="/prompt_manager/admin"]')).toBeVisible();
|
||||
await expect(main.locator('.panel')).toHaveCount(0);
|
||||
expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true);
|
||||
});
|
||||
|
||||
// #249 System S2 Task 5: the concept's renderSystem shell (app.js 698–706) at `#/system/<section>`, every section native
|
||||
// (S12 retired the D94 hand-off), and the Save row's truthful status (D92).
|
||||
// #249 System S2 Task 5: the concept's renderSystem shell (app.js 698–706) at `#/system/<section>`, every section a D94
|
||||
// hand-off until its PR, and the Save row's truthful status (D92).
|
||||
const SYSTEM_SECTION_LABELS = ['Overview', 'Appearance', 'Sources & scans', 'Integrations', 'Auto-tagging', 'Maintenance',
|
||||
'Backup & export', 'Diagnostics & logs'];
|
||||
const SYSTEM_HASHES = ['overview', 'appearance', 'sources', 'integrations', 'tagging', 'maintenance', 'backups', 'diagnostics'];
|
||||
|
||||
test('System is native: the concept shell, section navigation, sections and the Save row', async ({ page }) => {
|
||||
test('System is native: the concept shell, section navigation, sections, hand-offs and the Save row', async ({ page }) => {
|
||||
const mutations = [];
|
||||
page.on('request', item => { if (!['GET', 'HEAD', 'OPTIONS'].includes(item.method())) mutations.push(item.url()); });
|
||||
await page.goto(`${workspace}#/system`);
|
||||
@@ -466,8 +466,7 @@ test('System is native: the concept shell, section navigation, sections and the
|
||||
// (S4 Task 11, workspace-system-sources.spec.mjs, its update notice shown in a fresh browser), Integrations (S10
|
||||
// Task 27, workspace-system-integrations.spec.mjs), Maintenance (S7 Task 19,
|
||||
// workspace-system-maintenance.spec.mjs), Backup & export (S9 Task 24, workspace-system-backups.spec.mjs) and
|
||||
// Auto-tagging (S11 Task 6, workspace-system-tagging.spec.mjs) and Diagnostics & logs (S12,
|
||||
// workspace-system-diagnostics.spec.mjs): none hands off.
|
||||
// Auto-tagging (S11 Task 6, workspace-system-tagging.spec.mjs); every other section hands off.
|
||||
if (hash === 'tagging') await expect(layout.locator('#tag-model')).toBeVisible();
|
||||
if (hash === 'overview') await expect(layout.locator(':scope > .summary-metrics')).toBeVisible();
|
||||
if (hash === 'sources') await expect(layout.locator('#source-list .operation')).toHaveCount(2);
|
||||
@@ -477,9 +476,13 @@ test('System is native: the concept shell, section navigation, sections and the
|
||||
integrations: ['section.surface', 'section.surface', 'div.save-row'],
|
||||
maintenance: ['div.notice', 'section.surface', 'section.surface', 'div.save-row'],
|
||||
backups: ['div.two-col', 'section.surface', 'div.save-row'],
|
||||
tagging: ['section.surface', 'div.save-row'],
|
||||
diagnostics: ['section.surface', 'section.surface', 'div.save-row'] }[hash]);
|
||||
await expect(layout.locator('.handoff')).toHaveCount(0);
|
||||
tagging: ['section.surface', 'div.save-row'] }[hash]
|
||||
?? ['section.surface handoff', 'div.save-row']);
|
||||
if (['overview', 'appearance', 'sources', 'integrations', 'tagging', 'maintenance', 'backups'].includes(hash)) continue;
|
||||
const handoff = layout.locator('section.handoff');
|
||||
await expect(handoff.locator('h2')).toHaveText('Your current tools are ready');
|
||||
expect(await handoff.locator('a.workspace-link').evaluateAll(links => links.map(link => link.getAttribute('href'))))
|
||||
.toEqual(['/prompt_manager/admin']);
|
||||
await expect(layout.locator('#settings-status')).toHaveText('No unsaved changes.');
|
||||
await expect(page.locator('aside.sidebar a[href="#/system"]')).toHaveAttribute('aria-current', 'page');
|
||||
}
|
||||
|
||||
@@ -1,267 +0,0 @@
|
||||
"""System › Diagnostics & logs test hooks for the browser tests (#249 System S12).
|
||||
|
||||
The logger writes into ``<server root>/logs`` (``repoint_logs``, called by
|
||||
``create_app`` before any record is written), so no test reads or deletes a
|
||||
real log folder. Every route refuses a log folder outside the server root.
|
||||
|
||||
- ``POST /__test__/logs/reset`` restores the default logging configuration,
|
||||
empties the active log, removes every other entry of the log folder (a
|
||||
link is removed, never followed) and clears the in-memory buffer. The
|
||||
global ``/__test__/reset`` calls it too;
|
||||
- ``POST /__test__/logs/seed`` ``{records, filler, rotated, active, link,
|
||||
other}`` writes buffer records with fixed times (``records``: the concept's
|
||||
four lines, then one DEBUG, ERROR and CRITICAL record, older; ``filler``
|
||||
more INFO records older still), ``active`` bytes into the active log,
|
||||
rotated copies ``prompt_manager.log.<n>`` of the listed sizes (fixed
|
||||
times), on POSIX a symlink ``prompt_manager.log.<n+1>`` to a file outside
|
||||
the folder (``link``) and a user's ``prompt_manager.log.bak`` (``other``).
|
||||
``{mode: "concept"}`` reproduces the concept's Logs sample (app.js 695)
|
||||
for the comparison: the buffer holds only its four lines, the active log
|
||||
is 1.4 MB with three rotated copies ("prompt_manager.log · 1.4 MB · rotated
|
||||
copies: 3") and console logging is off, as the concept's toggle;
|
||||
- ``POST /__test__/logs/rotate`` rolls the copies over as the handler does
|
||||
(``.2`` → ``.3``, ``.1`` → ``.2``, a new ``.1``), so a reviewed plan no
|
||||
longer matches;
|
||||
- ``GET /__test__/logs/state`` answers the folder's entries (name, size, link)
|
||||
and the buffer's length.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
ACTIVE = "prompt_manager.log"
|
||||
SEED_DAY = datetime.datetime(2026, 10, 26)
|
||||
# The concept's four lines (app.js 695), oldest first: the viewer shows them
|
||||
# newest first.
|
||||
CONCEPT_RECORDS = (
|
||||
("09:38:52", logging.INFO, "Source monitor started: output"),
|
||||
("09:39:04", logging.INFO, "Database backup verified"),
|
||||
("09:42:18", logging.WARNING, "3 images have ambiguous prompt metadata"),
|
||||
("09:42:18", logging.INFO, "Scan completed: 12 enriched, 0 duplicate links"),
|
||||
)
|
||||
# Older than the concept's lines, so they come after them: the levels the
|
||||
# concept does not show.
|
||||
LEVEL_RECORDS = (
|
||||
("09:30:01", logging.DEBUG, "Thumbnail cache lookup"),
|
||||
("09:30:02", logging.ERROR, "A source folder could not be read"),
|
||||
("09:30:03", logging.CRITICAL, "The database could not be opened"),
|
||||
)
|
||||
ROTATED_TIME = datetime.datetime(2026, 10, 25, 14, 2)
|
||||
# The concept's "prompt_manager.log · 1.4 MB · rotated copies: 3": the active
|
||||
# log stays under 1.45 MB while the server writes its own lines.
|
||||
CONCEPT_ACTIVE = int(1.4 * 1024 * 1024)
|
||||
CONCEPT_ROTATED = (3072, 2048, 1024)
|
||||
MAX_SEED_BYTES = 64 * 1024 * 1024
|
||||
DEFAULT_CONFIG = {
|
||||
"level": "INFO",
|
||||
"max_file_size": 10 * 1024 * 1024,
|
||||
"backup_count": 5,
|
||||
"console_logging": True,
|
||||
"file_logging": True,
|
||||
"buffer_size": 1000,
|
||||
}
|
||||
|
||||
|
||||
def _manager():
|
||||
from utils.logging_config import get_logger_manager
|
||||
|
||||
return get_logger_manager()
|
||||
|
||||
|
||||
def repoint_logs(root: Path) -> Path:
|
||||
"""Point the logger at ``<root>/logs`` before anything is logged.
|
||||
|
||||
The logger is built while ``logs_dir`` answers that folder, so its first
|
||||
handler never opens a log file anywhere else (a node folder with a legacy
|
||||
database would otherwise name a real log folder).
|
||||
"""
|
||||
from utils import library_location
|
||||
|
||||
folder = Path(root) / "logs"
|
||||
folder.mkdir(exist_ok=True)
|
||||
original = library_location.logs_dir
|
||||
library_location.logs_dir = lambda **_kwargs: str(folder)
|
||||
try:
|
||||
manager = _manager()
|
||||
finally:
|
||||
library_location.logs_dir = original
|
||||
with manager._files_lock:
|
||||
manager.log_dir = folder
|
||||
manager._setup_loggers()
|
||||
return folder
|
||||
|
||||
|
||||
def _require_inside(root: Path, folder: Path) -> None:
|
||||
resolved = Path(os.path.realpath(folder))
|
||||
if Path(os.path.realpath(root)) not in resolved.parents:
|
||||
raise web.HTTPForbidden(text="The log folder is outside the test server root")
|
||||
|
||||
|
||||
def _record(time_text, level, message, index):
|
||||
record = logging.LogRecord(
|
||||
"prompt_manager.test", level, __file__, index, message, None, None
|
||||
)
|
||||
hours, minutes, seconds = (int(part) for part in time_text.split(":"))
|
||||
moment = SEED_DAY.replace(hour=hours, minute=minutes, second=seconds)
|
||||
# Equal times keep their order: each later record is a microsecond newer.
|
||||
record.created = moment.timestamp() + index / 1e6
|
||||
return record
|
||||
|
||||
|
||||
def _touch(path: Path, size: int, moment: datetime.datetime) -> None:
|
||||
# A link at this name is replaced, never written through.
|
||||
if path.is_symlink():
|
||||
path.unlink()
|
||||
with open(path, "wb") as handle:
|
||||
handle.write(b"x" * size)
|
||||
stamp = moment.timestamp()
|
||||
os.utime(path, (stamp, stamp))
|
||||
|
||||
|
||||
def _bounded(value, name, top=MAX_SEED_BYTES):
|
||||
if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= top:
|
||||
raise web.HTTPBadRequest(text=f"{name} must be an integer 0..{top}")
|
||||
return value
|
||||
|
||||
|
||||
def register_diagnostics_commands(app, routes, api, root: Path):
|
||||
"""Register the hooks; returns ``reset()`` for the global reset."""
|
||||
|
||||
def folder() -> Path:
|
||||
path = Path(_manager().log_dir)
|
||||
_require_inside(root, path)
|
||||
return path
|
||||
|
||||
def reset() -> None:
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
if manager.get_config() != DEFAULT_CONFIG:
|
||||
manager.update_config(dict(DEFAULT_CONFIG))
|
||||
with manager._files_lock:
|
||||
held = manager._acquire_file_handlers()
|
||||
try:
|
||||
with os.scandir(log_dir) as entries:
|
||||
found = list(entries)
|
||||
for entry in found:
|
||||
path = Path(entry.path)
|
||||
if entry.name == ACTIVE and entry.is_file(follow_symlinks=False):
|
||||
manager._truncate_main_log(path)
|
||||
elif entry.is_dir(follow_symlinks=False):
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
os.unlink(path)
|
||||
finally:
|
||||
manager._release_handlers(held)
|
||||
with manager._buffer_lock:
|
||||
manager._log_buffer.clear()
|
||||
|
||||
def seed_concept() -> None:
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
manager.update_config({"console_logging": False})
|
||||
with manager._buffer_lock:
|
||||
manager._log_buffer.clear()
|
||||
for index, (time_text, level, message) in enumerate(CONCEPT_RECORDS):
|
||||
record = _record(time_text, level, message, index)
|
||||
manager.add_to_buffer(record, record.getMessage())
|
||||
with open(log_dir / ACTIVE, "ab") as handle:
|
||||
handle.write(b"x" * CONCEPT_ACTIVE)
|
||||
for number, size in enumerate(CONCEPT_ROTATED, start=1):
|
||||
moment = ROTATED_TIME - datetime.timedelta(hours=number)
|
||||
_touch(log_dir / f"{ACTIVE}.{number}", size, moment)
|
||||
|
||||
def seed(data) -> None:
|
||||
if data.get("mode") == "concept":
|
||||
seed_concept()
|
||||
return
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
with_records = data.get("records", True) is not False
|
||||
filler = _bounded(data.get("filler", 0), "filler", 900)
|
||||
older = [
|
||||
(f"08:{index // 60:02}:{index % 60:02}", logging.INFO, f"Filler {index}")
|
||||
for index in range(filler)
|
||||
]
|
||||
named = list(LEVEL_RECORDS) + list(CONCEPT_RECORDS) if with_records else []
|
||||
ordered = older + named
|
||||
for index, (time_text, level, message) in enumerate(ordered):
|
||||
record = _record(time_text, level, message, index)
|
||||
manager.add_to_buffer(record, record.getMessage())
|
||||
active = _bounded(data.get("active", 0), "active")
|
||||
if active:
|
||||
with open(log_dir / ACTIVE, "ab") as handle:
|
||||
handle.write(b"x" * active)
|
||||
rotated = data.get("rotated", [])
|
||||
if not isinstance(rotated, list) or len(rotated) > 20:
|
||||
raise web.HTTPBadRequest(text="rotated must be a list of sizes")
|
||||
for number, size in enumerate(rotated, start=1):
|
||||
moment = ROTATED_TIME - datetime.timedelta(hours=number)
|
||||
_touch(log_dir / f"{ACTIVE}.{number}", _bounded(size, "size"), moment)
|
||||
if data.get("link") and os.name != "nt":
|
||||
target = Path(root) / "outside-log-target.txt"
|
||||
target.write_bytes(b"not a log of this folder")
|
||||
os.symlink(target, log_dir / f"{ACTIVE}.{len(rotated) + 1}")
|
||||
if data.get("other"):
|
||||
_touch(log_dir / f"{ACTIVE}.bak", 0, ROTATED_TIME)
|
||||
|
||||
def rotate() -> None:
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
with manager._files_lock:
|
||||
held = manager._acquire_file_handlers()
|
||||
try:
|
||||
for number in (2, 1):
|
||||
source = log_dir / f"{ACTIVE}.{number}"
|
||||
if source.is_file() and not source.is_symlink():
|
||||
os.replace(source, log_dir / f"{ACTIVE}.{number + 1}")
|
||||
_touch(log_dir / f"{ACTIVE}.1", 4096, datetime.datetime.now())
|
||||
finally:
|
||||
manager._release_handlers(held)
|
||||
|
||||
def state():
|
||||
manager = _manager()
|
||||
log_dir = folder()
|
||||
entries = []
|
||||
with os.scandir(log_dir) as listing:
|
||||
found = sorted(listing, key=lambda item: item.name)
|
||||
for entry in found:
|
||||
info = entry.stat(follow_symlinks=False)
|
||||
entries.append(
|
||||
{"name": entry.name, "size": info.st_size, "link": entry.is_symlink()}
|
||||
)
|
||||
with manager._buffer_lock:
|
||||
buffered = len(manager._log_buffer)
|
||||
return {"entries": entries, "buffer": buffered, "config": manager.get_config()}
|
||||
|
||||
async def body(request):
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
if not isinstance(data, dict):
|
||||
raise web.HTTPBadRequest()
|
||||
return data
|
||||
|
||||
@routes.post("/__test__/logs/reset")
|
||||
async def logs_reset(request: web.Request) -> web.Response:
|
||||
await api._run_in_executor(reset)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/logs/seed")
|
||||
async def logs_seed(request: web.Request) -> web.Response:
|
||||
data = await body(request)
|
||||
await api._run_in_executor(seed, data)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/logs/rotate")
|
||||
async def logs_rotate(request: web.Request) -> web.Response:
|
||||
await api._run_in_executor(rotate)
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.get("/__test__/logs/state")
|
||||
async def logs_state(request: web.Request) -> web.Response:
|
||||
return web.json_response(await api._run_in_executor(state))
|
||||
|
||||
return reset
|
||||
-107
@@ -1,107 +0,0 @@
|
||||
{
|
||||
"generation_params": [
|
||||
"case:basic",
|
||||
"case:shared-prompt-pipeline",
|
||||
"tests/fixtures/inspection_parity/f02.png",
|
||||
"tests/fixtures/inspection_parity/f03.png",
|
||||
"tests/fixtures/inspection_parity/f06c.png",
|
||||
"tests/fixtures/inspection_parity/f06e.png",
|
||||
"tests/fixtures/inspection_parity/f07a.png",
|
||||
"tests/fixtures/inspection_parity/f07b.png",
|
||||
"tests/fixtures/inspection_parity/f08.png",
|
||||
"tests/fixtures/inspection_parity/f09.png",
|
||||
"tests/fixtures/inspection_parity/f10.png",
|
||||
"tests/fixtures/inspection_parity/f11.png",
|
||||
"tests/fixtures/inspection_parity/f15.png",
|
||||
"tests/fixtures/inspection_parity/f16.png",
|
||||
"tests/fixtures/inspection_parity/f23.png"
|
||||
],
|
||||
"inspection_parity": [
|
||||
"f02",
|
||||
"f03",
|
||||
"f06c",
|
||||
"f06e",
|
||||
"f07a",
|
||||
"f07b",
|
||||
"f08",
|
||||
"f09",
|
||||
"f10",
|
||||
"f11",
|
||||
"f14",
|
||||
"f15",
|
||||
"f16",
|
||||
"f23"
|
||||
],
|
||||
"local_resolver": [
|
||||
"anchor-first-stage-no-latent",
|
||||
"anchor-same-links",
|
||||
"anchor-same-links-unanchored",
|
||||
"built-chars",
|
||||
"class-type-missing",
|
||||
"class-type-number",
|
||||
"combine-equal",
|
||||
"controlnet-slot1",
|
||||
"custom-cfg",
|
||||
"duplicate-key",
|
||||
"encode",
|
||||
"int-id",
|
||||
"latent-encode",
|
||||
"overwritten-bad-workflow",
|
||||
"pair-execution",
|
||||
"pair-execution-bad-json",
|
||||
"pair-execution-mismatch",
|
||||
"pair-execution-negative-changed",
|
||||
"pair-execution-other-node",
|
||||
"pair-execution-truncated",
|
||||
"pair-execution-two-records",
|
||||
"pair-execution-version-float",
|
||||
"pair-execution-version-true",
|
||||
"pair-graph",
|
||||
"pass-through-unknown-role-inputs",
|
||||
"passthrough",
|
||||
"prompt-and-parameters",
|
||||
"prompt-manager",
|
||||
"select-detailer-chain-same",
|
||||
"select-detailer-no-sampler-upstream",
|
||||
"select-detailer-output-walk-exhausted",
|
||||
"select-detailer-same-prompt",
|
||||
"select-edge-hires-fix-denoised-slot",
|
||||
"select-edge-hires-fix-lineage-cycle",
|
||||
"select-edge-hires-fix-two-refiners",
|
||||
"select-flux-concat",
|
||||
"select-flux-reroute-input",
|
||||
"select-flux-three-stage",
|
||||
"select-hires-fix",
|
||||
"select-hires-fix-both-saved",
|
||||
"select-hires-fix-reroute",
|
||||
"select-hires-fix-unlisted",
|
||||
"select-multistage-latent",
|
||||
"select-preview-dropped",
|
||||
"select-preview-through-bad-node",
|
||||
"select-runtime-flux-other-full-prompt",
|
||||
"select-runtime-flux-trigger",
|
||||
"select-runtime-search-list-append",
|
||||
"select-runtime-through-chained",
|
||||
"select-runtime-through-concat",
|
||||
"select-runtime-through-text-encode",
|
||||
"select-runtime-traced-prepend",
|
||||
"select-runtime-trigger-append-single",
|
||||
"select-runtime-trigger-pair",
|
||||
"select-runtime-trigger-prepend",
|
||||
"select-three-identical-one-advanced",
|
||||
"select-two-identical",
|
||||
"select-unconsumed-stage-kept",
|
||||
"several-samplers",
|
||||
"smz-false",
|
||||
"text-at-limit",
|
||||
"unknown-encoder",
|
||||
"workflow-type-dict",
|
||||
"workflow-type-in-subgraph",
|
||||
"workflow-type-list",
|
||||
"workflow-type-without-id",
|
||||
"zeroed"
|
||||
],
|
||||
"unknown_nodes": [
|
||||
"unknown-role-inputs"
|
||||
]
|
||||
}
|
||||
+1
-1
File diff suppressed because one or more lines are too long
@@ -36,48 +36,33 @@ function fixture() {
|
||||
const reply = (index, data) => requests[index].resolve({ json: async () => data });
|
||||
return { manager, requests, reply, notifications };
|
||||
}
|
||||
const SHA = 'a'.repeat(64);
|
||||
test('gallery review retains root identity and applies the tags to the image by its sha256 (#379)', async () => {
|
||||
test('gallery review retains root identity and applies tags with the returned prompt ID', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
assert.equal(requests[0].body.root_index, 1);
|
||||
assert.equal(requests[0].body.existing, 'merge');
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'], sha256: SHA, image_tags: [] });
|
||||
await load;
|
||||
manager.loadNextReviewImage = () => {};
|
||||
const apply = manager.applyReviewTags();
|
||||
assert.equal(requests[1].url, '/prompt_manager/autotag/apply');
|
||||
assert.deepEqual(requests[1].body, ImageHelpers.autotagApplyBody(
|
||||
{ id: 'digest', path: 'same.png', root_index: 1 }, SHA, ['smile']));
|
||||
assert.equal('prompt_id' in requests[1].body, false);
|
||||
reply(1, { success: true });
|
||||
await apply;
|
||||
});
|
||||
test('an unlinked image applies to the image like a linked one', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
reply(0, { success: true, prompt_id: null, tags: ['smile'], sha256: SHA, image_tags: [] });
|
||||
await load;
|
||||
manager.loadNextReviewImage = () => {};
|
||||
const apply = manager.applyReviewTags();
|
||||
assert.equal(requests[1].body.sha256, SHA);
|
||||
assert.deepEqual(requests[1].body.tags, ['smile']);
|
||||
reply(1, { success: true });
|
||||
await apply;
|
||||
});
|
||||
test('an answer without a captured sha256 never sends an apply', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'] });
|
||||
await load;
|
||||
manager.loadNextReviewImage = () => {};
|
||||
await manager.applyReviewTags();
|
||||
assert.equal(requests.filter(r => r.url === '/prompt_manager/autotag/apply').length, 0);
|
||||
const apply = manager.applyReviewTags();
|
||||
assert.deepEqual(requests[1].body, { prompt_id: 42, tags: ['smile'] });
|
||||
reply(1, { success: true });
|
||||
await apply;
|
||||
});
|
||||
test('unlinked images do not submit an invalid apply request or advance', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
reply(0, { success: true, prompt_id: null, tags: ['smile'] });
|
||||
await load;
|
||||
const apply = manager.applyReviewTags();
|
||||
if (requests[1]) reply(1, { success: false, error: "Unexpected apply" });
|
||||
await apply;
|
||||
assert.equal(requests.length, 1);
|
||||
assert.equal(manager.autoTagState.reviewIndex, 0);
|
||||
});
|
||||
test('a failed apply keeps the current image available for retry', async () => {
|
||||
const { manager, requests, reply } = fixture();
|
||||
const load = manager.loadNextReviewImage();
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'], sha256: SHA, image_tags: [] });
|
||||
reply(0, { success: true, prompt_id: 42, tags: ['smile'] });
|
||||
await load;
|
||||
const apply = manager.applyReviewTags();
|
||||
reply(1, { success: false, error: 'Try again' });
|
||||
|
||||
@@ -5,8 +5,6 @@ const {
|
||||
formatImageCaption,
|
||||
workflowDocumentHtml,
|
||||
autotagSingleBody,
|
||||
autotagApplyBody,
|
||||
autotagCompleteMessage,
|
||||
fallbackImageSource,
|
||||
} = require("../../web/js/image-helpers.js");
|
||||
|
||||
@@ -123,27 +121,3 @@ test('metadata lookup preserves encoded filename characters and works behind a p
|
||||
assert.equal(metadataLookupUrl('/prompt_manager/images/serve/a.png?v=123'), '/prompt_manager/images/prompt/a.png');
|
||||
assert.equal(metadataLookupUrl('/elsewhere/a.png'), null);
|
||||
});
|
||||
|
||||
test("autotagSingleBody asks to skip auto-tagged images only when told (#379)", () => {
|
||||
assert.equal(autotagSingleBody({ id: 1 }, WD14).existing, "merge");
|
||||
assert.equal(autotagSingleBody({ id: 1 }, { ...WD14, existing: "skip" }).existing, "skip");
|
||||
assert.equal(autotagSingleBody({ id: 1 }, { ...WD14, existing: "odd" }).existing, "merge");
|
||||
});
|
||||
|
||||
test("autotagApplyBody names the image, its bytes and the tags, never a prompt (#379)", () => {
|
||||
const sha = "a".repeat(64);
|
||||
assert.deepEqual(autotagApplyBody({ id: 4, prompt_id: 7 }, sha, ["bird"]), { image_id: 4, sha256: sha, tags: ["bird"] });
|
||||
assert.deepEqual(autotagApplyBody({ id: "digest", path: "sub/a.png", root_index: 1 }, sha, []),
|
||||
{ path: "sub/a.png", root_index: 1, sha256: sha, tags: [] });
|
||||
assert.deepEqual(autotagApplyBody({ id: 2 }, undefined, null), { image_id: 2, sha256: "", tags: [] });
|
||||
const tags = ["x"];
|
||||
autotagApplyBody({ id: 1 }, sha, tags).tags.push("y");
|
||||
assert.deepEqual(tags, ["x"]);
|
||||
});
|
||||
|
||||
test("autotagCompleteMessage counts images (#379)", () => {
|
||||
assert.equal(autotagCompleteMessage(2), "Auto tagging complete! Tagged 2 images.");
|
||||
assert.equal(autotagCompleteMessage(1), "Auto tagging complete! Tagged 1 image.");
|
||||
assert.equal(autotagCompleteMessage(0), "Auto tagging complete! Tagged 0 images.");
|
||||
assert.equal(autotagCompleteMessage(undefined), "Auto tagging complete! Tagged 0 images.");
|
||||
});
|
||||
|
||||
@@ -97,30 +97,6 @@ test("connect reports a non-2xx response through onerror and never onmessage", a
|
||||
assert.equal(source.readyState, source.CLOSED);
|
||||
});
|
||||
|
||||
test("a non-2xx JSON answer's error sentence reaches onerror as serverMessage", async () => {
|
||||
let error = null;
|
||||
const fetchImpl = async () => new Response(
|
||||
JSON.stringify({ success: false, error: "Auto-tagging is already running" }),
|
||||
{ status: 409, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
const source = connect("/x", { fetchImpl });
|
||||
source.onerror = (err) => { error = err; };
|
||||
await until(() => error !== null);
|
||||
assert.match(error.message, /409/);
|
||||
assert.equal(error.status, 409);
|
||||
assert.equal(error.serverMessage, "Auto-tagging is already running");
|
||||
});
|
||||
|
||||
test("a non-2xx answer without a JSON error has no serverMessage", async () => {
|
||||
let error = null;
|
||||
const fetchImpl = async () => new Response("nope", { status: 500 });
|
||||
const source = connect("/x", { fetchImpl });
|
||||
source.onerror = (err) => { error = err; };
|
||||
await until(() => error !== null);
|
||||
assert.equal(error.status, 500);
|
||||
assert.equal(error.serverMessage, undefined);
|
||||
});
|
||||
|
||||
test("connect reports a network failure through onerror", async () => {
|
||||
let error = null;
|
||||
const source = connect("/x", { fetchImpl: async () => { throw new TypeError("Failed to fetch"); } });
|
||||
|
||||
@@ -40,7 +40,7 @@ const semantics = compileSemantics(JSON.parse(fs.readFileSync(path.join(here, '.
|
||||
const { extractGenerationFacts } = require(path.join(here, '..', '..', '..', 'web', 'js', 'comfy-metadata.js'));
|
||||
|
||||
/** The local evidence for one set of record texts. */
|
||||
const resolved = (records, fileName = null) => ({ evidence: resolveLocalEvidence(records, semantics, fileName) });
|
||||
const resolved = records => ({ evidence: resolveLocalEvidence(records, semantics) });
|
||||
|
||||
/** The local wire's evidence and six facts for one read (no legacy fill, as in the vectors' own harness). */
|
||||
function localPayload(read, file, classification) {
|
||||
@@ -59,14 +59,14 @@ const output = {};
|
||||
for (const entry of JSON.parse(Buffer.concat(chunks).toString('utf8'))) {
|
||||
const { name } = entry;
|
||||
if (entry.mode === 'resolve') {
|
||||
if (entry.records !== undefined) { output[name] = resolved(entry.records, entry.fileName ?? null); continue; }
|
||||
if (entry.records !== undefined) { output[name] = resolved(entry.records); continue; }
|
||||
const blob = await fs.openAsBlob(entry.path);
|
||||
const { classification, result } = await readLocalFile(blob, { pngMetadata });
|
||||
const read = classification === null ? null : toInspectionRead(classification, result).read;
|
||||
if (classification === null) { output[name] = { unreadable: true }; continue; }
|
||||
// The evidence is the local wire's own (buildLocalPayload), and it equals resolveLocalEvidence of the records.
|
||||
const local = localPayload(result, { name: path.basename(entry.path), size: blob.size }, classification);
|
||||
const direct = read === null ? unreadableEvidence() : resolved(read.records, path.basename(entry.path)).evidence;
|
||||
const direct = read === null ? unreadableEvidence() : resolved(read.records).evidence;
|
||||
if (JSON.stringify(direct) !== JSON.stringify(local.evidence)) throw new Error(`${name}: the wire's roles are not the resolver's`);
|
||||
output[name] = read === null ? { unreadable: true, ...local } : local;
|
||||
continue;
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const source = import('../../web/js/workspace/images-multiselect.mjs');
|
||||
|
||||
const KEYS = Object.freeze(['0:a', '0:b', '0:c', '0:d', '1:e']);
|
||||
|
||||
test('a range runs from the anchor to the target in display order, either way', async () => {
|
||||
const { rangeKeys } = await source;
|
||||
assert.deepEqual(rangeKeys(KEYS, '0:b', '0:d'), ['0:b', '0:c', '0:d']);
|
||||
assert.deepEqual(rangeKeys(KEYS, '1:e', '0:c'), ['0:c', '0:d', '1:e']);
|
||||
assert.deepEqual(rangeKeys(KEYS, '0:c', '0:c'), ['0:c']);
|
||||
});
|
||||
|
||||
test('a range without a listed anchor is the target alone, and an unlisted target is empty', async () => {
|
||||
const { rangeKeys } = await source;
|
||||
assert.deepEqual(rangeKeys(KEYS, null, '0:c'), ['0:c']);
|
||||
assert.deepEqual(rangeKeys(KEYS, '9:gone', '0:c'), ['0:c']);
|
||||
assert.deepEqual(rangeKeys(KEYS, '0:a', '9:gone'), []);
|
||||
});
|
||||
|
||||
test('applying keys adds or removes them and keeps the same frozen set when nothing changes', async () => {
|
||||
const { applyKeys } = await source;
|
||||
const start = Object.freeze(new Set(['0:a']));
|
||||
const added = applyKeys(start, ['0:b', '0:c'], true);
|
||||
assert.deepEqual([...added], ['0:a', '0:b', '0:c']);
|
||||
assert.ok(Object.isFrozen(added));
|
||||
assert.deepEqual([...start], ['0:a']);
|
||||
const removed = applyKeys(added, ['0:a', '0:b'], false);
|
||||
assert.deepEqual([...removed], ['0:c']);
|
||||
assert.equal(applyKeys(removed, ['0:c'], true), removed);
|
||||
assert.equal(applyKeys(removed, ['0:a'], false), removed);
|
||||
});
|
||||
|
||||
test('every listed key is checked only when each one is in the set', async () => {
|
||||
const { everyChecked } = await source;
|
||||
assert.equal(everyChecked(new Set(KEYS), KEYS), true);
|
||||
assert.equal(everyChecked(new Set(KEYS.slice(1)), KEYS), false);
|
||||
assert.equal(everyChecked(new Set(['x', ...KEYS.slice(1)]), KEYS), false);
|
||||
assert.equal(everyChecked(new Set(), []), false);
|
||||
});
|
||||
|
||||
test('the select-all-matching offer needs every loaded image checked and more to load', async () => {
|
||||
const { offersAllMatching } = await source;
|
||||
const all = new Set(KEYS);
|
||||
assert.equal(offersAllMatching(all, KEYS, { total: 9, hasMore: true }), true);
|
||||
assert.equal(offersAllMatching(all, KEYS, { total: 5, hasMore: false }), false);
|
||||
assert.equal(offersAllMatching(new Set(KEYS.slice(1)), KEYS, { total: 9, hasMore: true }), false);
|
||||
assert.equal(offersAllMatching(new Set(), [], { total: 9, hasMore: true }), false);
|
||||
});
|
||||
|
||||
test('the counts read "N selected" and "All N matching selected", grouped', async () => {
|
||||
const { matchingLabel, matchingOffer } = await source;
|
||||
assert.equal(matchingLabel(12345), 'All 12,345 matching selected');
|
||||
assert.equal(matchingOffer(12345), 'Select all 12,345 matching');
|
||||
});
|
||||
|
||||
function fakeTimers() {
|
||||
const timers = new Map();
|
||||
let next = 1;
|
||||
return {
|
||||
set: (callback, delay) => { timers.set(next, { callback, delay }); return next++; },
|
||||
clear: id => timers.delete(id),
|
||||
fire: () => { for (const [id, { callback }] of [...timers]) { timers.delete(id); callback(); } },
|
||||
pending: () => timers.size,
|
||||
delays: () => [...timers.values()].map(timer => timer.delay),
|
||||
};
|
||||
}
|
||||
|
||||
test('a held press fires once after the delay and swallows the click that follows', async () => {
|
||||
const { createLongPress, LONG_PRESS_MS } = await source;
|
||||
const timers = fakeTimers(), fired = [];
|
||||
const press = createLongPress({ onLongPress: target => fired.push(target), setTimer: timers.set, clearTimer: timers.clear });
|
||||
press.down(10, 10, 'card-a');
|
||||
assert.deepEqual(timers.delays(), [LONG_PRESS_MS]);
|
||||
assert.equal(LONG_PRESS_MS, 500);
|
||||
press.move(14, 13);
|
||||
timers.fire();
|
||||
assert.deepEqual(fired, ['card-a']);
|
||||
press.up();
|
||||
assert.equal(press.consumeClick(), true);
|
||||
assert.equal(press.consumeClick(), false);
|
||||
});
|
||||
|
||||
test('moving past the slop, lifting early or cancelling never fires', async () => {
|
||||
const { createLongPress, LONG_PRESS_SLOP } = await source;
|
||||
assert.equal(LONG_PRESS_SLOP, 10);
|
||||
const timers = fakeTimers(), fired = [];
|
||||
const press = createLongPress({ onLongPress: target => fired.push(target), setTimer: timers.set, clearTimer: timers.clear });
|
||||
press.down(0, 0, 'a'); press.move(0, 11);
|
||||
assert.equal(timers.pending(), 0);
|
||||
press.down(0, 0, 'b'); press.up();
|
||||
assert.equal(timers.pending(), 0);
|
||||
press.down(0, 0, 'c');
|
||||
assert.equal(press.pending(), true);
|
||||
press.cancel();
|
||||
assert.equal(press.pending(), false);
|
||||
assert.equal(timers.pending(), 0);
|
||||
press.move(0, 0); press.up(); press.cancel();
|
||||
assert.deepEqual(fired, []);
|
||||
assert.equal(press.consumeClick(), false);
|
||||
// A new press drops a long press whose click never came.
|
||||
press.down(0, 0, 'd'); timers.fire();
|
||||
press.down(0, 0, 'e');
|
||||
assert.equal(press.consumeClick(), false);
|
||||
timers.fire();
|
||||
assert.deepEqual(fired, ['d', 'e']);
|
||||
// A cancel (a mouse press on a hybrid device) forgets a fired press whose click never came.
|
||||
press.cancel();
|
||||
assert.equal(press.consumeClick(), false);
|
||||
});
|
||||
@@ -494,11 +494,3 @@ test('defaults apply to a new visit but not to the current view or a saved view'
|
||||
assert.deepEqual(initialImagesView({}), { layout: 'grid', density: 'comfortable' });
|
||||
assert.deepEqual(initialImagesView({ preferences: {} }), { layout: 'grid', density: 'comfortable' });
|
||||
});
|
||||
|
||||
test('listingFilter is the listing query as an object, without paging (#392)', async () => {
|
||||
const { listingFilter } = await source;
|
||||
assert.deepEqual(listingFilter({q:'cat', root:'1', subfolder:'a/b', category:'Animals', tags:[' x ','y'], media:'video', sort:'oldest', favorites:true}),
|
||||
{q:'cat', root:'1', subfolder:'a/b', category:'Animals', tags:'x,y', media:'video', sort:'oldest', favorites:'1'});
|
||||
assert.deepEqual(listingFilter({root:'all', q:'', tags:[], media:'all'}), {sort:'newest'});
|
||||
assert.throws(() => listingFilter({sort:'sideways'}), TypeError);
|
||||
});
|
||||
|
||||
@@ -196,29 +196,3 @@ test('a preset starts the Tag image dialog on the chosen model and options', asy
|
||||
assert.throws(() => createImageTaggingController({ api, target, preset: { model: 'gguf', options: { instructions: '' } } }), TypeError);
|
||||
assert.throws(() => createImageTaggingController({ api, target, preset: { model: 'gguf', options: { nope: 1 } } }), TypeError);
|
||||
});
|
||||
|
||||
test('all matching sends the listing filter instead of targets, up to 10,000 images (#392)', async () => {
|
||||
const { openAddTags, openAutoTag, MAX_FILTER_IMAGES, TOO_MANY_MATCHING } = module();
|
||||
assert.equal(MAX_FILTER_IMAGES, 10000);
|
||||
assert.equal(TOO_MANY_MATCHING, 'Narrow the view to at most 10,000 images.');
|
||||
const toasts = [], posted = [], tagged = [];
|
||||
let ended = null;
|
||||
const store = { start: async (kind, body) => { posted.push([kind, body]); return { job: { id: JOB_ID, state: 'running' } }; },
|
||||
find: () => ended, subscribe: () => () => {} };
|
||||
const filter = { q: 'cat', sort: 'newest' };
|
||||
// The loaded images can be more than 500: the filter, not the identities, is sent.
|
||||
const loaded = Array.from({ length: 600 }, (_, index) => image(`bulk/${index}.png`));
|
||||
let dialog = null;
|
||||
openAddTags({ scope: pageScope(), store, images: loaded, filter, count: 1234, notify: text => toasts.push(text),
|
||||
onTagged: keys => tagged.push(keys.size), openInput: args => { dialog = args; } });
|
||||
assert.ok(dialog, 'the Add tags dialog opens');
|
||||
ended = { id: JOB_ID, state: 'finished', counters: { tagged: 3 } };
|
||||
await dialog.onApply('sunset');
|
||||
assert.deepEqual(posted, [['image-tags', { mode: 'add', filter, tags: ['sunset'] }]]);
|
||||
// The loaded files are read again when the job tagged any.
|
||||
assert.deepEqual(tagged, [600]);
|
||||
openAddTags({ scope: pageScope(), store, images: loaded, filter, count: 10001, notify: text => toasts.push(text),
|
||||
openInput: () => assert.fail('no dialog over the cap') });
|
||||
openAutoTag({ api: {}, scope: pageScope(), store, images: loaded, filter, count: 10001, notify: text => toasts.push(text) });
|
||||
assert.deepEqual(toasts, [TOO_MANY_MATCHING, TOO_MANY_MATCHING]);
|
||||
});
|
||||
|
||||
@@ -155,9 +155,7 @@ test('committed server answers equal the local evidence', () => {
|
||||
value !== null && typeof value === 'object' ? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
||||
let compared = 0;
|
||||
for (const {name, records, evidence} of vectors) {
|
||||
// A case may name the inspected file (`__file_name`, #381 saver anchoring); it is no record.
|
||||
const {__file_name: fileName = null, ...roles} = records;
|
||||
assert.deepEqual(canonical(resolver.resolveLocalEvidence(roles, semantics, fileName)), canonical(evidence), name);
|
||||
assert.deepEqual(canonical(resolver.resolveLocalEvidence(records, semantics)), canonical(evidence), name);
|
||||
compared++;
|
||||
}
|
||||
assert.ok(compared > 280, `compared ${compared}`);
|
||||
|
||||
@@ -23,7 +23,7 @@ test('tags is a native route titled Tags; only unbuilt routes hand off to legacy
|
||||
assert.equal(routeTitle(route), route.label);
|
||||
}
|
||||
assert.equal(routeTitle(resolveRoute('#/library/7')), 'Prompt library');
|
||||
// System is native from S2 (#249 System), and every section of it from S12 (D94 retired), not the route.
|
||||
// System is native from S2 (#249 System): its undelivered sections hand off per section (D94), not the route.
|
||||
for (const id of ['images', 'library', 'tags', 'inspect', 'overview', 'activity', 'review', 'system']) assert.equal(isHandoff(resolveRoute(`#/${id}`)), false, id);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
// System › Diagnostics & logs (#249 System S12, #329): the concept's two surfaces (app.js 694–695) with live values: the
|
||||
// meta-grid texts, the report's name and merge (D110), the log viewer's lines, Lines validation, statistics, retention
|
||||
// and file rows, and the Clear all logs copy (D109).
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const diagnostics = import('../../web/js/workspace/system-diagnostics.mjs');
|
||||
|
||||
const CONCEPT = fs.readFileSync(path.join(__dirname, '../../docs/concepts/workspace/app.js'), 'utf8');
|
||||
const SOURCE = fs.readFileSync(path.join(__dirname, '../../web/js/workspace/system-diagnostics.mjs'), 'utf8');
|
||||
const SYSTEM = fs.readFileSync(path.join(__dirname, '../../web/js/workspace/system.mjs'), 'utf8');
|
||||
|
||||
const REPORT = {
|
||||
checks: { database: { status: 'ok', prompt_count: 4210, has_images_table: true, image_count: 19956 } },
|
||||
database_info: { database_path: 'user/__promptmanager/prompts.db', database_size_bytes: 1717986918 },
|
||||
sources: { configured: 2, effective: 2, watching: 2, unavailable: 0 },
|
||||
viewerjs: '1.15.2',
|
||||
};
|
||||
|
||||
test('the section keeps the concept surfaces, terms, rows and labels; no sample copy ships', () => {
|
||||
const start = CONCEPT.indexOf('case "diagnostics":');
|
||||
const body = CONCEPT.slice(start, CONCEPT.indexOf('break;', start));
|
||||
const terms = [...body.matchAll(/<dt>([^<]+)<\/dt>/g)].map(([, term]) => term);
|
||||
assert.deepEqual(terms, ['Database', 'Sources', 'Viewer.js', 'Host renderers']);
|
||||
for (const text of ["'Diagnostics'", "'Logs'", "'Run checks'", "'Refresh'", "'Level'", "'Log files & retention'",
|
||||
"'Review log cleanup'", "'Runtime log level'", "'Console logging'", "'Image-link diagnostics'"]) {
|
||||
assert.ok(SOURCE.includes(text), text);
|
||||
}
|
||||
for (const id of ['runtime-level', 'console-log']) assert.ok(body.includes(`setting("${id}"`) && SOURCE.includes(`id: '${id}'`), id);
|
||||
assert.ok(SOURCE.includes("'toolbar spaced-16'") && body.includes('class="toolbar spaced-16"'));
|
||||
// Prototype copy is never shipped.
|
||||
for (const sample of ['sample', 'Sample']) assert.ok(!SOURCE.includes(`'${sample}`) && !/Download sample|Inspect sample/.test(SOURCE), sample);
|
||||
});
|
||||
|
||||
test('D94 is retired: Diagnostics & logs is a delivered section and nothing hands off', () => {
|
||||
assert.ok(SYSTEM.includes('diagnostics: mountDiagnostics'));
|
||||
assert.ok(!SYSTEM.includes('SECTION_HANDOFFS'));
|
||||
assert.ok(!SYSTEM.includes('handoff('));
|
||||
assert.ok(!SYSTEM.includes('NOT_DELIVERED'));
|
||||
});
|
||||
|
||||
test('database text covers readable, not readable, busy and loading', async () => {
|
||||
const { databaseText, LOADING, UNREADABLE } = await diagnostics;
|
||||
assert.equal(databaseText({ report: REPORT }), 'Readable · 1.6 GB · 19,956 images');
|
||||
assert.equal(databaseText({ report: { ...REPORT, checks: { database: { status: 'ok', image_count: 1 } },
|
||||
database_info: { database_size_bytes: 2048 } } }), 'Readable · 2.0 KB · 1 image');
|
||||
assert.equal(databaseText({ report: { checks: { database: { status: 'error', message: 'Database error: locked' } } } }),
|
||||
'Not readable: Database error: locked');
|
||||
assert.equal(databaseText({ loading: true }), LOADING);
|
||||
assert.equal(LOADING, 'Loading…');
|
||||
assert.equal(databaseText({ error: { code: 'busy', running: { id: 'j1', title: 'Restore' } } }), 'Busy: Restore is running.');
|
||||
assert.equal(databaseText({ error: { code: 'busy' } }), 'Busy: another job is running.');
|
||||
assert.equal(databaseText({ error: new Error('offline') }), UNREADABLE);
|
||||
assert.equal(UNREADABLE, 'Could not be read.');
|
||||
});
|
||||
|
||||
test('sources text follows the precedence table', async () => {
|
||||
const { sourcesText } = await diagnostics;
|
||||
const of = sources => sourcesText({ report: { sources } });
|
||||
assert.equal(of({ configured: 0, effective: 1, watching: 1, unavailable: 0 }), 'ComfyUI output folder · watching');
|
||||
assert.equal(of({ configured: 0, effective: 1, watching: 0, unavailable: 0 }), 'ComfyUI output folder · manual scan');
|
||||
// Unavailable wins over watching.
|
||||
assert.equal(of({ configured: 3, effective: 2, watching: 2, unavailable: 1 }), '3 sources · 1 not available');
|
||||
assert.equal(of({ configured: 2, effective: 2, watching: 2, unavailable: 0 }), '2 sources · 2 watching');
|
||||
assert.equal(of({ configured: 1, effective: 1, watching: 0, unavailable: 0 }), '1 source · 0 watching');
|
||||
assert.equal(of(undefined), 'ComfyUI output folder · manual scan');
|
||||
});
|
||||
|
||||
test('Viewer.js reads the pinned vendor version', async () => {
|
||||
const { viewerText } = await diagnostics;
|
||||
assert.equal(viewerText({ report: REPORT }), '1.15.2 · pinned vendor');
|
||||
assert.equal(viewerText({ report: {} }), 'unknown · pinned vendor');
|
||||
});
|
||||
|
||||
test('report filename uses local time', async () => {
|
||||
const { reportFilename } = await diagnostics;
|
||||
assert.equal(reportFilename(new Date(2026, 9, 6, 7, 5)), 'prompt-manager-diagnostics-20261006-0705.json');
|
||||
assert.equal(reportFilename(new Date(2026, 11, 31, 23, 59)), 'prompt-manager-diagnostics-20261231-2359.json');
|
||||
});
|
||||
|
||||
test('merged report adds only host_renderers', async () => {
|
||||
const { mergeReport } = await diagnostics;
|
||||
const merged = mergeReport(REPORT, 'Detected: Classic (1 ComfyUI window).');
|
||||
assert.deepEqual(Object.keys(merged).sort(), [...Object.keys(REPORT), 'host_renderers'].sort());
|
||||
assert.equal(merged.host_renderers, 'Detected: Classic (1 ComfyUI window).');
|
||||
assert.equal(mergeReport(REPORT, null).host_renderers, null);
|
||||
assert.equal(mergeReport(REPORT, undefined).host_renderers, null);
|
||||
// The server's fields are not changed.
|
||||
assert.equal(merged.sources, REPORT.sources);
|
||||
});
|
||||
|
||||
test('sizes read as the server labels them (1 KB = 1024 bytes)', async () => {
|
||||
const { sizeText } = await diagnostics;
|
||||
assert.equal(sizeText(0), '0 bytes');
|
||||
assert.equal(sizeText(1), '1 byte');
|
||||
assert.equal(sizeText(1000), '1,000 bytes');
|
||||
assert.equal(sizeText(2048), '2.0 KB');
|
||||
assert.equal(sizeText(10 * 2 ** 20), '10.0 MB');
|
||||
assert.equal(sizeText(1717986918), '1.6 GB');
|
||||
assert.equal(sizeText(undefined), '0 bytes');
|
||||
});
|
||||
|
||||
test("log line uses the concept's format", async () => {
|
||||
const { logLine } = await diagnostics;
|
||||
assert.equal(logLine({ timestamp: '2026-10-26T09:42:18.123456', level: 'INFO', message: 'Database backup verified' }),
|
||||
'09:42:18 INFO Database backup verified');
|
||||
assert.equal(logLine({ timestamp: '2026-10-26T09:42:18', level: 'WARNING', message: '3 images have ambiguous prompt metadata' }),
|
||||
'09:42:18 WARNING 3 images have ambiguous prompt metadata');
|
||||
assert.match(logLine({ timestamp: '2026-10-26T09:42:18', level: 'ERROR', message: 'x' }), /^\d\d:\d\d:\d\d [A-Z]+ +\S/);
|
||||
// The concept's sample lines read the same way.
|
||||
assert.ok(CONCEPT.includes('09:42:18 INFO Scan completed'));
|
||||
});
|
||||
|
||||
test('Lines accepts whole numbers 1 to 500 only', async () => {
|
||||
const { validLimit, logsUrl } = await diagnostics;
|
||||
for (const [value, expected] of [['1', 1], ['500', 500], [' 20 ', 20], ['100', 100]]) assert.equal(validLimit(value), expected, value);
|
||||
for (const value of ['0', '501', '2.5', '', ' ', '-1', '1e2', 'abc', undefined]) assert.equal(validLimit(value), null, String(value));
|
||||
assert.equal(logsUrl('', 20), '/prompt_manager/logs?limit=20');
|
||||
assert.equal(logsUrl('WARNING', 20), '/prompt_manager/logs?limit=20&level=WARNING');
|
||||
});
|
||||
|
||||
test('the level filter offers all six minimum levels; the runtime level adds CRITICAL after the concept four', async () => {
|
||||
const { LEVEL_OPTIONS, RUNTIME_LEVELS } = await diagnostics;
|
||||
assert.deepEqual(LEVEL_OPTIONS.map(([, label]) => label), ['All levels', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']);
|
||||
assert.equal(LEVEL_OPTIONS[0][0], '');
|
||||
assert.deepEqual(RUNTIME_LEVELS, ['INFO', 'WARNING', 'DEBUG', 'ERROR', 'CRITICAL']);
|
||||
assert.ok(CONCEPT.includes('["INFO", "WARNING", "DEBUG", "ERROR"]'));
|
||||
});
|
||||
|
||||
test('statistics text uses singular forms for one', async () => {
|
||||
const { statsText } = await diagnostics;
|
||||
assert.equal(statsText({ buffer_count: 1000, level_counts: { INFO: 980, WARNING: 12, ERROR: 2, CRITICAL: 1 },
|
||||
log_files_count: 5, total_log_size: 43201024 }), '1,000 entries in memory · 3 errors · 12 warnings · 5 files · 41.2 MB on disk');
|
||||
assert.equal(statsText({ buffer_count: 1, level_counts: { WARNING: 1, CRITICAL: 1 }, log_files_count: 1, total_log_size: 1 }),
|
||||
'1 entry in memory · 1 error · 1 warning · 1 file · 1 byte on disk');
|
||||
assert.equal(statsText({}), '0 entries in memory · 0 errors · 0 warnings · 0 files · 0 bytes on disk');
|
||||
});
|
||||
|
||||
test('retention, file line and file rows read the server values', async () => {
|
||||
const { retentionText, fileLine, fileDetail } = await diagnostics;
|
||||
assert.equal(retentionText({ max_file_size: 10485760, backup_count: 5, file_logging: true, buffer_size: 1000 }),
|
||||
'Rotates at 10 MB · keeps 5 rotated copies · file logging on · memory buffer 1,000 entries');
|
||||
assert.equal(retentionText({ max_file_size: 1572864, backup_count: 1, file_logging: false, buffer_size: 1 }),
|
||||
'Rotates at 1.5 MB · keeps 1 rotated copy · file logging off · memory buffer 1 entry');
|
||||
const files = [
|
||||
{ filename: 'prompt_manager.log', kind: 'active', size: 1468006, modified: '2026-10-26T09:42:18' },
|
||||
{ filename: 'prompt_manager.log.1', kind: 'rotated', size: 10485760, modified: '2026-10-25T14:02:00' },
|
||||
{ filename: 'prompt_manager.log.4', kind: 'link', size: 20, modified: '2026-10-25T14:02:00' },
|
||||
{ filename: 'prompt_manager.log.bak', kind: 'other', size: 0, modified: '2026-10-24T08:00:00' },
|
||||
];
|
||||
assert.equal(fileLine(files), 'prompt_manager.log · 1.4 MB · rotated copies: 1');
|
||||
assert.equal(fileLine([]), 'prompt_manager.log · not written yet · rotated copies: 0');
|
||||
assert.equal(fileDetail(files[1]), '10.0 MB · 2026-10-25 14:02 · Rotated copy');
|
||||
assert.equal(fileDetail(files[2]), '20 bytes · 2026-10-25 14:02 · Link, not downloaded');
|
||||
assert.equal(fileDetail(files[3]), '0 bytes · 2026-10-24 08:00 · Other file');
|
||||
assert.equal(fileDetail(files[0]), '1.4 MB · 2026-10-26 09:42 · Active log');
|
||||
});
|
||||
|
||||
test('Clear all logs states both counts and its toast says what was done', async () => {
|
||||
const { clearSentence, clearToast } = await diagnostics;
|
||||
const rotated = size => ({ kind: 'rotated', size });
|
||||
const files = [{ kind: 'active', size: 10 }, rotated(10 * 2 ** 20), rotated(10 * 2 ** 20), rotated(8.4 * 2 ** 20), { kind: 'other', size: 5 }];
|
||||
assert.equal(clearSentence(files), 'Empties prompt_manager.log, deletes 3 rotated copies (28.4 MB) and clears the lines shown '
|
||||
+ 'here. The other files in the log folder are kept. This cannot be undone.');
|
||||
assert.equal(clearSentence([{ kind: 'active', size: 1 }]), 'Empties prompt_manager.log and clears the lines shown here. '
|
||||
+ 'The other files in the log folder are kept. This cannot be undone.');
|
||||
assert.match(clearSentence(null), /^Empties prompt_manager.log, deletes its rotated copies and clears/);
|
||||
assert.equal(clearToast({ truncated: ['prompt_manager.log', 'prompt_manager.log.1', 'prompt_manager.log.2', 'prompt_manager.log.3'],
|
||||
errors: [] }), 'Logs cleared · 3 rotated copies deleted.');
|
||||
assert.equal(clearToast({ truncated: ['prompt_manager.log', 'prompt_manager.log.1', 'prompt_manager.log.2'],
|
||||
errors: ['Failed to truncate prompt_manager.log.3: in use'] }), 'Logs cleared · 2 rotated copies deleted · 1 could not be deleted.');
|
||||
assert.equal(clearToast({ truncated: ['prompt_manager.log', 'prompt_manager.log.1'], errors: [] }), 'Logs cleared · 1 rotated copy deleted.');
|
||||
assert.equal(clearToast(undefined), 'Logs cleared · 0 rotated copies deleted.');
|
||||
});
|
||||
|
||||
test('the test-link dialog names the prompt and how Missing-file links removes the record', async () => {
|
||||
const { testLinkSentence, promptName } = await diagnostics;
|
||||
assert.equal(testLinkSentence({ id: 812, text: 'Misty harbor' }), 'Writes a placeholder image record '
|
||||
+ '(prompt_manager_test_link.png) linked to prompt #812 “Misty harbor”. Maintenance › Missing-file links removes it later.');
|
||||
assert.equal(promptName({ text: ' a\n b ' }), 'a b');
|
||||
assert.equal(promptName({ text: 'x'.repeat(80) }).length, 60);
|
||||
});
|
||||
|
||||
test('runtime values and body map config.level to runtime_level and back', async () => {
|
||||
const { runtimeValues, runtimeBody } = await diagnostics;
|
||||
assert.deepEqual(runtimeValues({ level: 'INFO', console_logging: true, max_file_size: 1 }), { runtime_level: 'INFO', console_logging: true });
|
||||
assert.deepEqual(runtimeBody({ runtime_level: 'CRITICAL' }), { level: 'CRITICAL' });
|
||||
assert.deepEqual(runtimeBody({ console_logging: false }), { console_logging: false });
|
||||
assert.deepEqual(runtimeBody({ runtime_level: 'DEBUG', console_logging: true }), { level: 'DEBUG', console_logging: true });
|
||||
assert.deepEqual(runtimeBody({}), {});
|
||||
});
|
||||
@@ -169,44 +169,3 @@ test('a plan with nothing to do offers no Apply (S7 live test)', async () => {
|
||||
assert.equal(hasWork({ ...PLAN, scope: {} }), true);
|
||||
assert.equal(NOTHING_TO_APPLY, 'There is nothing to apply; nothing was changed.');
|
||||
});
|
||||
|
||||
// S12 (#329): the rotated-log cleanup plan changes no library record: its Backup reads "Not needed", the review shows no
|
||||
// D55 backup box, and its details are files.
|
||||
const LOG_PLAN = { operation: 'rotated_logs', token: 't', expires_at: '2026-10-26T23:15:00',
|
||||
scope: { count: 3, label: '3 rotated log copies · 28.4 MB' },
|
||||
protected: { count: 3, reasons: ['Kept: the active log', 'Kept: 1 link', 'Kept: 1 other file'] },
|
||||
backup: { policy: 'none', destination_label: null, expected_bytes: null },
|
||||
media: { deletes_files: true, label: 'Deletes only the listed log files. No image, video or library record changes.' },
|
||||
details: [{ label: 'prompt_manager.log.1', detail: '10.0 MB · 2026-10-25 14:02' }] };
|
||||
|
||||
test('a plan without a database change reads Not needed and shows no backup box', async () => {
|
||||
const { planGrid, backupText, backupBox, BACKUP_COPY, OPERATION_TITLES } = await review;
|
||||
assert.equal(backupText({ policy: 'none' }), 'Not needed: no library record changes');
|
||||
assert.deepEqual(planGrid(LOG_PLAN), [
|
||||
['Scope', '3 rotated log copies · 28.4 MB'],
|
||||
['Protected', 'Kept: the active log · Kept: 1 link · Kept: 1 other file'],
|
||||
['Backup', 'Not needed: no library record changes'],
|
||||
['Media on disk', 'Deletes only the listed log files. No image, video or library record changes.'],
|
||||
]);
|
||||
const onBackup = () => {};
|
||||
assert.equal(backupBox(LOG_PLAN, onBackup), null);
|
||||
// Every database plan keeps its D55 box.
|
||||
assert.deepEqual(backupBox({ ...LOG_PLAN, operation: 'optimize', backup: { policy: 'required' } }, onBackup), { copy: BACKUP_COPY, onBackup });
|
||||
assert.deepEqual(backupBox({ operation: 'missing_links' }, onBackup), { copy: BACKUP_COPY, onBackup });
|
||||
// Its title heads the dialog and lets Activity offer Review again.
|
||||
assert.equal(OPERATION_TITLES.rotated_logs, 'Rotated log copies');
|
||||
});
|
||||
|
||||
test('rotated_logs details read Listed files', async () => {
|
||||
const { detailsSummary, detailLines } = await review;
|
||||
assert.equal(detailsSummary(LOG_PLAN, 3), 'Listed files (3)');
|
||||
assert.equal(detailsSummary({ ...LOG_PLAN, scope: { count: 60 } }, 50), 'Listed files (50 of 60)');
|
||||
assert.equal(detailsSummary({ operation: 'missing_links', scope: { count: 2 } }, 2), 'Listed records (2)');
|
||||
assert.deepEqual(detailLines(LOG_PLAN), ['prompt_manager.log.1 · 10.0 MB · 2026-10-25 14:02']);
|
||||
});
|
||||
|
||||
test('a rotated-log cleanup the log overtook offers Review again (S12)', async () => {
|
||||
const { reviewAgainOf } = await jobView;
|
||||
assert.deepEqual(reviewAgainOf({ kind: 'maintenance', state: 'failed', scope: { operation: 'rotated_logs', failure: 'plan-changed' } }),
|
||||
{ operation: 'rotated_logs' });
|
||||
});
|
||||
|
||||
@@ -420,12 +420,3 @@ test('a partial host save keeps what saved, adopts its revision and fails only t
|
||||
assert.equal((await drafts.save()).host, 'failed');
|
||||
assert.equal(drafts.statusText(), SETTINGS_STATUS.partial);
|
||||
});
|
||||
|
||||
// S12 (#329): System's runtime domain is the running server's log level and console setting (`/logs/config`).
|
||||
test('runtime adapter maps config.level to runtime_level and back', async () => {
|
||||
const { runtimeValues, runtimeBody } = await import('../../web/js/workspace/system-diagnostics.mjs');
|
||||
assert.deepEqual(runtimeValues({ level: 'WARNING', console_logging: false, backup_count: 5 }),
|
||||
{ runtime_level: 'WARNING', console_logging: false });
|
||||
assert.deepEqual(runtimeBody({ runtime_level: 'WARNING' }), { level: 'WARNING' });
|
||||
assert.deepEqual(runtimeBody({ runtime_level: 'CRITICAL', console_logging: true }), { level: 'CRITICAL', console_logging: true });
|
||||
});
|
||||
|
||||
@@ -360,23 +360,19 @@ test('Preview tagging review opens the Tag image dialog with the section’s val
|
||||
} finally { scope.dispose(); dom.restore(); }
|
||||
});
|
||||
|
||||
const PLAN = { images: 3, already_tagged: 1, to_tag: 2, estimate_seconds: 3.2, estimate_device: 'cpu', suggestion_limit: 10,
|
||||
const PLAN = { images: 3, prompts: 2, already_tagged: 1, to_tag: 1,
|
||||
model: { type: 'wd14-swinv2', label: 'WD14 SwinV2', size_label: '~400 MB', downloaded: true }, existing: 'skip',
|
||||
review_required: false, backup: { policy: 'on' }, disk: { free_bytes: 10 * 2 ** 30, required_bytes: 2 ** 20, sufficient: true } };
|
||||
|
||||
test('the batch review grid, body and Cancel-only reasons (D134, #379, CUJ 4–5)', async () => {
|
||||
const { batchGrid, batchBody, batchCloseNote, existingLabel, REPLACE_NOTE, NOTHING_TO_TAG, NO_IMAGES } = await tagging;
|
||||
test('the batch review grid, body and Cancel-only reasons (D134, CUJ 4–5)', async () => {
|
||||
const { batchGrid, batchBody, batchCloseNote, existingLabel, REPLACE_NOTE, NOTHING_TO_TAG, NO_LINKED_IMAGES } = await tagging;
|
||||
assert.deepEqual(batchGrid(PLAN, HOST), [['Model', 'WD14 SwinV2 · ~400 MB'], ['Existing tags', 'Skip already tagged'],
|
||||
['Images', '3'], ['Already auto-tagged', '1'], ['Images to tag', 'about 2'],
|
||||
['Time', 'about 1 min on a processor; a graphics card can be faster.'],
|
||||
['Settings', 'General 0.35 · Character 0.85 · up to 10 tags per image'], ['Backup', 'Create and verify before changes']]);
|
||||
['Images linked', '3'], ['Prompts', '2'], ['Already tagged', '1'], ['Prompts to tag', 'up to 1'],
|
||||
['Settings', 'General 0.35 · Character 0.85'], ['Backup', 'Create and verify before changes']]);
|
||||
const llm = { ...HOST, autotag_model_type: 'gguf' };
|
||||
assert.deepEqual(batchGrid({ ...PLAN, model: { type: 'gguf' }, backup: { policy: 'off' }, estimate_seconds: null,
|
||||
estimate_device: null, suggestion_limit: 4 }, llm).slice(-3),
|
||||
[['Time', 'Language-model engines take longer per image. Activity shows the time left once tagging starts.'],
|
||||
['Settings', 'Template of 27 characters · up to 4 tags per image'], ['Backup', 'No safety backup (the policy is off)']]);
|
||||
assert.deepEqual(batchGrid({ ...PLAN, model: { type: 'gguf' }, backup: { policy: 'off' } }, llm).slice(-2),
|
||||
[['Settings', 'Template of 27 characters'], ['Backup', 'No safety backup (the policy is off)']]);
|
||||
assert.equal(batchGrid({}, HOST)[2][1], '0');
|
||||
assert.equal(batchGrid({}, HOST)[6][1], 'General 0.35 · Character 0.85');
|
||||
assert.equal(existingLabel('odd'), 'odd');
|
||||
assert.deepEqual(batchBody(HOST), { model_type: 'wd14-swinv2', existing: 'skip',
|
||||
options: { general_threshold: 0.35, character_threshold: 0.85 } });
|
||||
@@ -389,31 +385,9 @@ test('the batch review grid, body and Cancel-only reasons (D134, #379, CUJ 4–5
|
||||
assert.equal(batchCloseNote({ ...PLAN, model: { ...PLAN.model, downloaded: false } }), 'WD14 SwinV2 is not downloaded. Download it first.');
|
||||
assert.equal(batchCloseNote({ ...PLAN, disk: { free_bytes: 2 ** 20, required_bytes: 2 ** 21, sufficient: false } }),
|
||||
'Not enough free disk space: the safety backup needs about 2.0 MB and 1.0 MB is free. Nothing is written until there is room.');
|
||||
assert.equal(batchCloseNote({ ...PLAN, images: 0, to_tag: 0 }), NO_IMAGES);
|
||||
assert.equal(batchCloseNote({ ...PLAN, images: undefined }), NO_IMAGES);
|
||||
assert.equal(NO_IMAGES, 'The output folders contain no images to tag.');
|
||||
assert.equal(batchCloseNote({ ...PLAN, prompts: 0, to_tag: 0 }), NO_LINKED_IMAGES);
|
||||
assert.equal(batchCloseNote({ ...PLAN, to_tag: 0 }), NOTHING_TO_TAG);
|
||||
assert.equal(NOTHING_TO_TAG, 'Every output image is already auto-tagged.');
|
||||
});
|
||||
|
||||
test('the review Time row (DEC11, #379): minutes, hours and the engine it runs on', async () => {
|
||||
const { durationText, timeText, TIME_UNKNOWN } = await tagging;
|
||||
assert.equal(durationText(1), 'about 1 min');
|
||||
assert.equal(durationText(0), 'about 1 min');
|
||||
assert.equal(durationText(89), 'about 89 min');
|
||||
assert.equal(durationText(90), 'about 1 h 30 min');
|
||||
assert.equal(durationText(120), 'about 2 h');
|
||||
assert.equal(durationText(631), 'about 10 h 31 min');
|
||||
assert.equal(durationText(Number.NaN), 'about 1 min');
|
||||
// 23,645 images at 1.6 s on a processor (the design's CUJ 1).
|
||||
assert.equal(timeText({ estimate_seconds: 37832, estimate_device: 'cpu' }),
|
||||
'about 10 h 31 min on a processor; a graphics card can be faster.');
|
||||
assert.equal(timeText({ estimate_seconds: 59, estimate_device: 'gpu' }), 'about 1 min on the graphics card.');
|
||||
assert.equal(timeText({ estimate_seconds: 0, estimate_device: 'cpu' }), 'about 1 min on a processor; a graphics card can be faster.');
|
||||
for (const plan of [{ estimate_seconds: null }, {}, { estimate_seconds: -1 }, { estimate_seconds: Infinity }, null]) {
|
||||
assert.equal(timeText(plan), TIME_UNKNOWN);
|
||||
}
|
||||
assert.equal(TIME_UNKNOWN, 'Language-model engines take longer per image. Activity shows the time left once tagging starts.');
|
||||
assert.equal(NOTHING_TO_TAG, 'Every prompt with an output image is already tagged.');
|
||||
});
|
||||
|
||||
test('showBatchPlan opens the review with the backup box, or the Cancel-only review', async () => {
|
||||
|
||||
@@ -26,8 +26,6 @@ VECTORS_PATH = (
|
||||
/ "local-inspect"
|
||||
/ "resolver-vectors.json"
|
||||
)
|
||||
# A case may name the inspected file (#381 saver anchoring); it is no record.
|
||||
FILE_NAME = "__file_name"
|
||||
# Committed record sets stay small; the large bound cases are built at test time.
|
||||
MAX_VECTOR_CHARS = 20_000
|
||||
COMMITTED_FUZZ = (0, 60)
|
||||
@@ -908,60 +906,6 @@ def golden_cases():
|
||||
},
|
||||
),
|
||||
)
|
||||
# #381: unknown nodes between the prompt and the sampler pass through.
|
||||
try:
|
||||
from tests import unknown_node_graphs
|
||||
except ImportError: # pragma: no cover - discover -s tests puts tests/ first
|
||||
import unknown_node_graphs
|
||||
for name, graph in unknown_node_graphs.shapes().items():
|
||||
add(f"pass-through-{name}", graph)
|
||||
for name, records in {
|
||||
**unknown_node_graphs.recorded_shapes(),
|
||||
**unknown_node_graphs.review_shapes(),
|
||||
}.items():
|
||||
cases[f"pass-through-{name}"] = records
|
||||
try:
|
||||
from tests import saver_anchor_graphs as anchor
|
||||
except ImportError: # pragma: no cover - discover -s tests puts tests/ first
|
||||
import saver_anchor_graphs as anchor
|
||||
for name, graph, file_name in [
|
||||
("branch-3", anchor.branch_bench(), "s45_branch_3_00002_.png"),
|
||||
("branch-unanchored", anchor.branch_bench(), None),
|
||||
("branch-no-match", anchor.branch_bench(), "photo_00001_.png"),
|
||||
("branch-linked", anchor.linked_prefix_bench(), "s45_branch_2_00007_.png"),
|
||||
(
|
||||
"branch-unknown-saver",
|
||||
anchor.branch_bench(saver="VendorSaveImage"),
|
||||
"s45_branch_4_00001.webp",
|
||||
),
|
||||
("shared-prefix", anchor.shared_prefix(), "IMG_GEN_00371_.png"),
|
||||
("same-links", anchor.two_stage_same_conditioning(), "final_00003_.png"),
|
||||
("same-links-unanchored", anchor.two_stage_same_conditioning(), None),
|
||||
(
|
||||
"other-links",
|
||||
anchor.two_stage_same_conditioning(["91", 0]),
|
||||
"final_00003_.png",
|
||||
),
|
||||
("hidden-earlier", anchor.hidden_earlier_prompt(), "out_00001_.png"),
|
||||
(
|
||||
"hidden-override",
|
||||
anchor.hidden_earlier_prompt(override=True),
|
||||
"out_00001_.png",
|
||||
),
|
||||
] + [
|
||||
(f"first-stage-{kind}", anchor.first_stage_variant(kind), "final_00003_.png")
|
||||
for kind in (
|
||||
"no-latent",
|
||||
"literal-latent",
|
||||
"missing-latent",
|
||||
"cyclic-latent",
|
||||
"stage-shaped",
|
||||
"detailer",
|
||||
)
|
||||
]:
|
||||
cases[f"anchor-{name}"] = {"prompt": _dump(graph)}
|
||||
if file_name is not None:
|
||||
cases[f"anchor-{name}"][FILE_NAME] = file_name
|
||||
cases["parameters-only"] = {
|
||||
"parameters": "a fox\nNegative prompt: blur\nSteps: 20, Sampler: euler"
|
||||
}
|
||||
@@ -1115,8 +1059,7 @@ def boundary_cases():
|
||||
["7", 0],
|
||||
{
|
||||
"\U0001f600"
|
||||
# No text and no links: still named, not passed through (#381).
|
||||
* 70: {"class_type": long_name, "inputs": {"seed": 1}},
|
||||
* 70: {"class_type": long_name, "inputs": {"text": "x"}},
|
||||
"7": _encode("b"),
|
||||
},
|
||||
)
|
||||
@@ -1444,10 +1387,7 @@ def committed_vectors():
|
||||
"name": name,
|
||||
"records": records,
|
||||
"evidence": inspect_prompt_evidence(
|
||||
{key: value for key, value in records.items() if key != FILE_NAME},
|
||||
None,
|
||||
profile=INSPECTION_PROFILE,
|
||||
file_name=records.get(FILE_NAME),
|
||||
records, None, profile=INSPECTION_PROFILE
|
||||
),
|
||||
}
|
||||
for name, records in cases.items()
|
||||
|
||||
@@ -1,216 +0,0 @@
|
||||
"""Synthetic graphs for saver anchoring and shared-conditioning stages (#381).
|
||||
|
||||
``branch_bench`` reproduces the maintainer's ``gpu_unlock_bench`` run: four
|
||||
KSampler branches with different prompts, each saved by its own SaveImage whose
|
||||
literal prefix names the branch (``bench/run/s45_branch_N``). ``shared_prefix``
|
||||
is the dataset run's shape: every branch's saver uses one prefix, so the file
|
||||
name cannot name the branch. ``two_stage_same_conditioning`` reproduces the
|
||||
ANIMA run: a hires stage whose latent passes an unlisted node and reads the
|
||||
same positive and negative links as the first stage. Texts are neutral.
|
||||
"""
|
||||
|
||||
BRANCH_TEXTS = (
|
||||
"a knit beanie on a wooden table",
|
||||
"an old fishing boat at dawn",
|
||||
"a red vintage car in the rain",
|
||||
"a cozy library with tall shelves",
|
||||
)
|
||||
NEGATIVE = "blurry, lowres"
|
||||
|
||||
|
||||
def _encode(text):
|
||||
return {"class_type": "CLIPTextEncode", "inputs": {"text": text, "clip": ["1", 1]}}
|
||||
|
||||
|
||||
def _ksampler(positive, negative, latent, denoise=1.0):
|
||||
return {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"seed": 1,
|
||||
"steps": 20,
|
||||
"cfg": 7.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "normal",
|
||||
"denoise": denoise,
|
||||
"model": ["1", 0],
|
||||
"positive": positive,
|
||||
"negative": negative,
|
||||
"latent_image": latent,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _base():
|
||||
return {
|
||||
"1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "m"}},
|
||||
"2": {"class_type": "EmptyLatentImage", "inputs": {"width": 64}},
|
||||
"3": _encode(NEGATIVE),
|
||||
}
|
||||
|
||||
|
||||
def branch_bench(prefixes=None, saver="SaveImage"):
|
||||
"""Four branches; ``prefixes`` replaces each saver's ``filename_prefix``."""
|
||||
prefixes = prefixes or [
|
||||
f"bench/run/s45_branch_{index}" for index in range(1, len(BRANCH_TEXTS) + 1)
|
||||
]
|
||||
graph = _base()
|
||||
for index, text in enumerate(BRANCH_TEXTS, start=1):
|
||||
base = index * 10
|
||||
graph[str(base)] = _encode(text)
|
||||
graph[str(base + 1)] = _ksampler([str(base), 0], ["3", 0], ["2", 0])
|
||||
graph[str(base + 2)] = {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": [str(base + 1), 0], "vae": ["1", 2]},
|
||||
}
|
||||
graph[str(base + 3)] = {
|
||||
"class_type": saver,
|
||||
"inputs": {
|
||||
"filename_prefix": prefixes[index - 1],
|
||||
"images": [str(base + 2), 0],
|
||||
},
|
||||
}
|
||||
return graph
|
||||
|
||||
|
||||
def linked_prefix_bench():
|
||||
"""The branch run with each prefix coming from a PrimitiveString."""
|
||||
graph = branch_bench(prefixes=[["90", 0], ["91", 0], ["92", 0], ["93", 0]])
|
||||
for index in range(4):
|
||||
graph[str(90 + index)] = {
|
||||
"class_type": "PrimitiveString",
|
||||
"inputs": {"value": f"bench/run/s45_branch_{index + 1}"},
|
||||
}
|
||||
return graph
|
||||
|
||||
|
||||
def shared_prefix():
|
||||
"""Every branch saved with one prefix: the file name names no branch."""
|
||||
return branch_bench(prefixes=["dataset/IMG_GEN"] * len(BRANCH_TEXTS))
|
||||
|
||||
|
||||
def two_stage_same_conditioning(second_positive=None):
|
||||
"""Base and hires stage on the same links; the hires latent passes an
|
||||
unlisted node (``VendorGpuCleaner``), so the stages' lineage is unknown."""
|
||||
graph = _base()
|
||||
graph["90"] = _encode(BRANCH_TEXTS[0])
|
||||
graph["63"] = _ksampler(["90", 0], ["3", 0], ["2", 0])
|
||||
graph["30"] = {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["63", 0], "vae": ["1", 2]},
|
||||
}
|
||||
graph["98"] = {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"filename_prefix": "draft", "images": ["30", 0]},
|
||||
}
|
||||
graph["110"] = {"class_type": "VendorGpuCleaner", "inputs": {"anything": ["63", 0]}}
|
||||
graph["108"] = {
|
||||
"class_type": "LatentUpscaleBy",
|
||||
"inputs": {
|
||||
"upscale_method": "nearest-exact",
|
||||
"scale_by": 1.2,
|
||||
"samples": ["110", 0],
|
||||
},
|
||||
}
|
||||
graph["114"] = _ksampler(second_positive or ["90", 0], ["3", 0], ["108", 0], 0.5)
|
||||
graph["113"] = {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["114", 0], "vae": ["1", 2]},
|
||||
}
|
||||
graph["115"] = {
|
||||
"class_type": "VendorGpuCleaner",
|
||||
"inputs": {"anything": ["113", 0]},
|
||||
}
|
||||
graph["107"] = {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"filename_prefix": "final", "images": ["115", 0]},
|
||||
}
|
||||
if second_positive is not None:
|
||||
graph["91"] = _encode(BRANCH_TEXTS[1])
|
||||
return graph
|
||||
|
||||
|
||||
def hidden_earlier_prompt(override=False):
|
||||
"""Review shape: the skipped same-links stage continues a different prompt.
|
||||
|
||||
10 (cat) -> 11 (dog) -> unlisted LatentBlend -> 13 (dog, same links as 11)
|
||||
-> VAEDecode -> SaveImage ``out``. With ``override``, 11 is a
|
||||
``KSampler (Efficient)`` with a linked ``script``.
|
||||
"""
|
||||
graph = _base()
|
||||
graph["5"] = _encode("cat X")
|
||||
graph["6"] = _encode("dog Y")
|
||||
graph["10"] = _ksampler(["5", 0], ["3", 0], ["2", 0])
|
||||
graph["11"] = _ksampler(["6", 0], ["3", 0], ["10", 0])
|
||||
if override:
|
||||
graph["11"] = _ksampler(["6", 0], ["3", 0], ["2", 0])
|
||||
graph["11"]["class_type"] = "KSampler (Efficient)"
|
||||
graph["11"]["inputs"]["script"] = ["40", 0]
|
||||
graph["40"] = {"class_type": "XY Plot", "inputs": {"grid_spacing": 0}}
|
||||
graph["12"] = {
|
||||
"class_type": "LatentBlend",
|
||||
"inputs": {"samples1": ["11", 0], "samples2": ["2", 0], "blend_factor": 0.5},
|
||||
}
|
||||
graph["13"] = _ksampler(["6", 0], ["3", 0], ["12", 0], 0.5)
|
||||
graph["14"] = {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["13", 0], "vae": ["1", 2]},
|
||||
}
|
||||
graph["15"] = {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"filename_prefix": "out", "images": ["14", 0]},
|
||||
}
|
||||
return graph
|
||||
|
||||
|
||||
def deep_prefixes(chain):
|
||||
"""Review shape: two branches whose savers both trace the prefix ``dup``.
|
||||
|
||||
Saver A's prefix runs through ``chain`` DisplayText nodes; saver B's
|
||||
through one more, so a short visit allowance can finish only one trace.
|
||||
"""
|
||||
graph = branch_bench(prefixes=[["p0", 0], ["q", 0], "x_1", "x_2"])
|
||||
graph["pz"] = {"class_type": "StringConstant", "inputs": {"string": "dup"}}
|
||||
previous = "pz"
|
||||
for index in range(chain - 1, -1, -1):
|
||||
graph[f"p{index}"] = {
|
||||
"class_type": "DisplayText",
|
||||
"inputs": {"text": [previous, 0]},
|
||||
}
|
||||
previous = f"p{index}"
|
||||
graph["q"] = {"class_type": "DisplayText", "inputs": {"text": ["p0", 0]}}
|
||||
return graph
|
||||
|
||||
|
||||
def first_stage_variant(kind):
|
||||
"""``two_stage_same_conditioning`` whose first stage (63) is not plain.
|
||||
|
||||
``no-latent`` drops its latent input (still plain); ``literal-latent``,
|
||||
``missing-latent``, ``cyclic-latent``, ``stage-shaped`` and ``detailer``
|
||||
each make it something other than a plain first stage, so the later
|
||||
stage stays unattributed.
|
||||
"""
|
||||
graph = two_stage_same_conditioning()
|
||||
inputs = graph["63"]["inputs"]
|
||||
if kind == "no-latent":
|
||||
del inputs["latent_image"]
|
||||
elif kind == "literal-latent":
|
||||
inputs["latent_image"] = "latent"
|
||||
elif kind == "missing-latent":
|
||||
inputs["latent_image"] = ["999", 0]
|
||||
elif kind == "cyclic-latent":
|
||||
graph["70"] = {"class_type": "VendorLatentOp", "inputs": {"samples": ["63", 0]}}
|
||||
inputs["latent_image"] = ["70", 0]
|
||||
elif kind == "stage-shaped":
|
||||
graph["70"] = {
|
||||
"class_type": "VendorSampler",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"positive": ["90", 0],
|
||||
"latent_image": ["2", 0],
|
||||
},
|
||||
}
|
||||
inputs["latent_image"] = ["70", 0]
|
||||
elif kind == "detailer":
|
||||
graph["63"]["class_type"] = "FaceDetailer"
|
||||
inputs["image"] = ["30", 0]
|
||||
return graph
|
||||
@@ -35,8 +35,6 @@ REVIEW_TABLES = (
|
||||
"media_review_items",
|
||||
"media_review_metadata",
|
||||
)
|
||||
# The auto-tag marks (#379) cite annotation bytes too; an old backup lacks them.
|
||||
CITING_TABLES = REVIEW_TABLES + ("media_image_autotags",)
|
||||
# The exact DDL the plan pins; the module must produce precisely these objects.
|
||||
DDL = {
|
||||
METADATA: (
|
||||
@@ -216,7 +214,7 @@ class AnnotationSchemaTests(unittest.TestCase):
|
||||
"""
|
||||
with closing(sqlite3.connect(path)) as raw, raw:
|
||||
raw.execute("PRAGMA foreign_keys=OFF")
|
||||
for table in CITING_TABLES + tuple(reversed(TABLES)):
|
||||
for table in REVIEW_TABLES + tuple(reversed(TABLES)):
|
||||
raw.execute(f"DROP TABLE {table}")
|
||||
|
||||
def test_fresh_install_and_reopen_is_idempotent(self):
|
||||
|
||||
+7
-157
@@ -430,9 +430,7 @@ class ReplaceImageTagsTests(AnnotationTestCase):
|
||||
self.assertTrue(
|
||||
self.db.refresh_image_fingerprint(loc, DIGEST_A, SIZE, MTIME + 1)
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.image_tag_locators(self.db.image_tag_ids(["img"], "")), {loc}
|
||||
)
|
||||
list(self.db.iter_image_tag_records())
|
||||
self.assertEqual(self.db.get_prompt_by_id(prompt)["tags"], ["prompt-tag"])
|
||||
|
||||
|
||||
@@ -553,162 +551,14 @@ class CurrentImageTagsTests(AnnotationTestCase):
|
||||
self.assertEqual(current, {locators[0]: ["z", "y"], locators[502]: ["x"]})
|
||||
lookups = [s for s in statements if "media_annotation_files" in s]
|
||||
self.assertEqual(len(lookups), 2)
|
||||
# The tag search finds every record's file by tag id, and the empty
|
||||
# set's file never.
|
||||
self.assertEqual(
|
||||
self.db.image_tag_locators(self.db.image_tag_ids(["z", "x"], "")),
|
||||
{locators[0], locators[502]},
|
||||
list(self.db.iter_image_tag_records()),
|
||||
[
|
||||
(locators[0], SIZE, MTIME, ["z", "y"]),
|
||||
(locators[250], SIZE, MTIME + 1, []),
|
||||
(locators[502], SIZE, MTIME + 2, ["x"]),
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags({locators[250]: (SIZE, MTIME + 1)}),
|
||||
{locators[250]: []},
|
||||
)
|
||||
|
||||
def test_tag_locators_cover_more_ids_than_one_chunk(self):
|
||||
# A broad text search can match more than 500 names: every id's file
|
||||
# comes back across the chunks.
|
||||
locators = [self.locator(f"t{i:04}.png") for i in range(503)]
|
||||
for i, loc in enumerate(locators):
|
||||
self.replace(loc, tags=[f"name_{i:04}"])
|
||||
ids = self.db.image_tag_ids([], "name_")
|
||||
self.assertEqual(len(ids), 503)
|
||||
self.assertEqual(self.db.image_tag_locators(ids), set(locators))
|
||||
|
||||
|
||||
class AutotagMarkTests(AnnotationTestCase):
|
||||
"""The auto-tagged mark (#379): per byte record, written with the tags."""
|
||||
|
||||
def marked(self):
|
||||
conn = self.db.model.get_connection()
|
||||
return sorted(
|
||||
row[0] for row in conn.execute("SELECT * FROM media_image_autotags")
|
||||
)
|
||||
|
||||
def byte_id(self, loc, digest=DIGEST_A):
|
||||
conn = self.db.model.get_connection()
|
||||
return conn.execute(
|
||||
"SELECT b.id FROM media_annotation_bytes b JOIN media_annotation_files f "
|
||||
"ON f.id=b.file_id WHERE f.locator=? AND b.sha256=?",
|
||||
(loc, digest),
|
||||
).fetchone()[0]
|
||||
|
||||
def test_hand_tags_never_mark(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=["keeper"])
|
||||
self.add(loc, tags=["more"])
|
||||
self.assertEqual(self.marked(), [])
|
||||
self.assertEqual(self.db.autotag_state({loc: (SIZE, MTIME)}), {loc: False})
|
||||
self.assertEqual(self.db.autotagged_locators(), set())
|
||||
|
||||
def test_model_union_marks_including_replayed_and_empty_results(self):
|
||||
loc = self.locator("a.png")
|
||||
receipt = self.add(loc, tags=["bird"], mark_autotagged=True)
|
||||
self.assertEqual((receipt["added"], receipt["replayed"]), (1, False))
|
||||
self.assertEqual(self.marked(), [self.byte_id(loc)])
|
||||
# A replayed union and an empty result still mark (idempotently).
|
||||
other = self.locator("b.png")
|
||||
self.replace(other, DIGEST_B, ["keeper"])
|
||||
replayed = self.add(other, DIGEST_B, ["KEEPER"], mark_autotagged=True)
|
||||
self.assertTrue(replayed["replayed"])
|
||||
empty = self.locator("c.png")
|
||||
nothing = self.add(empty, tags=[], mark_autotagged=True)
|
||||
self.assertEqual((nothing["tags"], nothing["added"]), ([], 0))
|
||||
self.add(empty, tags=[], mark_autotagged=True)
|
||||
self.assertEqual(
|
||||
self.marked(),
|
||||
sorted(
|
||||
[self.byte_id(loc), self.byte_id(other, DIGEST_B), self.byte_id(empty)]
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.autotag_state(
|
||||
{loc: (SIZE, MTIME), other: (SIZE, MTIME), empty: (SIZE, MTIME)}
|
||||
),
|
||||
{loc: True, other: True, empty: True},
|
||||
)
|
||||
self.schema_clean()
|
||||
|
||||
def test_empty_union_without_mark_is_refused(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.add(self.locator("a.png"), tags=[])
|
||||
self.assertEqual(self.counts()["media_annotation_files"], 0)
|
||||
|
||||
def test_tag_limit_union_writes_no_tag_and_no_mark(self):
|
||||
loc = self.locator("a.png")
|
||||
self.replace(loc, tags=[f"t{i}" for i in range(100)])
|
||||
before = self.rows(*ANNOTATION_TABLES, "tags")
|
||||
with self.assertRaises(AnnotationCommandError) as ctx:
|
||||
self.add(loc, tags=["one more"], mark_autotagged=True)
|
||||
self.assertEqual(ctx.exception.code, "tag-limit")
|
||||
self.assertEqual(self.rows(*ANNOTATION_TABLES, "tags"), before)
|
||||
self.assertEqual(self.marked(), [])
|
||||
|
||||
def test_replace_marks_on_saved_and_replayed_receipts_not_on_conflict(self):
|
||||
loc = self.locator("a.png")
|
||||
saved = self.replace(loc, tags=["a"], mark_autotagged=True)
|
||||
self.assertFalse(saved["replayed"])
|
||||
self.assertEqual(self.marked(), [self.byte_id(loc)])
|
||||
other = self.locator("b.png")
|
||||
self.replace(other, DIGEST_B, ["x"])
|
||||
replayed = self.replace(
|
||||
other, DIGEST_B, ["x"], revision=1, mark_autotagged=True
|
||||
)
|
||||
self.assertTrue(replayed["replayed"])
|
||||
third = self.locator("c.png")
|
||||
self.replace(third, tags=["y"])
|
||||
with self.assertRaises(AnnotationCommandError):
|
||||
self.replace(third, tags=["z"], revision=5, mark_autotagged=True)
|
||||
self.assertEqual(
|
||||
self.marked(), sorted([self.byte_id(loc), self.byte_id(other, DIGEST_B)])
|
||||
)
|
||||
|
||||
def test_mark_follows_the_bytes_not_the_file(self):
|
||||
loc = self.locator("a.png")
|
||||
self.add(loc, tags=["bird"], mark_autotagged=True)
|
||||
# The file's bytes changed: the new record is current and unmarked.
|
||||
self.replace(loc, DIGEST_B, ["hand"], mtime_ns=MTIME + 1)
|
||||
self.assertEqual(self.db.autotag_state({loc: (SIZE, MTIME + 1)}), {loc: False})
|
||||
self.assertEqual(self.db.autotag_state({loc: (SIZE, MTIME)}), {loc: True})
|
||||
# The review's estimate counts any marked record of the locator.
|
||||
self.assertEqual(self.db.autotagged_locators(), {loc})
|
||||
missing = self.locator("none.png")
|
||||
self.assertEqual(self.db.autotag_state({missing: (1, 1)}), {missing: False})
|
||||
|
||||
def test_state_and_tags_choose_the_same_current_record(self):
|
||||
loc = self.locator("a.png")
|
||||
# Two records with the same verified fingerprint: the newest wins.
|
||||
self.replace(loc, DIGEST_A, ["old bytes"])
|
||||
with patch.object(
|
||||
annotations_module, "utc_now_iso", return_value="2099-01-01T00:00:00"
|
||||
):
|
||||
self.add(loc, DIGEST_B, ["new bytes"], mark_autotagged=True)
|
||||
stats = {loc: (SIZE, MTIME)}
|
||||
self.assertEqual(self.db.current_image_tags(stats), {loc: ["new bytes"]})
|
||||
self.assertEqual(self.db.autotag_state(stats), {loc: True})
|
||||
with patch.object(
|
||||
annotations_module, "utc_now_iso", return_value="2199-01-01T00:00:00"
|
||||
):
|
||||
self.add(loc, DIGEST_A, ["older bytes again"])
|
||||
self.assertEqual(
|
||||
self.db.current_image_tags(stats), {loc: ["old bytes", "older bytes again"]}
|
||||
)
|
||||
self.assertEqual(self.db.autotag_state(stats), {loc: False})
|
||||
|
||||
def test_marks_never_touch_prompt_rows(self):
|
||||
conn = self.db.model.get_connection()
|
||||
prompt_id = self.db.save_prompt("a beach", tags=["beach"])
|
||||
before = [
|
||||
[tuple(row) for row in conn.execute(f"SELECT * FROM {table} ORDER BY 1")]
|
||||
for table in ("prompts", "prompt_tags")
|
||||
]
|
||||
self.add(self.locator("a.png"), tags=["bird"], mark_autotagged=True)
|
||||
self.replace(self.locator("b.png"), DIGEST_B, ["waves"], mark_autotagged=True)
|
||||
after = [
|
||||
[tuple(row) for row in conn.execute(f"SELECT * FROM {table} ORDER BY 1")]
|
||||
for table in ("prompts", "prompt_tags")
|
||||
]
|
||||
self.assertIsNotNone(prompt_id)
|
||||
self.assertEqual(after, before)
|
||||
|
||||
|
||||
class AnnotationIsolationTests(AnnotationTestCase):
|
||||
|
||||
+21
-13
@@ -846,14 +846,6 @@ class TestSettingsReadMonitorThroughModule(AdminAPITestCase):
|
||||
self.assertEqual(data["settings"]["gallery_root_paths"], ["output"])
|
||||
|
||||
|
||||
def _wired_prompt(text):
|
||||
"""A sampler-less graph whose one encoder feeds a guider (#252)."""
|
||||
return {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": text}},
|
||||
"2": {"class_type": "BasicGuider", "inputs": {"conditioning": ["1", 0]}},
|
||||
}
|
||||
|
||||
|
||||
def _raise(*args, **kwargs):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
@@ -2194,7 +2186,11 @@ class TestOutputScanInternals(AdminAPITestCase):
|
||||
|
||||
def test_ingest_outcomes(self):
|
||||
f = Path("gen.png")
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a cat"))}
|
||||
meta = {
|
||||
"prompt": json.dumps(
|
||||
{"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat"}}}
|
||||
)
|
||||
}
|
||||
self.assertIsNone(self.api._ingest_scanned_file(f, {}))
|
||||
self.assertIsNone(self.api._ingest_scanned_file(f, {"prompt": "{}"}))
|
||||
self.assertEqual(
|
||||
@@ -2206,19 +2202,31 @@ class TestOutputScanInternals(AdminAPITestCase):
|
||||
|
||||
def test_ingest_link_failures_are_tolerated(self):
|
||||
f = Path("gen.png")
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a dog"))}
|
||||
meta = {
|
||||
"prompt": json.dumps(
|
||||
{"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a dog"}}}
|
||||
)
|
||||
}
|
||||
self.api.db.link_image_to_prompt = _raise
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "added")
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "found")
|
||||
|
||||
def test_ingest_save_failure_counts_as_found(self):
|
||||
f = Path("gen.png")
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a bird"))}
|
||||
meta = {
|
||||
"prompt": json.dumps(
|
||||
{"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a bird"}}}
|
||||
)
|
||||
}
|
||||
self.api.db.save_prompt = lambda *a, **k: None
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "found")
|
||||
|
||||
def test_batch_counts_and_per_file_errors(self):
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a fish"))}
|
||||
meta = {
|
||||
"prompt": json.dumps(
|
||||
{"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a fish"}}}
|
||||
)
|
||||
}
|
||||
self.api.db.get_prompt_by_hash = _raise
|
||||
counts = self.api._ingest_scan_batch_sync(
|
||||
[(Path("a.png"), meta), (Path("b.png"), {})]
|
||||
@@ -2907,7 +2915,7 @@ class TestScanReadsVideos(AdminAPITestCase):
|
||||
shutil.which("ffmpeg") and shutil.which("ffprobe"), "ffmpeg not installed"
|
||||
)
|
||||
async def test_scan_adds_prompts_found_in_videos(self):
|
||||
graph = _wired_prompt("a cat")
|
||||
graph = {"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat"}}}
|
||||
_write_mp4(
|
||||
self.output_dir / "clip.mp4", json.dumps({"prompt": json.dumps(graph)})
|
||||
)
|
||||
|
||||
@@ -363,20 +363,23 @@ class TestModelGate(SuggestTestCase):
|
||||
prompt_id = self._save_prompt("batch prompt")
|
||||
self.api.db.link_image_to_prompt(prompt_id, str(self.output_dir / "sub/a.png"))
|
||||
started, proceed = threading.Event(), threading.Event()
|
||||
# The legacy batch is the autotag job (#379): it suggests per image.
|
||||
self.tagger.block = (started, proceed)
|
||||
|
||||
def slow_generate(image_path, **kwargs):
|
||||
started.set()
|
||||
proceed.wait(WAIT)
|
||||
return ["batch"]
|
||||
|
||||
self.tagger.generate_tags = slow_generate
|
||||
batch = asyncio.ensure_future(
|
||||
self.client.post("/prompt_manager/autotag/start?model_type=wd14-vit")
|
||||
)
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, started.wait, WAIT)
|
||||
calls = len(self.tagger.calls)
|
||||
status, body = await self.suggest(self.suggest_body(observed, "gguf"))
|
||||
self.assertEqual((status, body["code"]), (409, "model-busy"))
|
||||
self.assertEqual(len(self.tagger.calls), calls)
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
self.assertEqual(self.tagger.loaded, "wd14-vit") # not swapped
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
response = await batch
|
||||
await response.read()
|
||||
|
||||
+574
-78
@@ -5,7 +5,6 @@ no network request is made.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
@@ -35,7 +34,6 @@ from aiohttp.test_utils import AioHTTPTestCase
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
from py.api import PromptManagerAPI
|
||||
from py.api.autotag_routes import MODEL_BUSY_ERROR
|
||||
from py.api.jobs import JobBusy
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
@@ -72,7 +70,6 @@ class FakeAutotagService:
|
||||
self.status_calls = 0
|
||||
self.status_threads = []
|
||||
self.status_paths = {}
|
||||
self.suggest_calls = 0
|
||||
|
||||
def get_models_status(self):
|
||||
self.status_calls += 1
|
||||
@@ -118,25 +115,6 @@ class FakeAutotagService:
|
||||
raise self.generate_error
|
||||
return list(self.tags)
|
||||
|
||||
def generate_tag_suggestions(
|
||||
self,
|
||||
image_path,
|
||||
*,
|
||||
limit,
|
||||
prompt=None,
|
||||
general_threshold=None,
|
||||
character_threshold=None,
|
||||
):
|
||||
"""The suggestion entry point (#379): the fixed tags, at most ``limit``."""
|
||||
self.suggest_calls += 1
|
||||
tags = self.generate_tags(
|
||||
image_path,
|
||||
prompt=prompt,
|
||||
general_threshold=general_threshold,
|
||||
character_threshold=character_threshold,
|
||||
)
|
||||
return [(name, None) for name in tags[:limit]]
|
||||
|
||||
|
||||
class AutotagAPITestCase(AioHTTPTestCase):
|
||||
"""Stands up the PromptManager routes with a temp DB, temp output dir and a
|
||||
@@ -197,17 +175,8 @@ class AutotagAPITestCase(AioHTTPTestCase):
|
||||
)
|
||||
|
||||
def _make_image(self, directory, name="img.png"):
|
||||
"""A small, valid image (the legacy review captures and probes it)."""
|
||||
from PIL import Image
|
||||
|
||||
path = directory / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shade = sum(name.encode()) % 256
|
||||
fmt = {".jpg": "JPEG", ".jpeg": "JPEG", ".webp": "WEBP"}.get(
|
||||
path.suffix.lower(), "PNG"
|
||||
)
|
||||
with Image.new("RGB", (4, 4), (shade, 64, 128)) as image:
|
||||
image.save(path, fmt)
|
||||
path.write_bytes(b"\x89PNG\r\n\x1a\n")
|
||||
return path
|
||||
|
||||
def _link_image(self, prompt_id, path):
|
||||
@@ -247,9 +216,10 @@ class TestAutotagSingleContainment(AutotagAPITestCase):
|
||||
self.assertEqual(data["tags"], ["1girl", "smile"])
|
||||
self.assertEqual(data["prompt_id"], prompt_id)
|
||||
self.assertEqual(len(self.service.generate_calls), 1)
|
||||
# #379: the model reads a private capture of the image, never the file.
|
||||
self.assertNotIn(str(self.output_dir), self.service.generate_calls[0][0])
|
||||
self.assertEqual(data["sha256"], hashlib.sha256(image.read_bytes()).hexdigest())
|
||||
self.assertEqual(
|
||||
os.path.realpath(self.service.generate_calls[0][0]),
|
||||
os.path.realpath(str(image)),
|
||||
)
|
||||
|
||||
async def test_path_inside_output_dir_is_tagged(self):
|
||||
image = self._make_image(self.output_dir)
|
||||
@@ -479,6 +449,238 @@ class TestAutotagSideEffectRoutesArePostOnly(AutotagAPITestCase):
|
||||
self.assertTrue((await resp.json())["success"])
|
||||
|
||||
|
||||
class TestAutotagBatch(AutotagAPITestCase):
|
||||
"""Batch tagging over the DB image list with SSE progress reporting."""
|
||||
|
||||
def _seed(self):
|
||||
"""Three prompts: A (one image), B (two images), C (missing file)."""
|
||||
a = self._save_prompt("prompt a")
|
||||
b = self._save_prompt("prompt b")
|
||||
c = self._save_prompt("prompt c")
|
||||
self._link_image(a, self._make_image(self.output_dir, "a1.png"))
|
||||
self._link_image(b, self._make_image(self.output_dir, "b1.png"))
|
||||
self._link_image(b, self._make_image(self.output_dir, "b2.png"))
|
||||
self._link_image(c, self.output_dir / "missing.png")
|
||||
return a, b, c
|
||||
|
||||
async def _start(self, query="model_type=gguf", body=None):
|
||||
if body is None:
|
||||
resp = await self.client.request(
|
||||
"POST", f"/prompt_manager/autotag/start?{query}"
|
||||
)
|
||||
else:
|
||||
resp = await self._post_json(f"/prompt_manager/autotag/start?{query}", body)
|
||||
return resp, self._sse_events(await resp.text())
|
||||
|
||||
async def test_batch_tags_each_prompt_once_and_reports_counts(self):
|
||||
a, b, c = self._seed()
|
||||
|
||||
resp, events = await self._start()
|
||||
|
||||
self.assertEqual(resp.status, 200)
|
||||
done = events[-1]
|
||||
self.assertEqual(done["type"], "complete")
|
||||
self.assertEqual(done["processed"], 2)
|
||||
self.assertEqual(done["tagged"], 2)
|
||||
self.assertEqual(done["skipped"], 2)
|
||||
self.assertEqual(done["errors"], 0)
|
||||
self.assertEqual(len(self.service.generate_calls), 2)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(a)["tags"], ["1girl", "smile"])
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(b)["tags"], ["1girl", "smile"])
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(c)["tags"], [])
|
||||
progress = [e for e in events if e["type"] == "progress"]
|
||||
self.assertTrue(all(0 <= e["progress"] <= 100 for e in progress))
|
||||
self.assertTrue(any("processed" in e for e in progress))
|
||||
|
||||
async def test_rows_outside_output_dirs_are_skipped_and_counted(self):
|
||||
inside = self._save_prompt("inside")
|
||||
outside = self._save_prompt("outside")
|
||||
self._link_image(inside, self._make_image(self.output_dir, "in.png"))
|
||||
secret = self._make_image(self.secret_dir, "password.png")
|
||||
self._link_image(outside, secret)
|
||||
|
||||
_, events = await self._start()
|
||||
|
||||
done = events[-1]
|
||||
self.assertEqual(done["type"], "complete")
|
||||
self.assertEqual(done["processed"], 1)
|
||||
self.assertEqual(done["tagged"], 1)
|
||||
self.assertEqual(done["skipped"], 1)
|
||||
self.assertEqual(done["skipped_outside_roots"], 1)
|
||||
self.assertEqual(len(self.service.generate_calls), 1)
|
||||
self.assertEqual(
|
||||
os.path.realpath(self.service.generate_calls[0][0]),
|
||||
os.path.realpath(str(self.output_dir / "in.png")),
|
||||
)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(outside)["tags"], [])
|
||||
progress = [e for e in events if e["type"] == "progress" and "skipped" in e]
|
||||
self.assertTrue(progress)
|
||||
self.assertTrue(all("skipped_outside_roots" in e for e in progress))
|
||||
body = json.dumps(events)
|
||||
self.assertNotIn(str(self.secret_dir), body)
|
||||
|
||||
async def test_second_start_while_running_is_409(self):
|
||||
self._seed()
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_generate(image_path, **kwargs):
|
||||
started.set()
|
||||
release.wait(5)
|
||||
return ["slow"]
|
||||
|
||||
self.service.generate_tags = slow_generate
|
||||
first = asyncio.ensure_future(self._start())
|
||||
try:
|
||||
await asyncio.get_event_loop().run_in_executor(None, started.wait, 5)
|
||||
self.assertTrue(started.is_set())
|
||||
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?model_type=gguf"
|
||||
)
|
||||
|
||||
self.assertEqual(resp.status, 409)
|
||||
self.assertEqual(
|
||||
await resp.json(),
|
||||
{"success": False, "error": "Auto-tagging is already running"},
|
||||
)
|
||||
finally:
|
||||
release.set()
|
||||
resp1, events = await first
|
||||
self.assertEqual(resp1.status, 200)
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
|
||||
async def test_lock_is_released_after_a_run(self):
|
||||
self._seed()
|
||||
|
||||
resp1, _ = await self._start()
|
||||
resp2, events = await self._start()
|
||||
|
||||
self.assertEqual(resp1.status, 200)
|
||||
self.assertEqual(resp2.status, 200)
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
|
||||
async def test_lock_is_released_when_the_run_fails(self):
|
||||
self._seed()
|
||||
self.service.load_error = RuntimeError("no model")
|
||||
|
||||
_, events = await self._start()
|
||||
self.assertEqual(events[-1]["type"], "error")
|
||||
|
||||
self.service.load_error = None
|
||||
resp, events = await self._start()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
|
||||
async def test_already_tagged_prompts_are_skipped_unless_asked(self):
|
||||
p = self._save_prompt("tagged", tags=["cat"])
|
||||
self._link_image(p, self._make_image(self.output_dir, "t.png"))
|
||||
|
||||
_, events = await self._start("model_type=gguf&skip_tagged=true")
|
||||
self.assertEqual(events[-1]["skipped"], 1)
|
||||
self.assertEqual(self.service.generate_calls, [])
|
||||
|
||||
_, events = await self._start("model_type=gguf&skip_tagged=false")
|
||||
self.assertEqual(events[-1]["tagged"], 1)
|
||||
self.assertEqual(
|
||||
self.api.db.get_prompt_by_id(p)["tags"], ["cat", "1girl", "smile"]
|
||||
)
|
||||
|
||||
async def test_prompt_already_holding_the_tags_counts_as_skipped(self):
|
||||
p = self._save_prompt("same", tags=["1girl", "smile"])
|
||||
self._link_image(p, self._make_image(self.output_dir, "s.png"))
|
||||
|
||||
_, events = await self._start("model_type=gguf&skip_tagged=false")
|
||||
|
||||
self.assertEqual(events[-1]["skipped"], 1)
|
||||
self.assertEqual(events[-1]["tagged"], 0)
|
||||
|
||||
async def test_generation_errors_are_counted_and_do_not_abort(self):
|
||||
self._seed()
|
||||
self.service.generate_error = RuntimeError("boom")
|
||||
|
||||
_, events = await self._start()
|
||||
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
# a1, b1 and b2 each fail: a failed prompt is retried on its next image
|
||||
self.assertEqual(events[-1]["errors"], 3)
|
||||
self.assertEqual(events[-1]["tagged"], 0)
|
||||
|
||||
async def test_json_body_parameters_are_accepted(self):
|
||||
self._seed()
|
||||
|
||||
_, events = await self._start(
|
||||
"", {"model_type": "wd14-vit", "general_threshold": 0.5, "prompt": "p"}
|
||||
)
|
||||
|
||||
self.assertEqual(events[-1]["type"], "complete")
|
||||
self.assertEqual(self.service.load_calls, [("wd14-vit", True)])
|
||||
self.assertEqual(self.service.generate_calls[0][2], 0.5)
|
||||
self.assertEqual(self.service.custom_prompt, "p")
|
||||
|
||||
async def test_keep_in_memory_false_unloads_after_batch(self):
|
||||
self._seed()
|
||||
|
||||
_, events = await self._start("model_type=gguf&keep_in_memory=false")
|
||||
|
||||
self.assertEqual(events[-1]["model_loaded"], False)
|
||||
self.assertEqual(self.service.unload_calls, 1)
|
||||
|
||||
async def test_model_not_downloaded_is_an_error_event(self):
|
||||
self.service.downloaded = False
|
||||
|
||||
_, events = await self._start()
|
||||
|
||||
self.assertEqual(events[-1]["type"], "error")
|
||||
self.assertEqual(self.service.load_calls, [])
|
||||
|
||||
async def test_load_failure_is_an_error_event_without_details(self):
|
||||
self.service.load_error = RuntimeError("/srv/models/secret.gguf is corrupt")
|
||||
|
||||
_, events = await self._start()
|
||||
|
||||
self.assertEqual(events[-1]["type"], "error")
|
||||
self.assertNotIn("/srv/models", events[-1]["message"])
|
||||
|
||||
async def test_non_numeric_threshold_is_400(self):
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?general_threshold=abc"
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
async def test_invalid_json_body_is_400(self):
|
||||
resp = await self.client.request(
|
||||
"POST",
|
||||
"/prompt_manager/autotag/start",
|
||||
data="{not json",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
async def test_client_disconnect_stops_the_batch_and_unloads(self):
|
||||
self._seed()
|
||||
real_write = web.StreamResponse.write
|
||||
state = {"writes": 0}
|
||||
|
||||
async def flaky_write(response, data):
|
||||
state["writes"] += 1
|
||||
if b"Processing " in data:
|
||||
raise ConnectionResetError("Cannot write to closing transport")
|
||||
return await real_write(response, data)
|
||||
|
||||
with patch.object(web.StreamResponse, "write", flaky_write):
|
||||
try:
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?keep_in_memory=false"
|
||||
)
|
||||
await resp.read()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.assertEqual(len(self.service.generate_calls), 1)
|
||||
self.assertEqual(self.service.unload_calls, 1)
|
||||
|
||||
|
||||
class TestAutotagModelsAndDownloadErrors(AutotagAPITestCase):
|
||||
|
||||
async def test_models_error_is_500(self):
|
||||
@@ -632,15 +834,7 @@ class TestAutotagSingleEdgeCases(AutotagAPITestCase):
|
||||
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(self.service.load_calls, [("wd14-vit", True)])
|
||||
# #379: the request's prompt is for this call only.
|
||||
self.assertEqual(self.service.custom_prompt, "")
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/single",
|
||||
{"path": str(image), "model_type": "gguf", "prompt": "terse"},
|
||||
)
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(self.service.generate_calls[-1][1], "terse")
|
||||
self.assertEqual(self.service.custom_prompt, "")
|
||||
self.assertEqual(self.service.custom_prompt, "terse")
|
||||
|
||||
async def test_image_id_without_stored_path_is_404(self):
|
||||
prompt_id = self._save_prompt()
|
||||
@@ -656,6 +850,64 @@ class TestAutotagSingleEdgeCases(AutotagAPITestCase):
|
||||
self.assertEqual(resp.status, 404)
|
||||
|
||||
|
||||
class TestApplyAutotag(AutotagAPITestCase):
|
||||
|
||||
async def test_missing_prompt_id_is_400(self):
|
||||
resp = await self._post_json("/prompt_manager/autotag/apply", {"tags": ["a"]})
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
async def test_no_tags_is_a_noop(self):
|
||||
resp = await self._post_json("/prompt_manager/autotag/apply", {"prompt_id": 1})
|
||||
data = await resp.json()
|
||||
self.assertTrue(data["success"])
|
||||
self.assertEqual(data["message"], "No tags to apply")
|
||||
|
||||
async def test_unknown_prompt_is_404(self):
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply", {"prompt_id": 999, "tags": ["a"]}
|
||||
)
|
||||
self.assertEqual(resp.status, 404)
|
||||
|
||||
async def test_tags_are_merged_without_duplicates(self):
|
||||
prompt_id = self._save_prompt(tags=["cat"])
|
||||
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply",
|
||||
{"prompt_id": prompt_id, "tags": ["cat", "dog"]},
|
||||
)
|
||||
|
||||
data = await resp.json()
|
||||
self.assertEqual(data["added_tags"], ["dog"])
|
||||
self.assertEqual(data["total_tags"], 2)
|
||||
self.assertEqual(
|
||||
self.api.db.get_prompt_by_id(prompt_id)["tags"], ["cat", "dog"]
|
||||
)
|
||||
|
||||
async def test_string_tags_from_db_are_split(self):
|
||||
prompt_id = self._save_prompt()
|
||||
with patch.object(
|
||||
self.api.db,
|
||||
"get_prompt_by_id",
|
||||
return_value={"id": prompt_id, "tags": "cat, dog"},
|
||||
):
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply",
|
||||
{"prompt_id": prompt_id, "tags": ["dog", "owl"]},
|
||||
)
|
||||
data = await resp.json()
|
||||
self.assertEqual(data["added_tags"], ["owl"])
|
||||
self.assertEqual(data["total_tags"], 3)
|
||||
|
||||
async def test_db_error_is_500(self):
|
||||
with patch.object(
|
||||
self.api.db, "get_prompt_by_id", side_effect=RuntimeError("db")
|
||||
):
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply", {"prompt_id": 1, "tags": ["a"]}
|
||||
)
|
||||
self.assertEqual(resp.status, 500)
|
||||
|
||||
|
||||
class TestAutotagErrorBodiesHideServerPaths(AutotagAPITestCase):
|
||||
"""Every autotag handler reports exceptions through the shared public helper."""
|
||||
|
||||
@@ -680,12 +932,9 @@ class TestAutotagErrorBodiesHideServerPaths(AutotagAPITestCase):
|
||||
await self._assert_hidden(resp)
|
||||
|
||||
async def test_apply_error(self):
|
||||
image = self._make_image(self.output_dir)
|
||||
digest = hashlib.sha256(image.read_bytes()).hexdigest()
|
||||
with patch.object(self.api.db, "add_image_tags", side_effect=self._locked()):
|
||||
with patch.object(self.api.db, "get_prompt_by_id", side_effect=self._locked()):
|
||||
resp = await self._post_json(
|
||||
"/prompt_manager/autotag/apply",
|
||||
{"path": str(image), "sha256": digest, "tags": ["a"]},
|
||||
"/prompt_manager/autotag/apply", {"prompt_id": 1, "tags": ["a"]}
|
||||
)
|
||||
await self._assert_hidden(resp)
|
||||
|
||||
@@ -759,13 +1008,10 @@ class TestLegacyRoutesShareTheModelGate(AutotagAPITestCase):
|
||||
"POST", "/prompt_manager/autotag/download/gguf"
|
||||
)
|
||||
)
|
||||
# The legacy batch is read as a stream: one error frame (#379).
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?model_type=gguf"
|
||||
)
|
||||
self.assertEqual(
|
||||
self._sse_events(await resp.text()),
|
||||
[{"type": "error", "status": "error", "message": MODEL_BUSY_ERROR}],
|
||||
await self._busy(
|
||||
await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?model_type=gguf"
|
||||
)
|
||||
)
|
||||
await self._busy(
|
||||
await self.client.request("POST", "/prompt_manager/autotag/unload")
|
||||
@@ -780,7 +1026,7 @@ class TestLegacyRoutesShareTheModelGate(AutotagAPITestCase):
|
||||
self.assertEqual(self.service.unload_calls, 0)
|
||||
self.assertEqual(self.service.get_loaded_model_type(), "wd14-vit")
|
||||
|
||||
# A refused start left no job behind: the next start runs.
|
||||
# A refused start released the batch lock: the next start runs.
|
||||
resp = await self.client.request(
|
||||
"POST", "/prompt_manager/autotag/start?model_type=gguf"
|
||||
)
|
||||
@@ -992,7 +1238,7 @@ if __name__ == "__main__":
|
||||
|
||||
|
||||
class TestGalleryReviewContract(AutotagAPITestCase):
|
||||
async def test_root_relative_gallery_targets_generate_and_apply_to_their_image(
|
||||
async def test_root_relative_gallery_targets_generate_and_apply_to_their_prompt(
|
||||
self,
|
||||
):
|
||||
second = self.tmp_root / "second-output"
|
||||
@@ -1009,22 +1255,16 @@ class TestGalleryReviewContract(AutotagAPITestCase):
|
||||
self.assertEqual(response.status, 200)
|
||||
data = await response.json()
|
||||
self.assertEqual(data["prompt_id"], pid)
|
||||
self.assertEqual(
|
||||
self.service.generate_calls[-1][0], str((root / "same.png").resolve())
|
||||
)
|
||||
response = await self._post_json(
|
||||
"/prompt_manager/autotag/apply",
|
||||
{
|
||||
"path": "same.png",
|
||||
"root_index": index,
|
||||
"sha256": data["sha256"],
|
||||
"tags": data["tags"],
|
||||
},
|
||||
{"prompt_id": data["prompt_id"], "tags": data["tags"]},
|
||||
)
|
||||
self.assertEqual(response.status, 200)
|
||||
# The image's own tags; the prompt is never written (#379).
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(pid)["tags"], [])
|
||||
locator = os.path.normcase(os.path.realpath(str(root / "same.png")))
|
||||
self.assertEqual(
|
||||
self.api.db.read_image_tags(locator, data["sha256"])["tags"],
|
||||
data["tags"],
|
||||
set(self.api.db.get_prompt_by_id(pid)["tags"]), set(data["tags"])
|
||||
)
|
||||
|
||||
async def test_gallery_unlinked_image_returns_no_prompt_id(self):
|
||||
@@ -1653,15 +1893,13 @@ class TestTagOutputImagesPreview(BatchJobTestCase):
|
||||
self.assertEqual(self.service.generate_calls, [])
|
||||
self.assertEqual(self.service.load_calls, [])
|
||||
|
||||
async def test_preview_counts_listed_still_images_not_prompts(self):
|
||||
async def test_preview_counts_skip_and_merge(self):
|
||||
self._seed()
|
||||
skip = await (await self._preview(existing="skip")).json()
|
||||
# t, f1, f2 and m are listed; gone.png is missing, o.png is outside.
|
||||
self.assertEqual(
|
||||
{k: skip[k] for k in ("images", "already_tagged", "to_tag")},
|
||||
{"images": 4, "already_tagged": 0, "to_tag": 4},
|
||||
{k: skip[k] for k in ("images", "prompts", "already_tagged", "to_tag")},
|
||||
{"images": 6, "prompts": 5, "already_tagged": 1, "to_tag": 4},
|
||||
)
|
||||
self.assertNotIn("prompts", skip)
|
||||
self.assertEqual(
|
||||
skip["model"],
|
||||
{
|
||||
@@ -1678,7 +1916,7 @@ class TestTagOutputImagesPreview(BatchJobTestCase):
|
||||
set(skip["disk"]), {"free_bytes", "required_bytes", "sufficient"}
|
||||
)
|
||||
merge = await (await self._preview(existing="merge")).json()
|
||||
self.assertEqual(merge["to_tag"], 4)
|
||||
self.assertEqual(merge["to_tag"], 5)
|
||||
leaked = [s for s in _strings(skip) if _absolute(s)]
|
||||
self.assertEqual(leaked, [])
|
||||
|
||||
@@ -1686,7 +1924,7 @@ class TestTagOutputImagesPreview(BatchJobTestCase):
|
||||
self._seed()
|
||||
data = await (await self._preview(existing="replace_after_review")).json()
|
||||
self.assertTrue(data["review_required"])
|
||||
self.assertEqual(data["images"], 4)
|
||||
self.assertEqual(data["prompts"], 5)
|
||||
|
||||
async def test_preview_refuses_bad_bodies(self):
|
||||
for body in (
|
||||
@@ -1741,6 +1979,119 @@ class TestTagOutputImagesJob(BatchJobTestCase):
|
||||
"WD14 ViT is not downloaded. Download it in System › Auto-tagging.",
|
||||
)
|
||||
|
||||
async def test_job_writes_what_legacy_writes(self):
|
||||
ids = self._seed()
|
||||
await self._run_job(existing="skip")
|
||||
native = {name: sorted(self._tags(pid)) for name, pid in ids.items()}
|
||||
# The same library through the legacy batch, in a fresh database.
|
||||
for name, pid in ids.items():
|
||||
self.api.db.update_prompt_metadata(
|
||||
pid,
|
||||
tags=(
|
||||
["existing"]
|
||||
if name == "tagged"
|
||||
else (["auto-scanned"] if name == "marked" else [])
|
||||
),
|
||||
)
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/autotag/start?model_type=wd14-vit&skip_tagged=true"
|
||||
)
|
||||
self.assertEqual(self._sse_events(await resp.text())[-1]["type"], "complete")
|
||||
legacy = {name: sorted(self._tags(pid)) for name, pid in ids.items()}
|
||||
self.assertEqual(native, legacy)
|
||||
self.assertEqual(native["tagged"], ["existing"])
|
||||
self.assertEqual(native["fresh"], ["1girl", "smile"])
|
||||
|
||||
async def test_job_outcomes_follow_the_mapping(self):
|
||||
ids = self._seed()
|
||||
self.api.db.update_prompt_metadata(
|
||||
ids["marked"], tags=["auto-scanned", "1girl", "smile"]
|
||||
)
|
||||
|
||||
def details(job):
|
||||
return {
|
||||
group: sorted(row["detail"] for row in rows)
|
||||
for group, rows in job.outcomes.items()
|
||||
}
|
||||
|
||||
merged = await self._run_job(existing="merge")
|
||||
self.assertEqual(merged.state, "finished")
|
||||
self.assertEqual(
|
||||
details(merged),
|
||||
{
|
||||
"tagged": ["Added 2 tags", "Added 2 tags"],
|
||||
"unchanged": ["No new tags"],
|
||||
"skipped": [
|
||||
"File not found",
|
||||
"Outside the output folders",
|
||||
"Prompt handled by another image",
|
||||
],
|
||||
"errors": [],
|
||||
},
|
||||
)
|
||||
self.assertEqual(merged.status_line, "Tagged 2 prompts")
|
||||
|
||||
skipped = await self._run_job(existing="skip")
|
||||
self.assertEqual(details(skipped)["tagged"], [])
|
||||
self.assertEqual(details(skipped)["skipped"].count("Prompt already tagged"), 3)
|
||||
self.assertEqual(skipped.status_line, "Tagged 0 prompts")
|
||||
|
||||
async def test_tagging_errors_are_fixed_strings(self):
|
||||
self._seed()
|
||||
self.service.generate_error = RuntimeError("/srv/secret/model exploded")
|
||||
job = await self._run_job(existing="merge")
|
||||
self.assertTrue(job.outcomes["errors"])
|
||||
self.assertEqual(
|
||||
{r["detail"] for r in job.outcomes["errors"]}, {"Tagging failed"}
|
||||
)
|
||||
self.assertNotIn("/srv/secret", json.dumps(job.report()))
|
||||
|
||||
async def test_job_progress_is_determinate(self):
|
||||
self._seed()
|
||||
seen = []
|
||||
original = self.service.generate_tags
|
||||
|
||||
def generate(*args, **kwargs):
|
||||
job = self.api._jobs.active("autotag")
|
||||
seen.append((job.processed, job.total))
|
||||
return original(*args, **kwargs)
|
||||
|
||||
self.service.generate_tags = generate
|
||||
job = await self._run_job(existing="merge")
|
||||
self.assertEqual((job.processed, job.total, job.progress), (6, 6, 100))
|
||||
self.assertTrue(all(total == 6 for _p, total in seen[1:]), seen)
|
||||
self.assertTrue(job.cancellable)
|
||||
|
||||
async def test_job_cancel_stops_before_the_next_image_and_keeps_writes(self):
|
||||
ids = self._seed()
|
||||
started = threading.Event()
|
||||
proceed = threading.Event()
|
||||
calls = []
|
||||
|
||||
def slow(image_path, **kwargs):
|
||||
calls.append(image_path)
|
||||
started.set()
|
||||
proceed.wait(30)
|
||||
return ["slow"]
|
||||
|
||||
self.service.generate_tags = slow
|
||||
resp = await self._start_job(existing="merge")
|
||||
job = self.api._jobs.get((await resp.json())["job"]["id"])
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, started.wait, 30)
|
||||
cancel = await self.client.post(f"/prompt_manager/jobs/{job.id}/cancel")
|
||||
self.assertTrue((await cancel.json())["cancellation_requested"])
|
||||
finally:
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), 30)
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.status_line, "Cancelled. Tags already added were kept.")
|
||||
self.assertEqual(len(calls), 1)
|
||||
tagged = [n for n, pid in ids.items() if "slow" in self._tags(pid)]
|
||||
self.assertEqual(len(tagged), 1)
|
||||
self.assertIsNone(model_gate.holder())
|
||||
self.assertEqual(self.service.unload_calls, 0)
|
||||
|
||||
async def test_job_refused_while_legacy_batch_holds(self):
|
||||
async with self.api._jobs.hold(
|
||||
"autotag", "Tag output images (legacy page)", admission="records"
|
||||
@@ -1752,6 +2103,41 @@ class TestTagOutputImagesJob(BatchJobTestCase):
|
||||
self.assertEqual(data["running"]["title"], "Tag output images (legacy page)")
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_legacy_refused_while_job_runs(self):
|
||||
self._seed()
|
||||
started, proceed = threading.Event(), threading.Event()
|
||||
|
||||
def slow(image_path, **kwargs):
|
||||
started.set()
|
||||
proceed.wait(30)
|
||||
return ["slow"]
|
||||
|
||||
self.service.generate_tags = slow
|
||||
resp = await self._start_job(existing="merge")
|
||||
job = self.api._jobs.get((await resp.json())["job"]["id"])
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, started.wait, 30)
|
||||
self.assertTrue(model_gate.holder().startswith("autotag-job:"))
|
||||
legacy = await self.client.post("/prompt_manager/autotag/start")
|
||||
# Both legacy pages read a stream: the refusal is one error frame
|
||||
# that names the running job.
|
||||
self.assertEqual(legacy.status, 200)
|
||||
self.assertEqual(
|
||||
self._sse_events(await legacy.text()),
|
||||
[
|
||||
{
|
||||
"type": "error",
|
||||
"status": "error",
|
||||
"message": "Tag output images already running",
|
||||
}
|
||||
],
|
||||
)
|
||||
unload = await self.client.post("/prompt_manager/autotag/unload")
|
||||
self.assertEqual((await unload.json())["operation"], "Tag output images")
|
||||
finally:
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), 30)
|
||||
|
||||
async def test_job_releases_gate_when_start_is_refused(self):
|
||||
class Running:
|
||||
id = "other"
|
||||
@@ -1799,7 +2185,7 @@ class TestTagOutputImagesJob(BatchJobTestCase):
|
||||
("database folder", "the safety backup", 10, 1),
|
||||
10,
|
||||
)
|
||||
with patch("py.api.image_tag_jobs.disk_plan", return_value=short):
|
||||
with patch("py.api.autotag_routes.disk_plan", return_value=short):
|
||||
resp = await self._start_job()
|
||||
data = await resp.json()
|
||||
self.assertEqual(resp.status, 507)
|
||||
@@ -1817,6 +2203,116 @@ class TestTagOutputImagesJob(BatchJobTestCase):
|
||||
inside = [row for row in rows if row["root_index"] == 0]
|
||||
self.assertTrue(inside)
|
||||
|
||||
async def test_native_batch_leaves_custom_prompt_unchanged(self):
|
||||
self._seed()
|
||||
self.service.custom_prompt = "legacy page prompt"
|
||||
await self._run_job(existing="merge", model_type="gguf")
|
||||
self.assertEqual(self.service.custom_prompt, "legacy page prompt")
|
||||
prompts = {call[1] for call in self.service.generate_calls}
|
||||
self.assertEqual(prompts, {"Tags please."})
|
||||
|
||||
|
||||
# The legacy batch's per-image frames over BatchJobTestCase's library, newest
|
||||
# first, with every throttle check passing (pinned from main's generator).
|
||||
GOLDEN_LEGACY_ROWS = [
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 24,
|
||||
"status": "Processing 1/6...",
|
||||
"processed": 1,
|
||||
"tagged": 1,
|
||||
"skipped": 0,
|
||||
"skipped_outside_roots": 0,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 38,
|
||||
"status": "Skipping 2/6 (outside allowed directories)...",
|
||||
"processed": 1,
|
||||
"tagged": 1,
|
||||
"skipped": 1,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 52,
|
||||
"status": "Skipping 3/6 (file missing)...",
|
||||
"processed": 1,
|
||||
"tagged": 1,
|
||||
"skipped": 2,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 66,
|
||||
"status": "Processing 4/6...",
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 2,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 80,
|
||||
"status": "Skipping 5/6 (prompt already processed)...",
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 3,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 95,
|
||||
"status": "Skipping 6/6 (already tagged)...",
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 4,
|
||||
"skipped_outside_roots": 1,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
class TestLegacyBatchFrames(BatchJobTestCase):
|
||||
|
||||
async def test_legacy_batch_events_unchanged(self):
|
||||
"""Golden frames of the legacy batch over the seeded library."""
|
||||
self._seed()
|
||||
ticks = iter(range(1000)) # every throttle check sees a second pass
|
||||
with patch("py.api.autotag_routes._monotonic", side_effect=lambda: next(ticks)):
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/autotag/start?model_type=wd14-vit&skip_tagged=true"
|
||||
)
|
||||
frames = self._sse_events(await resp.text()) # the stream runs here
|
||||
self.assertEqual(
|
||||
frames,
|
||||
[
|
||||
{"type": "progress", "progress": 0, "status": "Loading model..."},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 5,
|
||||
"status": "Model loaded. Fetching all images from database...",
|
||||
},
|
||||
{
|
||||
"type": "progress",
|
||||
"progress": 10,
|
||||
"status": "Found 6 images. Processing...",
|
||||
},
|
||||
*GOLDEN_LEGACY_ROWS,
|
||||
{
|
||||
"type": "complete",
|
||||
"progress": 100,
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 4,
|
||||
"errors": 0,
|
||||
"skipped_outside_roots": 1,
|
||||
"status": "Complete",
|
||||
"model_status": "Model kept in memory",
|
||||
"model_loaded": True,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class TestTagOutputImagesEdges(BatchJobTestCase):
|
||||
|
||||
@@ -1846,7 +2342,7 @@ class TestTagOutputImagesEdges(BatchJobTestCase):
|
||||
async def test_unexpected_error_fails_the_job_with_a_reason(self):
|
||||
self._seed()
|
||||
with patch.object(
|
||||
self.api, "_autotag_prepass", side_effect=RuntimeError("/srv/x")
|
||||
self.api.db, "autotag_batch_candidates", side_effect=[[], RuntimeError("x")]
|
||||
):
|
||||
job = await self._run_job(existing="merge")
|
||||
self.assertEqual(job.state, "failed")
|
||||
|
||||
+92
-24
@@ -1,6 +1,5 @@
|
||||
"""Tests for the core API module (py/api/__init__.py): helpers and base routes."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
@@ -166,19 +165,8 @@ class TestGzipMiddleware(AioHTTPTestCase):
|
||||
await resp.write_eof()
|
||||
return resp
|
||||
|
||||
async def declared_length(request):
|
||||
# The log download (#329) declares its length before the middleware
|
||||
# compresses it.
|
||||
body = b"x" * 3000
|
||||
return web.Response(
|
||||
body=body,
|
||||
content_type="text/plain",
|
||||
headers={"Content-Length": str(len(body))},
|
||||
)
|
||||
|
||||
app = web.Application(middlewares=[api_module._gzip_middleware])
|
||||
app.router.add_get("/prompt_manager/big", big_json)
|
||||
app.router.add_get("/prompt_manager/declared", declared_length)
|
||||
app.router.add_get("/prompt_manager/small", small_json)
|
||||
app.router.add_get("/prompt_manager/encoded", already_encoded)
|
||||
app.router.add_get("/prompt_manager/binary", binary)
|
||||
@@ -205,18 +193,6 @@ class TestGzipMiddleware(AioHTTPTestCase):
|
||||
|
||||
self.assertIn(b'"data"', gzip.decompress(await resp.read()))
|
||||
|
||||
async def test_compressed_body_declares_its_own_length(self):
|
||||
# A stale Content-Length (the uncompressed size) makes a browser wait
|
||||
# for bytes that never come (#329: Download log hung on any log > 1 KB).
|
||||
resp = await self._get("/prompt_manager/declared")
|
||||
self.assertEqual(resp.headers.get("Content-Encoding"), "gzip")
|
||||
# Bounded: a stale length would otherwise wait for the missing bytes.
|
||||
body = await asyncio.wait_for(resp.read(), timeout=10)
|
||||
self.assertEqual(int(resp.headers["Content-Length"]), len(body))
|
||||
import gzip
|
||||
|
||||
self.assertEqual(gzip.decompress(body), b"x" * 3000)
|
||||
|
||||
async def test_small_body_not_compressed(self):
|
||||
resp = await self._get("/prompt_manager/small")
|
||||
self.assertIsNone(resp.headers.get("Content-Encoding"))
|
||||
@@ -835,6 +811,7 @@ class TestMetadataParsing(unittest.TestCase):
|
||||
self.assertEqual(parsed["positive_prompt"], "a cat")
|
||||
self.assertEqual(parsed["negative_prompt"], "dog")
|
||||
self.assertEqual(parsed["parameters"]["parameters"], meta["parameters"])
|
||||
self.assertEqual(self.api._extract_readable_prompt(parsed), "a cat")
|
||||
|
||||
def test_parse_comfyui_fields(self):
|
||||
meta = {
|
||||
@@ -855,6 +832,97 @@ class TestMetadataParsing(unittest.TestCase):
|
||||
self.assertEqual(parsed["prompt"], "plain text")
|
||||
self.assertEqual(parsed["workflow"], {"nodes": []})
|
||||
|
||||
def test_extract_readable_prompt_variants(self):
|
||||
extract = self.api._extract_readable_prompt
|
||||
self.assertEqual(extract({"prompt": "plain"}), "plain")
|
||||
self.assertEqual(extract({"prompt": ["a", "", "b"]}), "a b")
|
||||
self.assertEqual(extract({"prompt": 42}), "42")
|
||||
self.assertEqual(
|
||||
extract({"prompt": None, "parameters": {"positive": ["x", "y"]}}), "x y"
|
||||
)
|
||||
self.assertIsNone(extract({"prompt": {}, "parameters": {}}))
|
||||
|
||||
graph = {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "pos"}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": "neg"}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["2", 0]},
|
||||
},
|
||||
}
|
||||
self.assertEqual(extract({"prompt": graph}), "pos")
|
||||
self.assertEqual(extract({"prompt": {"x": 1}, "workflow": graph}), "pos")
|
||||
|
||||
def test_node_inputs_and_text_lookup(self):
|
||||
self.assertEqual(self.api._get_node_inputs("nope"), {})
|
||||
self.assertEqual(self.api._get_node_inputs({"inputs": 5}), {})
|
||||
self.assertEqual(self.api._get_node_inputs({"inputs": {"a": 1}}), {"a": 1})
|
||||
listed = self.api._get_node_inputs(
|
||||
{"inputs": [{"name": "text", "link": None}, {"nope": 1}, "junk"]}
|
||||
)
|
||||
self.assertEqual(list(listed), ["text"])
|
||||
|
||||
self.assertIsNone(self.api._find_text_in_node(None))
|
||||
self.assertEqual(self.api._find_text_in_node({"inputs": {"text": "hi"}}), "hi")
|
||||
self.assertEqual(
|
||||
self.api._find_text_in_node(
|
||||
{"type": "CLIPTextEncode", "widgets_values": ["w"]}
|
||||
),
|
||||
"w",
|
||||
)
|
||||
self.assertIsNone(
|
||||
self.api._find_text_in_node(
|
||||
{"type": "CLIPTextEncode", "widgets_values": [" "]}
|
||||
)
|
||||
)
|
||||
self.assertIsNone(self.api._find_text_in_node({"type": "KSampler"}))
|
||||
|
||||
def test_positive_prompt_from_nodes_array_and_fallbacks(self):
|
||||
extract = self.api._extract_positive_prompt_from_comfyui_data
|
||||
self.assertIsNone(extract("x"))
|
||||
self.assertIsNone(extract({}))
|
||||
self.assertIsNone(extract({"nodes": []}))
|
||||
|
||||
workflow = {
|
||||
"nodes": [
|
||||
{"id": 1, "type": "CLIPTextEncode", "widgets_values": ["pos"]},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "CLIPTextEncode",
|
||||
"widgets_values": ["neg"],
|
||||
"title": "Negative",
|
||||
},
|
||||
{"id": 3, "type": "KSampler", "inputs": [{"name": "seed", "link": 4}]},
|
||||
"junk",
|
||||
]
|
||||
}
|
||||
self.assertEqual(extract(workflow), "pos")
|
||||
|
||||
only_negative = {
|
||||
"nodes": [
|
||||
{
|
||||
"id": 2,
|
||||
"type": "CLIPTextEncode",
|
||||
"widgets_values": ["neg"],
|
||||
"title": "neg",
|
||||
},
|
||||
]
|
||||
}
|
||||
self.assertEqual(extract(only_negative), "neg")
|
||||
|
||||
bad_link = {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "pos"}},
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": "oops", "negative": []},
|
||||
},
|
||||
"4": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {"positive": ["zz", 0], "negative": []},
|
||||
},
|
||||
}
|
||||
self.assertEqual(extract(bad_link), "pos")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -190,8 +190,7 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
"inputs": {"positive": ["5", 0], "negative": ["2", 0]},
|
||||
}
|
||||
else:
|
||||
# No text and no links: nothing passes through it (#381).
|
||||
source["1"] = {"class_type": "UnknownEncoder", "inputs": {"seed": 1}}
|
||||
source["1"]["class_type"] = "UnknownEncoder"
|
||||
png(
|
||||
path,
|
||||
prompt=source,
|
||||
@@ -216,7 +215,7 @@ class TestPromptInspector(ImageAPITestCase):
|
||||
result["prompts"]["negative"], self.side("", "embedded-parameters")
|
||||
)
|
||||
source = graph()
|
||||
source["2"] = {"class_type": "UnknownEncoder", "inputs": {"seed": 1}}
|
||||
source["2"]["class_type"] = "UnknownEncoder"
|
||||
png(path, prompt=source)
|
||||
_, result = await self.inspect()
|
||||
self.assertEqual(result["prompts"]["positive"], self.side("portrait"))
|
||||
|
||||
@@ -1831,39 +1831,6 @@ class TestWorkspaceCursorEncoding(unittest.TestCase):
|
||||
self.assertEqual(_workspace_cursor_key(token, "scope", order), key)
|
||||
|
||||
|
||||
# The pre-#379 candidate read, kept as the differential reference for the
|
||||
# indexed lookup (`image_tag_ids` + `image_tag_locators`): every byte record
|
||||
# streamed with its tag names, a locator kept when any record's names are
|
||||
# relevant (`_image_tags_relevant`).
|
||||
_REFERENCE_RECORDS = (
|
||||
"SELECT f.locator,b.id,t.name FROM media_annotation_files f "
|
||||
"JOIN media_annotation_bytes b ON b.file_id=f.id "
|
||||
"LEFT JOIN media_image_tags mit ON mit.byte_id=b.id "
|
||||
"LEFT JOIN tags t ON t.id=mit.tag_id "
|
||||
"ORDER BY f.locator,b.first_seen_at,b.id,mit.position"
|
||||
)
|
||||
|
||||
|
||||
def _image_tags_relevant(names, wanted, query):
|
||||
if wanted and not wanted.isdisjoint(names):
|
||||
return True
|
||||
return bool(query) and any(query in name.casefold() for name in names)
|
||||
|
||||
|
||||
def reference_candidates(db, wanted, query):
|
||||
"""Locators the old full stream admitted for (``wanted``, ``query``)."""
|
||||
records = {}
|
||||
for locator, byte_id, name in db.model.get_connection().execute(_REFERENCE_RECORDS):
|
||||
names = records.setdefault((locator, byte_id), [])
|
||||
if name is not None:
|
||||
names.append(name)
|
||||
return {
|
||||
locator
|
||||
for (locator, _byte), names in records.items()
|
||||
if _image_tags_relevant(names, set(wanted), query)
|
||||
}
|
||||
|
||||
|
||||
class TestWorkspaceImageTags(ImageAPITestCase):
|
||||
"""B3: image tags in the Tags filter and search (spec amendment S1)."""
|
||||
|
||||
@@ -2071,7 +2038,7 @@ class TestWorkspaceImageTags(ImageAPITestCase):
|
||||
leak = "no such table: media_annotation_files"
|
||||
with patch.object(
|
||||
self.api.db,
|
||||
"image_tag_locators",
|
||||
"iter_image_tag_records",
|
||||
side_effect=sqlite3.OperationalError(leak),
|
||||
):
|
||||
# As a failed link read does under a Tags filter: a fixed coded
|
||||
@@ -2089,140 +2056,6 @@ class TestWorkspaceImageTags(ImageAPITestCase):
|
||||
self.assertEqual(self.names(body), ["dunes_by_name.png"])
|
||||
self.assertIn("image tags", "\n".join(logged.output))
|
||||
|
||||
def seed_differential(self):
|
||||
"""Mixed-case and underscore names, a stale record, an unlisted file."""
|
||||
a = self.media("a.png", mtime=101)
|
||||
b = self.media("b.png", mtime=102)
|
||||
c = self.media("c.png", mtime=103)
|
||||
self.tag(a, ["Beach", "long_hair", "1girl"])
|
||||
self.tag(b, ["beach", "waves"])
|
||||
self.tag(c, ["Sunset"])
|
||||
# c's bytes change: its old record (Sunset) is stale, the new one current.
|
||||
make_png(c, size=(5, 5))
|
||||
os.utime(c, (104, 104))
|
||||
self.tag(c, ["night_sky", "BEACH"])
|
||||
outside = Path(self._output_tmp.name) / "elsewhere" / "x.png"
|
||||
outside.parent.mkdir(parents=True, exist_ok=True)
|
||||
make_png(outside)
|
||||
self.tag(outside, ["beach", "outside_only"])
|
||||
|
||||
async def test_indexed_candidates_equal_the_full_stream(self):
|
||||
self.seed_differential()
|
||||
cases = [
|
||||
(["beach"], ""),
|
||||
(["Beach"], ""),
|
||||
(["long_hair", "waves"], ""),
|
||||
([], "beach"),
|
||||
([], "hair"),
|
||||
([], "_"),
|
||||
([], "sun"),
|
||||
(["nope"], "zzz"),
|
||||
(["outside_only"], ""),
|
||||
(["beach"], "night"),
|
||||
]
|
||||
db = self.api.db
|
||||
for wanted, query in cases:
|
||||
with self.subTest(wanted=wanted, query=query):
|
||||
indexed = set(db.image_tag_locators(db.image_tag_ids(wanted, query)))
|
||||
self.assertEqual(indexed, reference_candidates(db, wanted, query))
|
||||
# The API answers are the indexed lookup's: exact names under Tags, a
|
||||
# case-folded substring under q, current records only.
|
||||
_, body = await self.get(tags="beach")
|
||||
self.assertEqual(sorted(self.names(body)), ["b.png"])
|
||||
_, body = await self.get(q="beach")
|
||||
self.assertEqual(sorted(self.names(body)), ["a.png", "b.png", "c.png"])
|
||||
_, body = await self.get(q="sunset")
|
||||
self.assertEqual(self.names(body), [])
|
||||
|
||||
async def test_tag_search_reads_image_tags_only_by_tag_id(self):
|
||||
self.seed_differential()
|
||||
statements = []
|
||||
original = self.api.db.model.get_connection
|
||||
|
||||
def trace_connection():
|
||||
connection = original()
|
||||
connection.set_trace_callback(statements.append)
|
||||
return connection
|
||||
|
||||
with patch.object(
|
||||
self.api.db.model, "get_connection", side_effect=trace_connection
|
||||
):
|
||||
for params in ({"tags": "beach"}, {"q": "hair"}):
|
||||
response, _ = await self.get(**params)
|
||||
self.assertEqual(response.status, 200)
|
||||
reads = [
|
||||
sql
|
||||
for sql in statements
|
||||
if "media_image_tags" in sql and sql.lstrip().upper().startswith("SELECT")
|
||||
]
|
||||
self.assertTrue(reads)
|
||||
for sql in reads:
|
||||
# Only a tag-id lookup, an EXISTS probe on the tag index, or the
|
||||
# current records' own tags (byte_id IN) read the junction.
|
||||
self.assertTrue(
|
||||
"tag_id IN" in sql or "EXISTS" in sql or "byte_id IN" in sql, sql
|
||||
)
|
||||
|
||||
async def test_the_image_tag_lookup_never_reads_prompt_tags(self):
|
||||
self.seed_differential()
|
||||
linked = self.media("linked.png", mtime=105)
|
||||
prompt = self._save_prompt(
|
||||
"Prompt with its own tag", tags=["prompt_only_beach"]
|
||||
)
|
||||
self.api.db.link_image_to_prompt(prompt, str(linked))
|
||||
db = self.api.db
|
||||
# The tag is only on a prompt: the image-tag lookup never yields it,
|
||||
# under the Tags filter or the text search.
|
||||
self.assertEqual(db.image_tag_ids(["prompt_only_beach"], ""), [])
|
||||
self.assertEqual(db.image_tag_ids([], "prompt_only"), [])
|
||||
self.assertEqual(
|
||||
db.image_tag_locators(db.image_tag_ids([], "beach")),
|
||||
reference_candidates(db, [], "beach"),
|
||||
)
|
||||
# The search still matches through the linked positive's prompt tags
|
||||
# (the design keeps that, #379 "Behavior that does not change").
|
||||
_, body = await self.get(q="prompt_only")
|
||||
self.assertEqual(self.names(body), ["linked.png"])
|
||||
|
||||
@unittest.skipUnless(os.environ.get("PM_PERF") == "1", "timing run (PM_PERF=1)")
|
||||
async def test_tag_search_timing(self):
|
||||
"""Linux timing for the PR body: 23,600 records, 10 tags each."""
|
||||
import hashlib
|
||||
import statistics
|
||||
import time
|
||||
|
||||
db = self.api.db
|
||||
epoch = db.get_execution_epoch()
|
||||
names = [f"name_{n:04d}" for n in range(2000)]
|
||||
root = os.path.realpath(self.output_dir)
|
||||
for n in range(23_600):
|
||||
path = self.output_dir / "perf" / f"{n:05d}.png"
|
||||
path.parent.mkdir(exist_ok=True)
|
||||
path.write_bytes(b"x")
|
||||
os.utime(path, (n + 1, n + 1))
|
||||
stat = path.stat()
|
||||
db.replace_image_tags(
|
||||
locator=os.path.normcase(os.path.join(root, "perf", path.name)),
|
||||
sha256=hashlib.sha256(str(n).encode()).hexdigest(),
|
||||
size=stat.st_size,
|
||||
mtime_ns=stat.st_mtime_ns,
|
||||
expected_epoch=epoch,
|
||||
expected_tag_revision=0,
|
||||
tags=[names[(n * 7 + k) % 2000] for k in range(10)],
|
||||
)
|
||||
await self.get(limit=1) # the listing, warm
|
||||
for label, params in (
|
||||
("one tag", {"tags": "name_0042"}),
|
||||
("text", {"q": "e_1"}),
|
||||
):
|
||||
times = []
|
||||
for _ in range(5):
|
||||
started = time.perf_counter()
|
||||
response, _ = await self.get(limit=100, **params)
|
||||
times.append(time.perf_counter() - started)
|
||||
self.assertEqual(response.status, 200)
|
||||
print(f"PM_PERF {label}: median {statistics.median(times) * 1000:.0f} ms")
|
||||
|
||||
async def test_legacy_output_failure_logs_its_traceback(self):
|
||||
self.media("legacy.png", mtime=100)
|
||||
with (
|
||||
@@ -2335,162 +2168,3 @@ class TestWorkspaceImageTags(ImageAPITestCase):
|
||||
)
|
||||
]
|
||||
self.assertFalse([step for step in plan if step.startswith("SCAN")], plan)
|
||||
|
||||
|
||||
class TestListingFilterSelection(ImageAPITestCase):
|
||||
"""#392: "Select all N matching" resolves the listing filter on the server."""
|
||||
|
||||
async def tearDownAsync(self):
|
||||
self.api.db.close_all()
|
||||
await super().tearDownAsync()
|
||||
|
||||
def media(self, name, mtime=100):
|
||||
target = self.output_dir / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
make_png(target)
|
||||
os.utime(target, (mtime, mtime))
|
||||
return target
|
||||
|
||||
async def listing(self, **params):
|
||||
response = await self.client.get(
|
||||
"/prompt_manager/images/output",
|
||||
params={"mode": "workspace", "limit": 500, **params},
|
||||
)
|
||||
return (await response.json())["images"]
|
||||
|
||||
async def test_resolve_lists_every_match_in_listing_order_beyond_a_page(self):
|
||||
for number in range(7):
|
||||
self.media(f"bulk/{number:02}.png", 100 + number)
|
||||
self.media("other/x.png", 500)
|
||||
from py.api.images import parse_listing_filter
|
||||
|
||||
params = parse_listing_filter({"subfolder": "bulk", "sort": "newest"})
|
||||
resolved = await self.api.resolve_listing_filter(params, cap=100)
|
||||
|
||||
listed = await self.listing(subfolder="bulk", sort="newest")
|
||||
self.assertEqual(
|
||||
[list(target) for target in resolved.targets],
|
||||
[[i["root_index"], i["root_scope"], i["path"]] for i in listed],
|
||||
)
|
||||
self.assertEqual(len(resolved.targets), 7)
|
||||
self.assertFalse(resolved.truncated)
|
||||
|
||||
async def test_resolve_reports_linked_positives_and_unlinked_count(self):
|
||||
first = self.media("a.png", 300)
|
||||
second = self.media("b.png", 200)
|
||||
self.media("c.png", 100)
|
||||
prompt = self._save_prompt("shared positive")
|
||||
self.api.db.link_image_to_prompt(prompt, str(first))
|
||||
self.api.db.link_image_to_prompt(prompt, str(second))
|
||||
from py.api.images import parse_listing_filter
|
||||
|
||||
resolved = await self.api.resolve_listing_filter(parse_listing_filter({}), 100)
|
||||
|
||||
self.assertEqual(resolved.positive_ids, [prompt])
|
||||
self.assertEqual(resolved.unlinked, 1)
|
||||
|
||||
async def test_resolve_marks_more_than_the_cap_as_truncated(self):
|
||||
for number in range(4):
|
||||
self.media(f"{number}.png", 100 + number)
|
||||
from py.api.images import parse_listing_filter
|
||||
|
||||
resolved = await self.api.resolve_listing_filter(parse_listing_filter({}), 3)
|
||||
|
||||
self.assertTrue(resolved.truncated)
|
||||
|
||||
def test_parse_listing_filter_rejects_anything_but_listing_strings(self):
|
||||
from py.api.images import parse_listing_filter
|
||||
|
||||
self.assertEqual(
|
||||
parse_listing_filter({"q": "cat", "tags": "a,b", "root": "0"}),
|
||||
{"q": "cat", "tags": "a,b", "root": "0"},
|
||||
)
|
||||
for bad in (
|
||||
None,
|
||||
[],
|
||||
"q=cat",
|
||||
{"mode": "workspace"},
|
||||
{"cursor": "x"},
|
||||
{"path": "a.png"},
|
||||
{"q": 1},
|
||||
{"q": "x" * 2049},
|
||||
{"sort": "sideways"},
|
||||
{"media": "audio"},
|
||||
{"favorites": "yes"},
|
||||
{"subfolder": "../up"},
|
||||
{"root": "-1"},
|
||||
):
|
||||
with self.subTest(bad=bad):
|
||||
with self.assertRaises(ValueError):
|
||||
params = parse_listing_filter(bad)
|
||||
# Root bounds need the roots: the resolver checks them.
|
||||
from py.api.images import _workspace_root
|
||||
|
||||
_workspace_root(params, [self.output_dir])
|
||||
|
||||
async def test_bulk_category_takes_the_listing_filter(self):
|
||||
first = self.media("keep/a.png", 300)
|
||||
second = self.media("keep/b.png", 200)
|
||||
self.media("keep/c.png", 100)
|
||||
other = self.media("skip/d.png", 400)
|
||||
prompt = self._save_prompt("in the filter")
|
||||
outside = self._save_prompt("outside the filter")
|
||||
self.api.db.link_image_to_prompt(prompt, str(first))
|
||||
self.api.db.link_image_to_prompt(prompt, str(second))
|
||||
self.api.db.link_image_to_prompt(outside, str(other))
|
||||
|
||||
response = await self.client.post(
|
||||
"/prompt_manager/bulk/category",
|
||||
json={"images_filter": {"subfolder": "keep"}, "category": "Kept"},
|
||||
)
|
||||
data = await response.json()
|
||||
|
||||
self.assertEqual(response.status, 200, data)
|
||||
self.assertEqual(
|
||||
(data["updated_count"], data["prompt_count"], data["skipped"]), (1, 1, 1)
|
||||
)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(prompt)["category"], "Kept")
|
||||
self.assertNotEqual(self.api.db.get_prompt_by_id(outside)["category"], "Kept")
|
||||
|
||||
async def test_bulk_category_filter_with_no_linked_image_changes_nothing(self):
|
||||
self.media("a.png")
|
||||
|
||||
response = await self.client.post(
|
||||
"/prompt_manager/bulk/category",
|
||||
json={"images_filter": {}, "category": "Kept"},
|
||||
)
|
||||
data = await response.json()
|
||||
|
||||
self.assertEqual(response.status, 200, data)
|
||||
self.assertEqual(
|
||||
(data["updated_count"], data["prompt_count"], data["skipped"]), (0, 0, 1)
|
||||
)
|
||||
|
||||
async def test_bulk_category_refuses_a_bad_or_doubled_filter(self):
|
||||
self.media("a.png")
|
||||
prompt = self._save_prompt("p")
|
||||
for body in (
|
||||
{"images_filter": {"sort": "sideways"}, "category": "x"},
|
||||
{"images_filter": "all", "category": "x"},
|
||||
{"images_filter": {}, "prompt_ids": [prompt], "category": "x"},
|
||||
{"images_filter": {"root": "9"}, "category": "x"},
|
||||
):
|
||||
with self.subTest(body=body):
|
||||
response = await self.client.post(
|
||||
"/prompt_manager/bulk/category", json=body
|
||||
)
|
||||
self.assertEqual(response.status, 400, await response.text())
|
||||
|
||||
async def test_bulk_category_refuses_a_renumbered_source(self):
|
||||
self.media("a.png")
|
||||
|
||||
response = await self.client.post(
|
||||
"/prompt_manager/bulk/category",
|
||||
json={
|
||||
"images_filter": {"root": "0", "root_scope": "f" * 64},
|
||||
"category": "x",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status, 409)
|
||||
self.assertEqual((await response.json())["code"], "source-changed")
|
||||
|
||||
@@ -1,696 +0,0 @@
|
||||
"""Tag output images tags each image, never its prompt (#379, PR 1).
|
||||
|
||||
``POST /prompt_manager/jobs/autotag`` runs the ``image-tags`` loop over every
|
||||
still image the output folders list, linked to a prompt or not: each image
|
||||
gets its own model tags in its image tag set and is marked auto-tagged.
|
||||
"Skip already tagged" skips images whose current bytes are marked, counted
|
||||
before the loop without opening them. Prompt tags and prompt text never
|
||||
change. No real model is loaded: the engine is a fake that answers per image.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import ntpath
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import sqlite3
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from test_api_images import make_png
|
||||
from test_image_tag_jobs import HookedTagger, ImageTagJobTestCase
|
||||
except ImportError:
|
||||
from tests.test_api_images import make_png
|
||||
from tests.test_image_tag_jobs import HookedTagger, ImageTagJobTestCase
|
||||
|
||||
from database.execution_history_schema import rotate_history_epoch
|
||||
from py import model_gate
|
||||
from py.api import admin as admin_module
|
||||
from py.api import image_tag_jobs as jobs_module
|
||||
from py.config import PromptManagerConfig
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
JOB = "/prompt_manager/jobs/autotag"
|
||||
PREVIEW = "/prompt_manager/autotag/batch/preview"
|
||||
WAIT = 30.0
|
||||
WD14 = {"general_threshold": 0.35, "character_threshold": 0.85}
|
||||
CANCELLED = "Cancelled. Tags already added were kept."
|
||||
_DRIVE = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
|
||||
|
||||
class PerImageTagger(HookedTagger):
|
||||
"""Answers each image from its own bytes: ``by_bytes`` maps exact file
|
||||
contents to the names the "model" sees; any other image gets SCORED."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.by_bytes = {}
|
||||
|
||||
def generate_tag_suggestions(self, image_path, **kwargs):
|
||||
result = super().generate_tag_suggestions(image_path, **kwargs)
|
||||
names = self.by_bytes.get(self.read_bytes[-1])
|
||||
if names is None:
|
||||
return result
|
||||
return [(name, 0.9) for name in names][: kwargs["limit"]]
|
||||
|
||||
|
||||
def _strings(value):
|
||||
if isinstance(value, str):
|
||||
yield value
|
||||
elif isinstance(value, dict):
|
||||
for item in value.values():
|
||||
yield from _strings(item)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
for item in value:
|
||||
yield from _strings(item)
|
||||
|
||||
|
||||
def _absolute(text):
|
||||
return posixpath.isabs(text) or ntpath.isabs(text) or bool(_DRIVE.match(text))
|
||||
|
||||
|
||||
class BatchTestCase(ImageTagJobTestCase):
|
||||
async def get_application(self):
|
||||
app = await super().get_application()
|
||||
self.tagger = PerImageTagger()
|
||||
patcher = patch("py.autotag.get_autotag_service", return_value=self.tagger)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
backup = PromptManagerConfig.SAFETY_BACKUP
|
||||
self.addCleanup(setattr, PromptManagerConfig, "SAFETY_BACKUP", backup)
|
||||
PromptManagerConfig.SAFETY_BACKUP = False
|
||||
return app
|
||||
|
||||
# -- helpers ---------------------------------------------------------
|
||||
|
||||
def image(self, name, color, sees=None):
|
||||
path = make_png(self.output_dir / name, size=(8, 6), color=color)
|
||||
if sees is not None:
|
||||
self.tagger.by_bytes[path.read_bytes()] = list(sees)
|
||||
return path
|
||||
|
||||
async def relist(self):
|
||||
"""Walk the output folder again (a cached listing is used as it is)."""
|
||||
await self.api._listing(self.output_dir, fresh=True)
|
||||
|
||||
def prompt(self, text, tags=()):
|
||||
return self.api.db.save_prompt(
|
||||
text=text,
|
||||
category=None,
|
||||
tags=list(tags),
|
||||
rating=None,
|
||||
prompt_hash=generate_prompt_hash(text),
|
||||
)
|
||||
|
||||
def link(self, prompt_id, path):
|
||||
return self.api.db.link_image_to_prompt(prompt_id, str(path))
|
||||
|
||||
def prompt_rows(self):
|
||||
conn = sqlite3.connect(self._temp_db.name)
|
||||
try:
|
||||
return {
|
||||
table: conn.execute(f"SELECT * FROM {table} ORDER BY 1,2").fetchall()
|
||||
for table in ("prompts", "prompt_tags")
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def marked(self):
|
||||
conn = sqlite3.connect(self._temp_db.name)
|
||||
try:
|
||||
return {
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT f.locator FROM media_image_autotags m JOIN "
|
||||
"media_annotation_bytes b ON b.id=m.byte_id JOIN "
|
||||
"media_annotation_files f ON f.id=b.file_id"
|
||||
)
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
async def preview(self, existing="skip", model_type="wd14-vit"):
|
||||
response = await self.client.post(
|
||||
PREVIEW, json={"model_type": model_type, "existing": existing}
|
||||
)
|
||||
return response.status, await self.json_of(response)
|
||||
|
||||
async def start_batch(self, existing="skip", model_type="wd14-vit", options=None):
|
||||
if options is None:
|
||||
options = (
|
||||
dict(WD14)
|
||||
if model_type.startswith("wd14")
|
||||
else {"instructions": "Tags please."}
|
||||
)
|
||||
response = await self.client.post(
|
||||
JOB,
|
||||
json={"model_type": model_type, "existing": existing, "options": options},
|
||||
)
|
||||
return response.status, await self.json_of(response)
|
||||
|
||||
async def run_batch(self, **kwargs):
|
||||
status, data = await self.start_batch(**kwargs)
|
||||
self.assertEqual(status, 202, data)
|
||||
job = self.api._jobs.get(data["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
return job
|
||||
|
||||
|
||||
class TestPerImageWrites(BatchTestCase):
|
||||
async def test_two_images_of_one_prompt_get_their_own_tags_and_no_prompt_changes(
|
||||
self,
|
||||
):
|
||||
prompt_id = self.prompt("a girl at the beach", tags=["beach"])
|
||||
a = self.image("a.png", (10, 20, 30), sees=["1girl", "bird"])
|
||||
b = self.image("b.png", (40, 50, 60), sees=["1girl", "waves"])
|
||||
self.link(prompt_id, a)
|
||||
self.link(prompt_id, b)
|
||||
before = self.prompt_rows()
|
||||
|
||||
job = await self.run_batch(existing="skip")
|
||||
|
||||
self.assertEqual(job.state, "finished", job.status_line)
|
||||
self.assertEqual(self.stored("a.png")["tags"], ["1girl", "bird"])
|
||||
self.assertEqual(self.stored("b.png")["tags"], ["1girl", "waves"])
|
||||
self.assertEqual(self.prompt_rows(), before)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(prompt_id)["tags"], ["beach"])
|
||||
self.assertEqual(self.marked(), {self.locator("a.png"), self.locator("b.png")})
|
||||
self.assertEqual(
|
||||
job.counters, {"tagged": 2, "unchanged": 0, "skipped": 0, "errors": 0}
|
||||
)
|
||||
self.assertEqual(job.status_line, "Tagged 2 images")
|
||||
self.assertEqual((job.processed, job.total, job.progress), (2, 2, 100))
|
||||
self.assertEqual(job.scope["images"], 2)
|
||||
# Inference read the private captures, one per image.
|
||||
self.assertEqual(len(self.tagger.calls), 2)
|
||||
|
||||
async def test_unlinked_images_are_tagged_and_videos_are_not_targets(self):
|
||||
self.image("free.png", (1, 2, 3), sees=["tree"])
|
||||
(self.output_dir / "clip.mp4").write_bytes(b"\x00\x00\x00\x18ftypmp42" * 8)
|
||||
|
||||
job = await self.run_batch()
|
||||
|
||||
self.assertEqual(self.stored("free.png")["tags"], ["tree"])
|
||||
self.assertEqual(job.total, 1)
|
||||
self.assertEqual(job.status_line, "Tagged 1 image")
|
||||
rows = [row["path"] for rows in job.outcomes.values() for row in rows]
|
||||
self.assertNotIn("clip.mp4", rows)
|
||||
|
||||
async def test_hand_tags_do_not_block_a_skip_run(self):
|
||||
path = self.image("c.png", (7, 7, 7), sees=["bird"])
|
||||
self.saved_set("c.png", path.read_bytes(), ["keeper"])
|
||||
|
||||
job = await self.run_batch(existing="skip")
|
||||
|
||||
self.assertEqual(self.stored("c.png")["tags"], ["keeper", "bird"])
|
||||
self.assertEqual(job.counters["tagged"], 1)
|
||||
|
||||
async def test_skip_counts_marked_images_without_opening_them_and_merge_retags(
|
||||
self,
|
||||
):
|
||||
self.image("a.png", (1, 1, 1), sees=["bird"])
|
||||
self.image("b.png", (2, 2, 2), sees=["cat"])
|
||||
first = await self.run_batch()
|
||||
self.assertEqual(first.counters["tagged"], 2)
|
||||
self.tagger.calls.clear()
|
||||
|
||||
captures = []
|
||||
store = self.api._capture_store()
|
||||
real_capture = store.capture
|
||||
|
||||
def spy(*args, **kwargs):
|
||||
captures.append(args)
|
||||
return real_capture(*args, **kwargs)
|
||||
|
||||
with patch.object(store, "capture", side_effect=spy):
|
||||
skipped = await self.run_batch(existing="skip")
|
||||
self.assertEqual(captures, [])
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
self.assertEqual(
|
||||
skipped.counters, {"tagged": 0, "unchanged": 0, "skipped": 2, "errors": 0}
|
||||
)
|
||||
self.assertEqual(
|
||||
sorted(self.rows(skipped, "skipped")),
|
||||
[("a.png", "Already auto-tagged"), ("b.png", "Already auto-tagged")],
|
||||
)
|
||||
self.assertFalse(any("Already auto-tagged" in line for line in skipped.log))
|
||||
self.assertEqual((skipped.processed, skipped.total), (2, 2))
|
||||
self.assertEqual(skipped.status_line, "Tagged 0 images")
|
||||
|
||||
self.tagger.by_bytes[(self.output_dir / "a.png").read_bytes()] = ["bird", "sky"]
|
||||
merged = await self.run_batch(existing="merge")
|
||||
self.assertEqual(len(self.tagger.calls), 2)
|
||||
self.assertEqual(self.stored("a.png")["tags"], ["bird", "sky"])
|
||||
self.assertEqual(self.rows(merged, "tagged"), [("a.png", "Added 1 tag")])
|
||||
self.assertEqual(self.rows(merged, "unchanged"), [("b.png", "No new tags")])
|
||||
|
||||
async def test_changed_bytes_are_tagged_again(self):
|
||||
self.image("a.png", (1, 1, 1), sees=["bird"])
|
||||
await self.run_batch()
|
||||
self.image("a.png", (9, 9, 9), sees=["cat"])
|
||||
job = await self.run_batch(existing="skip")
|
||||
self.assertEqual(self.rows(job, "tagged"), [("a.png", "Added 1 tag")])
|
||||
self.assertEqual(self.stored("a.png")["tags"], ["cat"])
|
||||
|
||||
async def test_an_image_with_no_model_tags_is_marked_and_skipped_next_time(self):
|
||||
self.image("plain.png", (3, 3, 3), sees=[])
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(self.rows(job, "unchanged"), [("plain.png", "No new tags")])
|
||||
self.assertEqual(self.marked(), {self.locator("plain.png")})
|
||||
again = await self.run_batch()
|
||||
self.assertEqual(
|
||||
self.rows(again, "skipped"), [("plain.png", "Already auto-tagged")]
|
||||
)
|
||||
|
||||
async def test_unsupported_still_is_marked_and_skipped_by_the_next_run(self):
|
||||
(self.output_dir / "broken.png").write_bytes(b"not an image")
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(
|
||||
self.rows(job, "skipped"), [("broken.png", "Not a supported still image")]
|
||||
)
|
||||
self.assertEqual(self.marked(), {self.locator("broken.png")})
|
||||
again = await self.run_batch()
|
||||
self.assertEqual(
|
||||
self.rows(again, "skipped"), [("broken.png", "Already auto-tagged")]
|
||||
)
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
|
||||
async def test_too_large_is_skipped_before_any_copy(self):
|
||||
self.image("big.png", (5, 5, 5))
|
||||
self.api._capture_store().max_file = 16
|
||||
store = self.api._capture_store()
|
||||
with patch.object(store, "capture", side_effect=AssertionError("copied")):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(
|
||||
self.rows(job, "skipped"), [("big.png", "Too large to verify")]
|
||||
)
|
||||
self.assertEqual(self.marked(), set())
|
||||
|
||||
async def test_overlapping_roots_list_a_file_once(self):
|
||||
self.image("sub/a.png", (4, 4, 4), sees=["bird"])
|
||||
sub = self.output_dir / "sub"
|
||||
self.api._get_all_output_dirs = lambda: [self.output_dir, sub]
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.total, 1)
|
||||
self.assertEqual(len(self.tagger.calls), 1)
|
||||
self.assertEqual(self.rows(job, "tagged"), [("sub/a.png", "Added 1 tag")])
|
||||
status, data = await self.preview()
|
||||
self.assertEqual((status, data["images"]), (200, 1))
|
||||
|
||||
async def test_rows_log_and_status_carry_no_absolute_paths(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
(self.output_dir / "broken.png").write_bytes(b"not an image")
|
||||
job = await self.run_batch()
|
||||
leaked = [s for s in _strings(job.report()) if _absolute(s)]
|
||||
self.assertEqual(leaked, [])
|
||||
self.assertNotIn(str(self.output_dir), json.dumps(job.report()))
|
||||
|
||||
|
||||
class TestBatchLifecycle(BatchTestCase):
|
||||
async def test_cancel_after_one_image_keeps_it_and_the_rerun_skips_it(self):
|
||||
for n in range(3):
|
||||
self.image(f"{n}.png", (n, n, n), sees=[f"t{n}"])
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
status, data = await self.start_batch()
|
||||
self.assertEqual(status, 202, data)
|
||||
job = self.api._jobs.get(data["job"]["id"])
|
||||
try:
|
||||
await self.wait_for(entered)
|
||||
cancel = await self.client.post(f"/prompt_manager/jobs/{job.id}/cancel")
|
||||
self.assertEqual(cancel.status, 200)
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.status_line, CANCELLED)
|
||||
self.assertEqual(job.counters["tagged"], 1)
|
||||
self.assertEqual(len(self.marked()), 1)
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
lines = []
|
||||
real_update = type(job).update
|
||||
|
||||
def spy(self_job, **kwargs):
|
||||
if kwargs.get("status_line"):
|
||||
lines.append(kwargs["status_line"])
|
||||
return real_update(self_job, **kwargs)
|
||||
|
||||
with patch.object(type(job), "update", spy):
|
||||
rerun = await self.run_batch()
|
||||
self.assertEqual(rerun.counters["skipped"], 1)
|
||||
self.assertEqual(rerun.counters["tagged"], 2)
|
||||
self.assertIn("Processing file 2 of 3", lines)
|
||||
self.assertIn("Processing file 3 of 3", lines)
|
||||
self.assertNotIn("Processing file 1 of 3", lines)
|
||||
self.assertEqual((rerun.processed, rerun.total), (3, 3))
|
||||
|
||||
async def test_cancel_in_each_phase_writes_nothing(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
for phase in (
|
||||
"_safety_backup_step",
|
||||
"_prepare_tagging_model",
|
||||
"_autotag_targets",
|
||||
):
|
||||
with self.subTest(phase):
|
||||
real = getattr(self.api, phase)
|
||||
|
||||
async def cancelling(*args, _real=real, **kwargs):
|
||||
result = await _real(*args, **kwargs)
|
||||
self.api._jobs.active("autotag").cancelled = True
|
||||
return result
|
||||
|
||||
with patch.object(self.api, phase, side_effect=cancelling):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.status_line, CANCELLED)
|
||||
self.assertEqual(self.marked(), set())
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_cancel_between_prepass_chunks_keeps_counts(self):
|
||||
for n in range(3):
|
||||
self.image(f"{n}.png", (n, n, n))
|
||||
await self.run_batch()
|
||||
seen = []
|
||||
real = jobs_module._prepass_chunk
|
||||
|
||||
def chunk(db, items, skip_marked, max_file):
|
||||
result = real(db, items, skip_marked, max_file)
|
||||
job = self.api._jobs.active("autotag")
|
||||
seen.append(job.processed)
|
||||
job.cancelled = True # cancel after the first chunk
|
||||
return result
|
||||
|
||||
with (
|
||||
patch.object(jobs_module, "PREPASS_CHUNK", 1),
|
||||
patch.object(jobs_module, "_prepass_chunk", side_effect=chunk),
|
||||
):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.counters["skipped"], 1)
|
||||
self.assertEqual((job.processed, job.total), (1, 3))
|
||||
self.assertEqual(seen, [0])
|
||||
|
||||
async def test_the_job_ends_only_after_its_cleanup(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
PromptManagerConfig.SAFETY_BACKUP = True
|
||||
states = []
|
||||
|
||||
async def prune(job, _db):
|
||||
states.append((job.state, model_gate.holder() is not None))
|
||||
|
||||
with (
|
||||
patch("py.api.maintenance.take_safety_backup", return_value="db.backup_x"),
|
||||
patch.object(self.api, "_prune_backup_pool", side_effect=prune),
|
||||
):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "finished")
|
||||
self.assertEqual(states, [("running", True)])
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_prepass_publishes_its_progress(self):
|
||||
for n in range(2):
|
||||
self.image(f"{n}.png", (n, n, n))
|
||||
await self.run_batch()
|
||||
published = []
|
||||
real = type(self.api._jobs).start
|
||||
|
||||
def start(registry, *args, **kwargs):
|
||||
job = real(registry, *args, **kwargs)
|
||||
original = job.publish
|
||||
job.publish = lambda payload: (
|
||||
published.append(dict(payload)),
|
||||
original(payload),
|
||||
)
|
||||
return job
|
||||
|
||||
with patch.object(type(self.api._jobs), "start", start):
|
||||
await self.run_batch()
|
||||
self.assertIn({"type": "progress", "progress": 100}, published)
|
||||
self.assertEqual(published[-1], {"type": "finished", "progress": 100})
|
||||
|
||||
async def test_restore_mid_run_fails_the_job(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
self.image("b.png", (2, 2, 2))
|
||||
|
||||
def restore(_path):
|
||||
conn = sqlite3.connect(self._temp_db.name)
|
||||
try:
|
||||
rotate_history_epoch(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
self.tagger.on_generate = restore
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertEqual(
|
||||
job.status_line, "Prompt storage was restored; start the job again."
|
||||
)
|
||||
|
||||
async def test_listing_error_fails_the_job(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
with patch.object(
|
||||
self.api, "_get_gallery_files", side_effect=OSError("/secret/out")
|
||||
):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertEqual(
|
||||
job.status_line, "The output folders could not be listed. Try again."
|
||||
)
|
||||
self.assertNotIn("/secret", json.dumps(job.report()))
|
||||
|
||||
async def test_model_that_cannot_load_fails_the_job(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
self.tagger.load_error = RuntimeError("no CUDA")
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "failed")
|
||||
self.assertEqual(
|
||||
job.status_line, "Auto-tagging failed. Check server logs for details."
|
||||
)
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_zero_targets_finish_with_tagged_zero(self):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.state, "finished")
|
||||
self.assertEqual(job.status_line, "Tagged 0 images")
|
||||
self.assertEqual((job.processed, job.total), (0, 0))
|
||||
|
||||
async def test_runner_unloads_after_a_cancel_when_not_kept_in_memory(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
owner = model_gate.new_owner("legacy-batch")
|
||||
self.assertTrue(model_gate.try_acquire(owner))
|
||||
run = jobs_module.AutotagRun(
|
||||
model_type="wd14-vit",
|
||||
existing="skip",
|
||||
options={**WD14, "use_gpu": True, "suggestion_limit": 10},
|
||||
keep_in_memory=False,
|
||||
backup_on=False,
|
||||
owner=owner,
|
||||
)
|
||||
|
||||
from py.api.jobs import AUTOTAG_GROUP_NAMES, Job as RealJob
|
||||
|
||||
job = RealJob(
|
||||
"autotag",
|
||||
"Tag output images",
|
||||
admission="records",
|
||||
groups=AUTOTAG_GROUP_NAMES,
|
||||
)
|
||||
real_targets = self.api._autotag_targets
|
||||
|
||||
async def cancel_after_listing():
|
||||
targets = await real_targets()
|
||||
job.cancelled = True
|
||||
return targets
|
||||
|
||||
with patch.object(
|
||||
self.api, "_autotag_targets", side_effect=cancel_after_listing
|
||||
):
|
||||
await self.api._run_autotag_batch(run, job)
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(self.tagger.unload_calls, 1)
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_time_left_uses_the_models_rate_only(self):
|
||||
for n in range(4):
|
||||
self.image(f"{n}.png", (n, n, n))
|
||||
await self.run_batch()
|
||||
for n in range(4, 8):
|
||||
self.image(f"{n}.png", (n, n, n))
|
||||
await self.relist()
|
||||
calls = []
|
||||
real = jobs_module._time_left
|
||||
|
||||
def spy(*args):
|
||||
calls.append(args)
|
||||
return real(*args)
|
||||
|
||||
with patch.object(jobs_module, "_time_left", side_effect=spy):
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(job.counters["skipped"], 4)
|
||||
# (done, total, batches, elapsed): over the 4 images the model runs on.
|
||||
self.assertEqual(
|
||||
[(c[0], c[1], c[2]) for c in calls], [(n, 4, n) for n in range(4)]
|
||||
)
|
||||
# The shared D56 text is unchanged.
|
||||
self.assertEqual(admin_module._time_left(5, 10, 5, 300), " · about 5 min left")
|
||||
|
||||
|
||||
class TestStartRefusals(BatchTestCase):
|
||||
async def test_unavailable_model_and_bad_bodies(self):
|
||||
self.tagger.downloaded["wd14-vit"] = False
|
||||
status, data = await self.start_batch()
|
||||
self.assertEqual((status, data["code"]), (409, "model-unavailable"))
|
||||
status, data = await self.start_batch(existing="replace_after_review")
|
||||
self.assertEqual((status, data["code"]), (400, "review-required"))
|
||||
status, _data = await self.start_batch(options={**WD14, "suggestion_limit": 3})
|
||||
self.assertEqual(status, 400)
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
async def test_model_busy_when_the_gate_is_held(self):
|
||||
owner = model_gate.new_owner("image-annotations")
|
||||
self.assertTrue(model_gate.try_acquire(owner))
|
||||
try:
|
||||
status, data = await self.start_batch()
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
self.assertEqual((status, data["code"]), (409, "model-busy"))
|
||||
|
||||
|
||||
class TestBatchBeside(BatchTestCase):
|
||||
async def test_legacy_batch_and_unload_are_refused_while_the_job_runs(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
status, data = await self.start_batch()
|
||||
self.assertEqual(status, 202, data)
|
||||
job = self.api._jobs.get(data["job"]["id"])
|
||||
try:
|
||||
await self.wait_for(entered)
|
||||
self.assertTrue(model_gate.holder().startswith("autotag-job:"))
|
||||
legacy = await self.client.post("/prompt_manager/autotag/start")
|
||||
self.assertEqual(legacy.status, 200)
|
||||
frames = [
|
||||
json.loads(line[len("data: ") :])
|
||||
for line in (await legacy.text()).splitlines()
|
||||
if line.startswith("data: ")
|
||||
]
|
||||
self.assertEqual(
|
||||
frames,
|
||||
[
|
||||
{
|
||||
"type": "error",
|
||||
"status": "error",
|
||||
"message": "Tag output images already running",
|
||||
}
|
||||
],
|
||||
)
|
||||
unload = await self.client.post("/prompt_manager/autotag/unload")
|
||||
self.assertEqual((await unload.json())["operation"], "Tag output images")
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
self.assertEqual(job.state, "finished")
|
||||
|
||||
async def test_language_model_batch_passes_instructions_per_call(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
job = await self.run_batch(model_type="gguf")
|
||||
self.assertEqual(job.state, "finished", job.status_line)
|
||||
self.assertEqual(self.tagger.calls[0]["prompt"], "Tags please.")
|
||||
self.assertEqual(self.tagger.custom_prompt, "legacy prompt")
|
||||
|
||||
async def test_inference_errors_are_fixed_strings_and_not_marked(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
|
||||
def explode(_path):
|
||||
raise RuntimeError("/srv/secret/model exploded")
|
||||
|
||||
self.tagger.on_generate = explode
|
||||
job = await self.run_batch()
|
||||
self.assertEqual(
|
||||
self.rows(job, "errors"),
|
||||
[("a.png", "Auto-tagging failed. Check server logs for details.")],
|
||||
)
|
||||
self.assertNotIn("/srv/secret", json.dumps(job.report()))
|
||||
self.assertEqual(self.marked(), set())
|
||||
|
||||
|
||||
class TestPreview(BatchTestCase):
|
||||
async def test_preview_counts_images_and_estimates_time(self):
|
||||
prompt_id = self.prompt("p")
|
||||
a = self.image("a.png", (1, 1, 1))
|
||||
self.image("b.png", (2, 2, 2))
|
||||
self.image("c.png", (3, 3, 3))
|
||||
self.link(prompt_id, a)
|
||||
(self.output_dir / "clip.mp4").write_bytes(b"\x00\x00\x00\x18ftypmp42" * 8)
|
||||
status, fresh = await self.preview()
|
||||
self.assertEqual(status, 200, fresh)
|
||||
self.assertEqual(
|
||||
{k: fresh[k] for k in ("images", "already_tagged", "to_tag")},
|
||||
{"images": 3, "already_tagged": 0, "to_tag": 3},
|
||||
)
|
||||
self.assertNotIn("prompts", fresh)
|
||||
self.assertEqual(fresh["suggestion_limit"], 10)
|
||||
self.assertEqual(fresh["estimate_device"], "cpu")
|
||||
self.assertAlmostEqual(fresh["estimate_seconds"], 3 * 1.6)
|
||||
|
||||
self.tagger.block = None
|
||||
await self.run_batch()
|
||||
self.image("d.png", (4, 4, 4))
|
||||
await self.relist()
|
||||
skip = (await self.preview("skip"))[1]
|
||||
self.assertEqual(
|
||||
{k: skip[k] for k in ("images", "already_tagged", "to_tag")},
|
||||
{"images": 4, "already_tagged": 3, "to_tag": 1},
|
||||
)
|
||||
merge = (await self.preview("merge"))[1]
|
||||
self.assertEqual(merge["to_tag"], 4)
|
||||
for model_type in ("gguf", "hf"):
|
||||
llm = (await self.preview(model_type=model_type))[1]
|
||||
self.assertIsNone(llm["estimate_seconds"])
|
||||
self.assertIsNone(llm["estimate_device"])
|
||||
self.assertEqual([s for s in _strings(skip) if _absolute(s)], [])
|
||||
|
||||
async def test_preview_uses_the_graphics_card_rate_when_wd14_runs_on_cuda(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
self.tagger.wd14_device = lambda: "gpu"
|
||||
data = (await self.preview())[1]
|
||||
self.assertEqual(data["estimate_device"], "gpu")
|
||||
from py.tagging_defaults import WD14_GPU_SECONDS_PER_IMAGE
|
||||
|
||||
self.assertAlmostEqual(data["estimate_seconds"], WD14_GPU_SECONDS_PER_IMAGE)
|
||||
|
||||
async def test_preview_writes_nothing_and_opens_no_media_file(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
await self.run_batch()
|
||||
self.image("b.png", (2, 2, 2))
|
||||
await self.relist() # the listing, primed
|
||||
files = [self._temp_db.name + suffix for suffix in ("", "-wal")]
|
||||
before = [Path(f).read_bytes() if os.path.exists(f) else None for f in files]
|
||||
media = str(self.output_dir)
|
||||
real_open, real_stat = open, os.stat
|
||||
|
||||
def guarded_open(file, *args, **kwargs):
|
||||
if str(file).startswith(media):
|
||||
raise AssertionError(f"opened {file}")
|
||||
return real_open(file, *args, **kwargs)
|
||||
|
||||
def guarded_stat(path, *args, **kwargs):
|
||||
if str(path).startswith(media) and str(path) != media:
|
||||
raise AssertionError(f"stat {path}")
|
||||
return real_stat(path, *args, **kwargs)
|
||||
|
||||
with patch("builtins.open", guarded_open), patch("os.stat", guarded_stat):
|
||||
status, data = await self.preview()
|
||||
self.assertEqual(status, 200, data)
|
||||
after = [Path(f).read_bytes() if os.path.exists(f) else None for f in files]
|
||||
self.assertEqual(after, before)
|
||||
self.assertEqual(data["to_tag"], 1)
|
||||
self.assertEqual(self.tagger.calls[1:], [])
|
||||
@@ -884,29 +884,3 @@ class TestImageConversion(EngineTestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestWd14Device(EngineTestCase):
|
||||
"""Where WD14 runs, for the Tag output images estimate (#379)."""
|
||||
|
||||
def test_loaded_session_answers_from_its_first_provider(self):
|
||||
cpu = FakeOnnxSession([0.5] * len(WD14_TAGS))
|
||||
service = self._service(("wd14", (cpu, WD14_TAGS)))
|
||||
_touch_model_files(self.models_dir, "wd14-vit")
|
||||
service.load_model("wd14-vit")
|
||||
self.assertEqual(service.wd14_device(), "cpu")
|
||||
cpu.get_providers = lambda: ["CUDAExecutionProvider", "CPUExecutionProvider"]
|
||||
self.assertEqual(service.wd14_device(), "gpu")
|
||||
|
||||
def test_without_a_session_the_available_providers_answer(self):
|
||||
service = self._service(None)
|
||||
fake = types.SimpleNamespace(
|
||||
get_available_providers=lambda: ["CUDAExecutionProvider"]
|
||||
)
|
||||
with patch.dict(sys.modules, {"onnxruntime": fake}):
|
||||
self.assertEqual(service.wd14_device(), "gpu")
|
||||
fake.get_available_providers = lambda: ["CPUExecutionProvider"]
|
||||
with patch.dict(sys.modules, {"onnxruntime": fake}):
|
||||
self.assertEqual(service.wd14_device(), "cpu")
|
||||
with patch.dict(sys.modules, {"onnxruntime": None}):
|
||||
self.assertEqual(service.wd14_device(), "cpu")
|
||||
|
||||
@@ -1,617 +0,0 @@
|
||||
"""The legacy pages and the Tag image dialog tag images, never prompts (#379, PR 2).
|
||||
|
||||
- ``POST /autotag/start`` (legacy admin and gallery batch) starts the
|
||||
``autotag`` job titled "Tag output images (legacy page)" and relays it as
|
||||
the legacy frames; closing the stream cancels it.
|
||||
- ``POST /autotag/single`` suggests on a capture with the saved cap and
|
||||
answers the captured ``sha256``; ``existing: "skip"`` answers
|
||||
``already_tagged`` for an auto-tagged image without the model gate.
|
||||
- ``POST /autotag/apply`` unions into that image's tags and marks it; a
|
||||
``prompt_id`` body is refused.
|
||||
- A Tag image dialog save after Generate suggestions marks the image.
|
||||
|
||||
No real model runs: the per-image fake from test_autotag_batch answers.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
from unittest.mock import patch
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
try:
|
||||
from test_autotag_batch import BatchTestCase, _absolute, _strings
|
||||
except ImportError:
|
||||
from tests.test_autotag_batch import BatchTestCase, _absolute, _strings
|
||||
|
||||
from database.execution_history_schema import rotate_history_epoch
|
||||
from py import model_gate
|
||||
from py.api import autotag_routes, image_tag_jobs
|
||||
from py.config import AutotagConfig
|
||||
|
||||
START = "/prompt_manager/autotag/start"
|
||||
SINGLE = "/prompt_manager/autotag/single"
|
||||
APPLY = "/prompt_manager/autotag/apply"
|
||||
SUGGEST = "/prompt_manager/image-annotations/suggest"
|
||||
OBSERVE = "/prompt_manager/image-annotations/observe"
|
||||
TAGS = "/prompt_manager/image-annotations/tags"
|
||||
WAIT = 30.0
|
||||
LEGACY_TITLE = "Tag output images (legacy page)"
|
||||
|
||||
|
||||
def frames_of(text):
|
||||
return [
|
||||
json.loads(line[len("data: ") :])
|
||||
for line in text.splitlines()
|
||||
if line.startswith("data: ")
|
||||
]
|
||||
|
||||
|
||||
class LegacyTestCase(BatchTestCase):
|
||||
async def legacy(self, query="model_type=wd14-vit&skip_tagged=true"):
|
||||
response = await self.client.post(f"{START}?{query}")
|
||||
return response, frames_of(await response.text())
|
||||
|
||||
def digest(self, path):
|
||||
return hashlib.sha256((self.output_dir / path).read_bytes()).hexdigest()
|
||||
|
||||
async def single(self, body):
|
||||
response = await self.client.post(SINGLE, json=body)
|
||||
return response.status, await response.json()
|
||||
|
||||
async def apply(self, body):
|
||||
response = await self.client.post(APPLY, json=body)
|
||||
return response.status, await response.json()
|
||||
|
||||
def snapshot_prompts(self):
|
||||
conn = sqlite3.connect(self._temp_db.name)
|
||||
try:
|
||||
return [
|
||||
conn.execute(f"SELECT * FROM {t} ORDER BY 1,2").fetchall()
|
||||
for t in ("prompts", "prompt_tags")
|
||||
]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class TestLegacyBatch(LegacyTestCase):
|
||||
async def test_the_legacy_batch_tags_each_image_and_relays_frames(self):
|
||||
prompt = self.prompt("one prompt", tags=["beach"])
|
||||
a = self.image("a.png", (1, 2, 3), sees=["bird"])
|
||||
b = self.image("b.png", (4, 5, 6), sees=["waves"])
|
||||
self.link(prompt, a)
|
||||
self.link(prompt, b)
|
||||
before = self.snapshot_prompts()
|
||||
ticks = iter(range(1000))
|
||||
with patch.object(
|
||||
autotag_routes, "_monotonic", side_effect=lambda: next(ticks)
|
||||
):
|
||||
response, frames = await self.legacy()
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(self.stored("a.png")["tags"], ["bird"])
|
||||
self.assertEqual(self.stored("b.png")["tags"], ["waves"])
|
||||
self.assertEqual(self.snapshot_prompts(), before)
|
||||
phases = [f["status"] for f in frames if f.get("progress") in (0, 5, 8)]
|
||||
self.assertEqual(
|
||||
phases,
|
||||
sorted(
|
||||
phases,
|
||||
key=[
|
||||
"Taking a safety backup…",
|
||||
"Loading model…",
|
||||
"Listing output images…",
|
||||
].index,
|
||||
),
|
||||
)
|
||||
self.assertIn("Listing output images…", phases)
|
||||
rows = [f for f in frames if f["type"] == "progress" and "processed" in f]
|
||||
self.assertTrue(rows)
|
||||
for row in rows:
|
||||
self.assertEqual(
|
||||
set(row),
|
||||
{"type", "progress", "status", "processed", "tagged", "skipped"},
|
||||
)
|
||||
self.assertTrue(10 <= row["progress"] <= 95, row)
|
||||
self.assertEqual(
|
||||
frames[-1],
|
||||
{
|
||||
"type": "complete",
|
||||
"progress": 100,
|
||||
"processed": 2,
|
||||
"tagged": 2,
|
||||
"skipped": 0,
|
||||
"errors": 0,
|
||||
"skipped_outside_roots": 0,
|
||||
"status": "Complete",
|
||||
"model_status": "Model kept in memory",
|
||||
"model_loaded": True,
|
||||
},
|
||||
)
|
||||
job = self.api._jobs.list()[0]
|
||||
self.assertEqual(
|
||||
(job.title, job.kind, job.state), (LEGACY_TITLE, "autotag", "finished")
|
||||
)
|
||||
self.assertEqual([s for s in _strings(frames) if _absolute(s)], [])
|
||||
|
||||
# Skip: the marked images are skipped before the model runs.
|
||||
self.tagger.calls.clear()
|
||||
_, again = await self.legacy()
|
||||
self.assertEqual(again[-1]["skipped"], 2)
|
||||
self.assertEqual(again[-1]["processed"], 0)
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
|
||||
async def test_no_still_images_and_keep_in_memory_false(self):
|
||||
_, frames = await self.legacy("model_type=wd14-vit&keep_in_memory=false")
|
||||
self.assertEqual(frames[-1]["status"], "No still images found")
|
||||
self.assertEqual(frames[-1]["model_loaded"], False)
|
||||
self.assertEqual(frames[-1]["model_status"], "Model unloaded")
|
||||
self.assertEqual(self.tagger.unload_calls, 1)
|
||||
|
||||
async def test_language_model_options_come_from_the_page_or_the_template(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
await self.legacy("model_type=gguf&prompt=Tags%20only")
|
||||
self.assertEqual(self.tagger.calls[-1]["prompt"], "Tags only")
|
||||
self.assertEqual(self.tagger.calls[-1]["limit"], 10)
|
||||
self.image("b.png", (2, 2, 2))
|
||||
await self.relist()
|
||||
await self.legacy("model_type=gguf&prompt=%20")
|
||||
self.assertEqual(self.tagger.calls[-1]["prompt"], AutotagConfig.TEMPLATE)
|
||||
self.assertEqual(self.tagger.custom_prompt, "legacy prompt")
|
||||
|
||||
async def test_each_refusal_is_one_error_frame(self):
|
||||
self.tagger.downloaded["wd14-vit"] = False
|
||||
_, frames = await self.legacy()
|
||||
self.assertEqual(len(frames), 1)
|
||||
self.assertEqual(frames[0]["type"], "error")
|
||||
self.assertEqual(frames[0]["status"], "error")
|
||||
self.assertIn("WD14 ViT is not downloaded", frames[0]["message"])
|
||||
self.tagger.downloaded["wd14-vit"] = True
|
||||
|
||||
_, frames = await self.legacy("model_type=wd14-vit&general_threshold=2")
|
||||
self.assertEqual(
|
||||
frames,
|
||||
[
|
||||
{
|
||||
"type": "error",
|
||||
"status": "error",
|
||||
"message": "general_threshold must be between 0 and 1",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
owner = model_gate.new_owner("image-annotations")
|
||||
self.assertTrue(model_gate.try_acquire(owner))
|
||||
try:
|
||||
_, frames = await self.legacy()
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
self.assertEqual(frames[0]["message"], autotag_routes.MODEL_BUSY_ERROR)
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
self.assertIsNone(model_gate.holder())
|
||||
|
||||
response = await self.client.post(f"{START}?general_threshold=abc")
|
||||
self.assertEqual(response.status, 400)
|
||||
|
||||
async def test_busy_jobs_refuse_the_legacy_batch(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
status, data = await self.start_batch()
|
||||
self.assertEqual(status, 202, data)
|
||||
job = self.api._jobs.get(data["job"]["id"])
|
||||
try:
|
||||
await self.wait_for(entered)
|
||||
_, frames = await self.legacy()
|
||||
self.assertEqual(
|
||||
frames,
|
||||
[
|
||||
{
|
||||
"type": "error",
|
||||
"status": "error",
|
||||
"message": "Tag output images already running",
|
||||
}
|
||||
],
|
||||
)
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
|
||||
# A second legacy batch keeps its JSON 409.
|
||||
self.image("b.png", (2, 2, 2))
|
||||
await self.relist()
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
first = asyncio.ensure_future(
|
||||
self.legacy("model_type=wd14-vit&skip_tagged=false")
|
||||
)
|
||||
try:
|
||||
await self.wait_for(entered)
|
||||
second = await self.client.post(f"{START}?model_type=wd14-vit")
|
||||
self.assertEqual(second.status, 409)
|
||||
self.assertEqual(
|
||||
await second.json(),
|
||||
{"success": False, "error": "Auto-tagging is already running"},
|
||||
)
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
_, frames = await first
|
||||
self.assertEqual(frames[-1]["type"], "complete")
|
||||
|
||||
async def test_a_legacy_start_that_loses_the_race_gets_the_json_409(self):
|
||||
# Two starts can both pass the title check before either registers its
|
||||
# job; the loser's busy refusal names the legacy batch and answers the
|
||||
# same JSON 409 as a second legacy batch.
|
||||
refusal = image_tag_jobs.Refusal(
|
||||
409,
|
||||
"busy",
|
||||
f"{autotag_routes.LEGACY_BATCH_TITLE} already running",
|
||||
web.json_response({}, status=409),
|
||||
)
|
||||
|
||||
async def refused(**_kwargs):
|
||||
return None, refusal
|
||||
|
||||
with patch.object(self.api, "_start_autotag", side_effect=refused):
|
||||
response = await self.client.post(f"{START}?model_type=wd14-vit")
|
||||
self.assertEqual(response.status, 409)
|
||||
self.assertEqual(
|
||||
await response.json(),
|
||||
{"success": False, "error": "Auto-tagging is already running"},
|
||||
)
|
||||
|
||||
async def test_closing_the_stream_cancels_the_job(self):
|
||||
for n in range(3):
|
||||
self.image(f"{n}.png", (n, n, n))
|
||||
real_write = web.StreamResponse.write
|
||||
|
||||
async def flaky_write(response, data):
|
||||
if b'"processed"' in data:
|
||||
raise ConnectionResetError("Cannot write to closing transport")
|
||||
return await real_write(response, data)
|
||||
|
||||
with patch.object(web.StreamResponse, "write", flaky_write):
|
||||
try:
|
||||
response = await self.client.post(f"{START}?model_type=wd14-vit")
|
||||
await response.read()
|
||||
except Exception: # nosec B110 - the client sees the dropped stream
|
||||
pass
|
||||
job = self.api._jobs.list()[0]
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertLess(job.counters["tagged"], 3)
|
||||
|
||||
async def test_a_quiet_job_repeats_its_last_frame(self):
|
||||
self.image("a.png", (1, 1, 1))
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
with patch.object(autotag_routes, "LEGACY_REPEAT_AFTER", 0.05):
|
||||
run = asyncio.ensure_future(self.legacy())
|
||||
await self.wait_for(entered)
|
||||
await asyncio.sleep(0.3)
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
_, frames = await run
|
||||
statuses = [f.get("status") for f in frames]
|
||||
repeated = [s for s in statuses if s and statuses.count(s) > 1]
|
||||
self.assertTrue(repeated, statuses)
|
||||
self.assertEqual(frames[-1]["type"], "complete")
|
||||
|
||||
async def test_a_cancel_from_activity_ends_with_a_cancelled_frame(self):
|
||||
for n in range(2):
|
||||
self.image(f"{n}.png", (n, n, n))
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
run = asyncio.ensure_future(self.legacy())
|
||||
try:
|
||||
await self.wait_for(entered)
|
||||
job = self.api._jobs.active("autotag")
|
||||
await self.client.post(f"/prompt_manager/jobs/{job.id}/cancel")
|
||||
finally:
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
_, frames = await run
|
||||
self.assertEqual(
|
||||
frames[-1],
|
||||
{
|
||||
"type": "cancelled",
|
||||
"message": "Cancelled. Tags already added were kept.",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class TestLegacySingleAndApply(LegacyTestCase):
|
||||
async def test_single_suggests_on_a_capture_and_apply_tags_only_that_image(self):
|
||||
prompt = self.prompt("shared prompt", tags=["keep"])
|
||||
a = self.image("a.png", (1, 2, 3), sees=["bird", "sky"])
|
||||
b = self.image("b.png", (4, 5, 6), sees=["cat"])
|
||||
image_a = self.link(prompt, a)
|
||||
self.link(prompt, b)
|
||||
self.saved_set("b.png", b.read_bytes(), ["hand"])
|
||||
before = self.snapshot_prompts()
|
||||
status, data = await self.single(
|
||||
{"image_id": image_a, "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertEqual(status, 200, data)
|
||||
self.assertEqual(data["tags"], ["bird", "sky"])
|
||||
self.assertEqual(data["sha256"], self.digest("a.png"))
|
||||
self.assertEqual(data["image_tags"], [])
|
||||
self.assertEqual(data["prompt_id"], prompt)
|
||||
self.assertEqual(self.tagger.custom_prompt, "legacy prompt")
|
||||
# Inference read the private capture, not the file.
|
||||
self.assertNotIn(str(self.output_dir), self.tagger.read_paths[-1])
|
||||
status, applied = await self.apply(
|
||||
{"image_id": image_a, "sha256": data["sha256"], "tags": ["bird"]}
|
||||
)
|
||||
self.assertEqual(
|
||||
(status, applied),
|
||||
(200, {"success": True, "added_tags": ["bird"], "total_tags": 1}),
|
||||
)
|
||||
self.assertEqual(self.stored("a.png")["tags"], ["bird"])
|
||||
self.assertEqual(self.stored("b.png")["tags"], ["hand"])
|
||||
self.assertEqual(self.snapshot_prompts(), before)
|
||||
self.assertEqual(self.marked(), {self.locator("a.png")})
|
||||
|
||||
async def test_skip_answers_already_tagged_without_the_gate(self):
|
||||
a = self.image("a.png", (1, 2, 3), sees=["bird"])
|
||||
await self.run_batch()
|
||||
self.tagger.calls.clear()
|
||||
owner = model_gate.new_owner("image-tag-job")
|
||||
self.assertTrue(model_gate.try_acquire(owner))
|
||||
try:
|
||||
status, data = await self.single(
|
||||
{
|
||||
"path": "a.png",
|
||||
"root_index": 0,
|
||||
"existing": "skip",
|
||||
"model_type": "wd14-vit",
|
||||
}
|
||||
)
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
self.assertEqual(status, 200, data)
|
||||
self.assertEqual(data["already_tagged"], True)
|
||||
self.assertEqual(data["image_tags"], ["bird"])
|
||||
self.assertEqual(self.tagger.calls, [])
|
||||
# Hand tags alone do not count; merge always suggests.
|
||||
b = self.image("b.png", (9, 9, 9))
|
||||
self.saved_set("b.png", b.read_bytes(), ["hand"])
|
||||
status, data = await self.single(
|
||||
{"path": str(b), "existing": "skip", "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertNotIn("already_tagged", data)
|
||||
self.assertEqual(data["image_tags"], ["hand"])
|
||||
status, data = await self.single(
|
||||
{"path": str(a), "existing": "merge", "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertNotIn("already_tagged", data)
|
||||
|
||||
async def test_single_uses_the_saved_cap_and_the_refusal_table(self):
|
||||
self.image("a.png", (1, 2, 3))
|
||||
AutotagConfig.SUGGESTION_LIMIT = 3
|
||||
status, data = await self.single(
|
||||
{"path": "a.png", "root_index": 0, "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertEqual((status, len(data["tags"])), (200, 3))
|
||||
(self.output_dir / "broken.png").write_bytes(b"not an image")
|
||||
await self.relist()
|
||||
status, data = await self.single(
|
||||
{"path": "broken.png", "root_index": 0, "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (415, "not-a-still"))
|
||||
self.assertIn(self.locator("broken.png"), self.marked())
|
||||
self.tagger.downloaded["gguf"] = False
|
||||
status, data = await self.single(
|
||||
{"path": "a.png", "root_index": 0, "model_type": "gguf"}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (409, "model-unavailable"))
|
||||
self.api._capture_store().max_file = 16
|
||||
status, data = await self.single(
|
||||
{"path": "a.png", "root_index": 0, "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (413, "too-large"))
|
||||
self.api._capture_store().max_file = 2**20
|
||||
owner = model_gate.new_owner("image-annotations")
|
||||
self.assertTrue(model_gate.try_acquire(owner))
|
||||
try:
|
||||
status, data = await self.single(
|
||||
{"path": "a.png", "root_index": 0, "model_type": "wd14-vit"}
|
||||
)
|
||||
finally:
|
||||
model_gate.release(owner)
|
||||
self.assertEqual((status, data["code"]), (409, "model-busy"))
|
||||
|
||||
async def test_apply_body_and_answer_table(self):
|
||||
a = self.image("a.png", (1, 2, 3))
|
||||
digest = self.digest("a.png")
|
||||
status, data = await self.apply({"prompt_id": 1, "tags": ["a"]})
|
||||
self.assertEqual(
|
||||
(status, data["error"]), (400, "This page is out of date. Reload it.")
|
||||
)
|
||||
status, _ = await self.apply(
|
||||
{"path": "a.png", "root_index": 0, "sha256": "x", "tags": ["a"]}
|
||||
)
|
||||
self.assertEqual(status, 400)
|
||||
status, _ = await self.apply(
|
||||
{"path": "a.png", "root_index": 0, "sha256": digest, "tags": ["bad,name"]}
|
||||
)
|
||||
self.assertEqual(status, 400)
|
||||
status, _ = await self.apply({"path": "a.png", "sha256": digest, "tags": []})
|
||||
self.assertEqual(status, 403) # a bare relative path is not inside a root
|
||||
# An empty reviewed list only marks the image.
|
||||
status, data = await self.apply({"path": str(a), "sha256": digest, "tags": []})
|
||||
self.assertEqual((status, data["added_tags"]), (200, []))
|
||||
self.assertEqual(self.marked(), {self.locator("a.png")})
|
||||
# The file changed since its tags were generated.
|
||||
status, data = await self.apply(
|
||||
{"path": str(a), "sha256": "0" * 64, "tags": ["x"]}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (409, "file-changed"))
|
||||
# Too many tags.
|
||||
full = self.image("full.png", (7, 7, 7))
|
||||
self.saved_set("full.png", full.read_bytes(), [f"t{i}" for i in range(100)])
|
||||
status, data = await self.apply(
|
||||
{"path": str(full), "sha256": self.digest("full.png"), "tags": ["one_more"]}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (409, "tag-limit"))
|
||||
# Gone.
|
||||
status, data = await self.apply(
|
||||
{"path": "gone.png", "root_index": 0, "sha256": digest, "tags": ["x"]}
|
||||
)
|
||||
self.assertIn(status, (403, 404))
|
||||
|
||||
async def test_apply_after_a_restore_writes_nothing(self):
|
||||
a = self.image("a.png", (1, 2, 3))
|
||||
digest = self.digest("a.png")
|
||||
real = self.api.db.get_execution_epoch
|
||||
|
||||
def stale_epoch():
|
||||
epoch = real()
|
||||
conn = sqlite3.connect(self._temp_db.name)
|
||||
try:
|
||||
rotate_history_epoch(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
return epoch
|
||||
|
||||
with patch.object(self.api.db, "get_execution_epoch", side_effect=stale_epoch):
|
||||
status, data = await self.apply(
|
||||
{"path": str(a), "sha256": digest, "tags": ["x"]}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (409, "storage-changed"))
|
||||
self.assertEqual(self.stored("a.png")["tags"], [])
|
||||
|
||||
async def test_image_id_and_path_bodies_reach_the_same_bytes(self):
|
||||
prompt = self.prompt("p")
|
||||
a = self.image("sub/a.png", (1, 2, 3), sees=["x"])
|
||||
image_id = self.link(prompt, a)
|
||||
_, by_id = await self.single({"image_id": image_id, "model_type": "wd14-vit"})
|
||||
_, by_path = await self.single(
|
||||
{"path": "sub/a.png", "root_index": 0, "model_type": "wd14-vit"}
|
||||
)
|
||||
self.assertEqual(by_id["sha256"], by_path["sha256"])
|
||||
status, _ = await self.apply(
|
||||
{"image_id": image_id, "sha256": by_id["sha256"], "tags": ["x"]}
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
status, data = await self.apply(
|
||||
{
|
||||
"path": "sub/a.png",
|
||||
"root_index": 0,
|
||||
"sha256": by_path["sha256"],
|
||||
"tags": ["y"],
|
||||
}
|
||||
)
|
||||
self.assertEqual((status, data["total_tags"]), (200, 2))
|
||||
|
||||
|
||||
class TestDialogMarks(LegacyTestCase):
|
||||
"""DEC4: a Tag image dialog save after Generate suggestions marks the image."""
|
||||
|
||||
def target(self, path):
|
||||
return {
|
||||
"root_index": 0,
|
||||
"root_scope": self.api._gallery_root_scope(self.output_dir, 0),
|
||||
"path": path,
|
||||
}
|
||||
|
||||
async def observe(self, path):
|
||||
response = await self.client.post(OBSERVE, json={"target": self.target(path)})
|
||||
data = await response.json()
|
||||
self.assertEqual(response.status, 200, data)
|
||||
return data
|
||||
|
||||
async def suggest(self, observed, path):
|
||||
response = await self.client.post(
|
||||
SUGGEST,
|
||||
json={
|
||||
"target": self.target(path),
|
||||
"capture_token": observed["capture"]["token"],
|
||||
"expected_epoch": observed["library_epoch"],
|
||||
"model_type": "wd14-vit",
|
||||
"options": {"general_threshold": 0.35, "character_threshold": 0.85},
|
||||
},
|
||||
)
|
||||
self.assertEqual(response.status, 200, await response.text())
|
||||
|
||||
async def save(self, observed, path, tags, revision=None):
|
||||
response = await self.client.put(
|
||||
TAGS,
|
||||
json={
|
||||
"target": self.target(path),
|
||||
"capture_token": observed["capture"]["token"],
|
||||
"sha256": observed["capture"]["sha256"],
|
||||
"expected_epoch": observed["library_epoch"],
|
||||
"expected_tag_revision": (
|
||||
observed["tag_revision"] if revision is None else revision
|
||||
),
|
||||
"tags": tags,
|
||||
},
|
||||
)
|
||||
return response.status, await response.json()
|
||||
|
||||
async def test_save_after_generate_marks_and_a_skip_batch_leaves_it(self):
|
||||
self.image("d.png", (1, 2, 3), sees=["bird", "sky"])
|
||||
observed = await self.observe("d.png")
|
||||
await self.suggest(observed, "d.png")
|
||||
status, _ = await self.save(observed, "d.png", ["sky"]) # bird removed
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(self.marked(), {self.locator("d.png")})
|
||||
job = await self.run_batch(existing="skip")
|
||||
self.assertEqual(self.rows(job, "skipped"), [("d.png", "Already auto-tagged")])
|
||||
self.assertEqual(self.stored("d.png")["tags"], ["sky"])
|
||||
|
||||
async def test_save_without_generate_does_not_mark(self):
|
||||
self.image("e.png", (1, 2, 3))
|
||||
observed = await self.observe("e.png")
|
||||
status, _ = await self.save(observed, "e.png", ["hand"])
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(self.marked(), set())
|
||||
|
||||
async def test_a_conflict_then_a_saved_retry_marks(self):
|
||||
f = self.image("f.png", (1, 2, 3))
|
||||
observed = await self.observe("f.png")
|
||||
await self.suggest(observed, "f.png")
|
||||
self.saved_set("f.png", f.read_bytes(), ["other"]) # another writer
|
||||
status, data = await self.save(observed, "f.png", ["mine"])
|
||||
self.assertEqual((status, data["code"]), (409, "tag-conflict"))
|
||||
self.assertEqual(self.marked(), set())
|
||||
status, _ = await self.save(
|
||||
observed, "f.png", ["mine"], revision=data["tag_revision"]
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(self.marked(), {self.locator("f.png")})
|
||||
|
||||
async def test_another_capture_does_not_inherit_the_mark(self):
|
||||
self.image("g.png", (1, 2, 3))
|
||||
self.image("h.png", (4, 5, 6))
|
||||
first = await self.observe("g.png")
|
||||
await self.suggest(first, "g.png")
|
||||
second = await self.observe("h.png")
|
||||
status, _ = await self.save(second, "h.png", ["x"])
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(self.marked(), set())
|
||||
self.assertIn(first["capture"]["token"], self.api._suggested_tokens)
|
||||
|
||||
async def test_gone_tokens_are_dropped_on_the_next_insert_without_touching_clocks(
|
||||
self,
|
||||
):
|
||||
self.image("i.png", (1, 2, 3))
|
||||
self.image("j.png", (4, 5, 6))
|
||||
first = await self.observe("i.png")
|
||||
await self.suggest(first, "i.png")
|
||||
store = self.api._capture_store()
|
||||
store.release(first["capture"]["token"])
|
||||
second = await self.observe("j.png")
|
||||
seen = []
|
||||
real_get = store.get
|
||||
with patch.object(
|
||||
store, "get", side_effect=lambda t: seen.append(t) or real_get(t)
|
||||
):
|
||||
await self.suggest(second, "j.png")
|
||||
self.assertEqual(seen, [])
|
||||
self.assertEqual(self.api._suggested_tokens, {second["capture"]["token"]})
|
||||
@@ -1,318 +0,0 @@
|
||||
"""Auto-tagged marks schema group (#379): install, validation and restore."""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.models import PromptModel
|
||||
|
||||
try: # guarded so a missing module fails assertions, not test loading
|
||||
from database.autotag_schema import (
|
||||
AUTOTAG_TABLE,
|
||||
autotag_schema_error,
|
||||
autotag_without_annotations_error,
|
||||
migrate_autotag_state,
|
||||
)
|
||||
except ImportError:
|
||||
AUTOTAG_TABLE = None
|
||||
autotag_schema_error = None
|
||||
autotag_without_annotations_error = None
|
||||
migrate_autotag_state = None
|
||||
|
||||
TABLE = "media_image_autotags"
|
||||
DDL = (
|
||||
f"CREATE TABLE {TABLE} (byte_id TEXT PRIMARY KEY "
|
||||
"REFERENCES media_annotation_bytes(id) ON DELETE CASCADE) WITHOUT ROWID"
|
||||
)
|
||||
ANNOTATION_TABLES = (
|
||||
"media_annotation_metadata",
|
||||
"media_annotation_files",
|
||||
"media_annotation_bytes",
|
||||
"media_image_tags",
|
||||
)
|
||||
REVIEW_TABLES = (
|
||||
"media_review_metadata",
|
||||
"media_review_items",
|
||||
"media_review_decisions",
|
||||
)
|
||||
FILE_ID = "0f8e4b2a-1c3d-4e5f-8a9b-0c1d2e3f4a5b"
|
||||
BYTE_ID = "6a1b2c3d-4e5f-4a6b-8c9d-0e1f2a3b4c5d"
|
||||
OTHER_ID = "7b2c3d4e-5f6a-4b7c-9d0e-1f2a3b4c5d6e"
|
||||
DIGEST = hashlib.sha256(b"img").hexdigest()
|
||||
NOW = "2026-10-27T00:00:00.000000+00:00"
|
||||
NEEDS_ANNOTATIONS = "Image auto-tag marks need the image annotation tables"
|
||||
INVALID_VALUES = "Image auto-tag marks contain invalid values"
|
||||
INVALID_SCHEMA = "Image auto-tag marks have an invalid schema"
|
||||
|
||||
|
||||
def _compact(sql):
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def catalog(conn):
|
||||
return {
|
||||
(kind, name, _compact(sql))
|
||||
for kind, name, sql in conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name=? OR tbl_name=?",
|
||||
(TABLE, TABLE),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def marks(conn):
|
||||
return [tuple(row) for row in conn.execute(f"SELECT byte_id FROM {TABLE}")]
|
||||
|
||||
|
||||
def insert_byte(conn, file_id=FILE_ID, byte_id=BYTE_ID, locator="/out/img.png"):
|
||||
conn.execute(
|
||||
"INSERT INTO media_annotation_files VALUES(?,?,?)", (file_id, locator, NOW)
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO media_annotation_bytes VALUES(?,?,?,?,?,?,?,0)",
|
||||
(byte_id, file_id, DIGEST, 3, 1_700_000_000_000_000_000, NOW, NOW),
|
||||
)
|
||||
|
||||
|
||||
class AutotagSchemaTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.addCleanup(assert_closed, self, self.temp.name)
|
||||
self.addCleanup(PromptModel.reset_schema_cache)
|
||||
self.models = []
|
||||
self.addCleanup(self.close_models)
|
||||
self.path = str(Path(self.temp.name) / "live.db")
|
||||
|
||||
def close_models(self):
|
||||
for model in self.models:
|
||||
model.close_all()
|
||||
|
||||
def model(self, path=None):
|
||||
model = PromptModel(path or self.path)
|
||||
self.models.append(model)
|
||||
return model
|
||||
|
||||
def require_module(self):
|
||||
self.assertIsNotNone(
|
||||
autotag_schema_error, "database.autotag_schema is not implemented"
|
||||
)
|
||||
|
||||
def raw(self, path, *statements, foreign_keys=False):
|
||||
with closing(sqlite3.connect(path)) as raw, raw:
|
||||
raw.execute(f"PRAGMA foreign_keys={'ON' if foreign_keys else 'OFF'}")
|
||||
for statement in statements:
|
||||
if isinstance(statement, tuple):
|
||||
raw.execute(*statement)
|
||||
else:
|
||||
raw.execute(statement)
|
||||
|
||||
def closed_current(self, name="current.db", *, with_mark=True):
|
||||
"""A closed, current database with one byte record (and its mark)."""
|
||||
path = str(Path(self.temp.name) / name)
|
||||
model = self.model(path)
|
||||
with model.get_connection() as conn:
|
||||
insert_byte(conn)
|
||||
if with_mark:
|
||||
conn.execute(f"INSERT INTO {TABLE} VALUES(?)", (BYTE_ID,))
|
||||
model.close_all()
|
||||
return path
|
||||
|
||||
def test_fresh_install_has_exactly_the_pinned_table(self):
|
||||
self.require_module()
|
||||
self.assertEqual(AUTOTAG_TABLE, TABLE)
|
||||
model = self.model()
|
||||
with model.get_connection() as conn:
|
||||
# WITHOUT ROWID with a primary key: no sqlite_autoindex row.
|
||||
self.assertEqual(catalog(conn), {("table", TABLE, _compact(DDL))})
|
||||
self.assertEqual(
|
||||
[tuple(row) for row in conn.execute(f"PRAGMA table_info({TABLE})")],
|
||||
[(0, "byte_id", "TEXT", 1, None, 1)],
|
||||
)
|
||||
self.assertIsNone(autotag_schema_error(conn, allow_missing=False))
|
||||
self.assertEqual(marks(conn), [])
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
|
||||
def test_reopen_and_rerun_are_idempotent(self):
|
||||
self.require_module()
|
||||
path = self.closed_current()
|
||||
for _ in range(2):
|
||||
PromptModel.reset_schema_cache()
|
||||
model = self.model(path)
|
||||
with model.get_connection() as conn:
|
||||
migrate_autotag_state(conn)
|
||||
self.assertEqual(marks(conn), [(BYTE_ID,)])
|
||||
self.assertEqual(catalog(conn), {("table", TABLE, _compact(DDL))})
|
||||
model.close_all()
|
||||
|
||||
def test_old_database_without_the_table_gains_it_and_keeps_its_rows(self):
|
||||
self.require_module()
|
||||
path = self.closed_current(with_mark=False)
|
||||
self.raw(path, f"DROP TABLE {TABLE}")
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
before = raw.execute("SELECT * FROM media_annotation_bytes").fetchall()
|
||||
self.assertIsNone(autotag_schema_error(raw))
|
||||
self.assertEqual(
|
||||
autotag_schema_error(raw, allow_missing=False),
|
||||
"Missing image auto-tag marks schema",
|
||||
)
|
||||
self.assertTrue(PromptModel.verify_database_file(path)[0])
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(path, require_library_state=True),
|
||||
(False, "Missing image auto-tag marks schema"),
|
||||
)
|
||||
PromptModel.reset_schema_cache()
|
||||
model = self.model(path)
|
||||
with model.get_connection() as conn:
|
||||
self.assertEqual(catalog(conn), {("table", TABLE, _compact(DDL))})
|
||||
self.assertEqual(
|
||||
[
|
||||
tuple(row)
|
||||
for row in conn.execute("SELECT * FROM media_annotation_bytes")
|
||||
],
|
||||
before,
|
||||
)
|
||||
model.close_all()
|
||||
|
||||
def test_table_without_annotation_group_is_refused(self):
|
||||
self.require_module()
|
||||
path = self.closed_current(with_mark=False)
|
||||
self.raw(
|
||||
path,
|
||||
*(
|
||||
f"DROP TABLE {t}"
|
||||
for t in REVIEW_TABLES + tuple(reversed(ANNOTATION_TABLES))
|
||||
),
|
||||
)
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertEqual(autotag_without_annotations_error(raw), NEEDS_ANNOTATIONS)
|
||||
for allow_missing in (True, False):
|
||||
self.assertEqual(
|
||||
autotag_schema_error(raw, allow_missing=allow_missing),
|
||||
NEEDS_ANNOTATIONS,
|
||||
)
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(path), (False, NEEDS_ANNOTATIONS)
|
||||
)
|
||||
PromptModel.reset_schema_cache()
|
||||
with self.assertRaises(sqlite3.DatabaseError) as raised:
|
||||
self.model(path)
|
||||
self.assertIn(NEEDS_ANNOTATIONS, str(raised.exception))
|
||||
|
||||
def test_invalid_values_are_refused_without_repair(self):
|
||||
self.require_module()
|
||||
cases = {
|
||||
"not-a-uuid": (f"INSERT INTO {TABLE} VALUES('not-a-uuid')",),
|
||||
"upper-uuid": ((f"INSERT INTO {TABLE} VALUES(?)", (OTHER_ID.upper(),)),),
|
||||
"dangling": ((f"INSERT INTO {TABLE} VALUES(?)", (OTHER_ID,)),),
|
||||
}
|
||||
for name, statements in cases.items():
|
||||
with self.subTest(name):
|
||||
path = self.closed_current(f"{name}.db", with_mark=False)
|
||||
self.raw(path, *statements)
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertEqual(autotag_schema_error(raw), INVALID_VALUES)
|
||||
before = marks(raw)
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(path), (False, INVALID_VALUES)
|
||||
)
|
||||
PromptModel.reset_schema_cache()
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
self.model(path)
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertEqual(marks(raw), before)
|
||||
|
||||
def test_altered_definitions_are_refused(self):
|
||||
self.require_module()
|
||||
cases = {
|
||||
"rowid": f"CREATE TABLE {TABLE} (byte_id TEXT PRIMARY KEY "
|
||||
"REFERENCES media_annotation_bytes(id) ON DELETE CASCADE)",
|
||||
"restrict": f"CREATE TABLE {TABLE} (byte_id TEXT PRIMARY KEY "
|
||||
"REFERENCES media_annotation_bytes(id) ON DELETE RESTRICT) WITHOUT ROWID",
|
||||
"extra column": f"CREATE TABLE {TABLE} (byte_id TEXT PRIMARY KEY "
|
||||
"REFERENCES media_annotation_bytes(id) ON DELETE CASCADE, engine TEXT) "
|
||||
"WITHOUT ROWID",
|
||||
}
|
||||
for name, ddl in cases.items():
|
||||
with self.subTest(name):
|
||||
path = self.closed_current(f"{name}.db", with_mark=False)
|
||||
self.raw(path, f"DROP TABLE {TABLE}", ddl)
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertEqual(autotag_schema_error(raw), INVALID_SCHEMA)
|
||||
self.assertFalse(PromptModel.verify_database_file(path)[0])
|
||||
path = self.closed_current("index.db", with_mark=False)
|
||||
self.raw(path, f"CREATE INDEX idx_extra ON {TABLE}(byte_id)")
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
self.assertEqual(autotag_schema_error(raw), INVALID_SCHEMA)
|
||||
|
||||
def test_deleting_the_byte_record_cascades_to_its_mark(self):
|
||||
self.require_module()
|
||||
model = self.model()
|
||||
with model.get_connection() as conn:
|
||||
insert_byte(conn)
|
||||
conn.execute(f"INSERT INTO {TABLE} VALUES(?)", (BYTE_ID,))
|
||||
conn.execute(f"INSERT OR IGNORE INTO {TABLE} VALUES(?)", (BYTE_ID,))
|
||||
self.assertEqual(marks(conn), [(BYTE_ID,)])
|
||||
conn.execute("DELETE FROM media_annotation_files")
|
||||
self.assertEqual(marks(conn), [])
|
||||
|
||||
def test_restore_of_a_pre_change_backup_migrates(self):
|
||||
self.require_module()
|
||||
src = self.closed_current("backup.db", with_mark=False)
|
||||
self.raw(src, f"DROP TABLE {TABLE}")
|
||||
live = self.model()
|
||||
live.restore_from_file(src)
|
||||
with closing(sqlite3.connect(self.path)) as raw:
|
||||
self.assertEqual(catalog(raw), {("table", TABLE, _compact(DDL))})
|
||||
self.assertEqual(
|
||||
raw.execute("SELECT id FROM media_annotation_bytes").fetchall(),
|
||||
[(BYTE_ID,)],
|
||||
)
|
||||
self.assertEqual(
|
||||
PromptModel.verify_database_file(self.path, require_library_state=True),
|
||||
(True, "ok"),
|
||||
)
|
||||
with closing(sqlite3.connect(src)) as raw: # the source is untouched
|
||||
self.assertEqual(catalog(raw), set())
|
||||
|
||||
def test_install_failure_rolls_back(self):
|
||||
self.require_module()
|
||||
src = self.closed_current("copy.db", with_mark=False)
|
||||
self.raw(src, f"DROP TABLE {TABLE}")
|
||||
path = str(Path(self.temp.name) / "rollback.db")
|
||||
shutil.copyfile(src, path)
|
||||
with closing(sqlite3.connect(path)) as raw:
|
||||
raw.execute("BEGIN")
|
||||
raw.execute("CREATE TABLE marker(x)")
|
||||
# An invalid group already present: nothing created, outer work kept.
|
||||
raw.execute(f"CREATE TABLE {TABLE}(byte_id TEXT)")
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
migrate_autotag_state(raw)
|
||||
self.assertEqual(
|
||||
raw.execute(
|
||||
f"SELECT sql FROM sqlite_master WHERE name='{TABLE}'"
|
||||
).fetchone()[0],
|
||||
f"CREATE TABLE {TABLE}(byte_id TEXT)",
|
||||
)
|
||||
self.assertIsNotNone(
|
||||
raw.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE name='marker'"
|
||||
).fetchone()
|
||||
)
|
||||
raw.rollback()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1398,6 +1398,50 @@ class TestModelHousekeeping(DatabaseTestCase):
|
||||
self.assertIsNot(self.db.model.get_connection(), failing)
|
||||
|
||||
|
||||
class TestAutotagBatchCandidates(DatabaseTestCase):
|
||||
"""#328: the batch's candidate read never loads workflow documents."""
|
||||
|
||||
def test_autotag_candidates_skip_workflow_data(self):
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
first = self.db.save_prompt(
|
||||
text="a", tags=["one", "two"], prompt_hash=generate_prompt_hash("a")
|
||||
)
|
||||
second = self.db.save_prompt(text="b", prompt_hash=generate_prompt_hash("b"))
|
||||
old = self.db.link_image_to_prompt(
|
||||
first, "/out/a.png", {"workflow": {"big": "x" * 1000}}
|
||||
)
|
||||
new = self.db.link_image_to_prompt(second, "/out/b.png")
|
||||
statements = []
|
||||
conn = self.db.model.get_connection() # this thread's own connection
|
||||
conn.set_trace_callback(statements.append)
|
||||
try:
|
||||
rows = self.db.autotag_batch_candidates()
|
||||
finally:
|
||||
conn.set_trace_callback(None)
|
||||
self.assertEqual(
|
||||
rows[0],
|
||||
{
|
||||
"id": new,
|
||||
"image_path": "/out/b.png",
|
||||
"prompt_id": second,
|
||||
"prompt_tags": [],
|
||||
},
|
||||
)
|
||||
self.assertEqual(
|
||||
{**rows[1], "prompt_tags": sorted(rows[1]["prompt_tags"])},
|
||||
{
|
||||
"id": old,
|
||||
"image_path": "/out/a.png",
|
||||
"prompt_id": first,
|
||||
"prompt_tags": ["one", "two"],
|
||||
},
|
||||
)
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertTrue(statements)
|
||||
self.assertFalse(any("workflow_data" in sql for sql in statements))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
@@ -468,7 +468,6 @@ class TestRestoreFromFile(BackupTestCase):
|
||||
|
||||
def test_restore_old_backup_without_annotations(self):
|
||||
tables = (
|
||||
"media_image_autotags",
|
||||
"media_review_decisions",
|
||||
"media_review_items",
|
||||
"media_review_metadata",
|
||||
|
||||
@@ -356,8 +356,6 @@ class HistoryHooksTests(unittest.TestCase):
|
||||
self.assertGreaterEqual(time.monotonic() - began, 1.8)
|
||||
self.assertLess(time.monotonic() - began, CEILING_SECONDS)
|
||||
release.set()
|
||||
# Drain first: the 2 s bound is not under test here (#370).
|
||||
self.drain(registration)
|
||||
self.assertTrue(registration.dispose(2))
|
||||
|
||||
def test_restore_disables_capture_and_clears_handles_before_fresh_epoch(self):
|
||||
|
||||
@@ -399,10 +399,6 @@ class OutputHooksTests(fixtures.unittest.TestCase):
|
||||
self.queue.task_done(1, history("rejected.png"), fixtures.status())
|
||||
registration.writer.invalidate()
|
||||
release.set()
|
||||
# Let the writer work through the 1,024 queued commands first
|
||||
# (#370's pattern): on a busy Windows runner that took longer
|
||||
# than dispose's 2 s bound (merge queue, #382).
|
||||
self.drain(registration)
|
||||
self.assertTrue(registration.dispose(2))
|
||||
self.assertEqual(self.outputs(0)["state"], "not_captured")
|
||||
self.assertEqual(len(self.rows()), 1)
|
||||
|
||||
@@ -28,11 +28,6 @@ except ImportError:
|
||||
# Upper bound a blocked fake write waits for its release. Tests prove ordering
|
||||
# with events, so this only caps a failing run and never decides a pass.
|
||||
BLOCK_SECONDS = 10
|
||||
# A restore is a full verify/migrate/backup/replace cycle and is bound by the
|
||||
# disk: a busy Windows runner took over 10 s (merge queue, #382). Waits that
|
||||
# span a restore use this ceiling, which stays under the runner's 120 s hang
|
||||
# watchdog, so a real hang still fails with the thread dump.
|
||||
RESTORE_SECONDS = 90
|
||||
|
||||
|
||||
class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
@@ -322,8 +317,8 @@ class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
|
||||
Returns the patch and an event set once that rotation has returned on
|
||||
the worker. A restore is a full verify/migrate/backup/replace cycle, so
|
||||
the handoff waits on this event with the ``RESTORE_SECONDS`` ceiling
|
||||
instead of the short worker-behavior deadline (TESTING.md,
|
||||
the handoff waits on this event with the generous ``BLOCK_SECONDS``
|
||||
ceiling instead of the short worker-behavior deadline (TESTING.md,
|
||||
Windows portability preflight).
|
||||
"""
|
||||
backup = os.path.join(self.directory.name, f"restore-{start.run_id}.db")
|
||||
@@ -366,7 +361,7 @@ class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
)
|
||||
with rotation:
|
||||
self.finalization(writer, start)
|
||||
self.assertTrue(rotated.wait(RESTORE_SECONDS))
|
||||
self.assertTrue(rotated.wait(BLOCK_SECONDS))
|
||||
self.wait_for(lambda: writer.status().pending == 0)
|
||||
self.assertEqual(
|
||||
self.db.get_execution_outputs(start.run_id)["state"],
|
||||
@@ -408,7 +403,7 @@ class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
self.assertEqual(writer.status().pending, 1)
|
||||
self.assertEqual(writer.context_budget.charged_bytes, charge)
|
||||
release.set()
|
||||
self.assertTrue(rotated.wait(RESTORE_SECONDS))
|
||||
self.assertTrue(rotated.wait(BLOCK_SECONDS))
|
||||
self.wait_for(lambda: writer.status().pending == 0)
|
||||
self.assertEqual(writer.context_budget.charged_bytes, 0)
|
||||
self.assertFalse(writer.status().accepting)
|
||||
@@ -572,8 +567,6 @@ class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
self.assertIs(replacement.context_budget, writer.context_budget)
|
||||
self.assertIsNone(replacement.context_budget.reserve(8388608))
|
||||
release.set()
|
||||
# Drain first: the 2 s bound is not under test here (#370).
|
||||
drain_writer(writer)
|
||||
self.assertTrue(writer.close(2))
|
||||
finally:
|
||||
release.set()
|
||||
@@ -615,8 +608,6 @@ class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
writer.invalidate()
|
||||
self.assertEqual(writer.context_budget.charged_bytes, 0)
|
||||
release.set()
|
||||
# Drain first: the 2 s bound is not under test here (#370).
|
||||
drain_writer(writer)
|
||||
self.assertTrue(writer.close(2))
|
||||
|
||||
def test_malformed_context_omission_preserves_independent_valid_output(self):
|
||||
@@ -707,8 +698,6 @@ class OutputWriterTests(fixtures.unittest.TestCase):
|
||||
release.set()
|
||||
invalidator.join(BLOCK_SECONDS)
|
||||
self.assertFalse(invalidator.is_alive())
|
||||
# Drain first: the 2 s bound is not under test here (#370).
|
||||
drain_writer(writer)
|
||||
self.assertTrue(writer.close(2))
|
||||
self.assertEqual(
|
||||
self.db.get_execution_outputs(start.run_id)["state"], "complete"
|
||||
|
||||
@@ -11,7 +11,6 @@ engine is B2's fake service.
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import unittest
|
||||
@@ -691,32 +690,6 @@ class TestFixRound(ImageTagJobTestCase):
|
||||
],
|
||||
)
|
||||
|
||||
async def test_an_ended_job_shows_no_time_left(self):
|
||||
"""#379 live check: a finished or cancelled job never says "min left"."""
|
||||
for index in range(3):
|
||||
self.png(f"f{index}.png", color=(index, 7, 7))
|
||||
clock = iter([0.0, 0.0, 120.0, 240.0])
|
||||
with patch(
|
||||
"py.api.image_tag_jobs._monotonic", side_effect=lambda: next(clock, 240.0)
|
||||
):
|
||||
job = await self.run_job(self.auto_body([f"f{i}.png" for i in range(3)]))
|
||||
self.assertEqual(job.state, "finished")
|
||||
self.assertEqual(job.status_line, "Auto-tag finished · 3 files processed")
|
||||
|
||||
self.png("g0.png", color=(1, 1, 1))
|
||||
self.png("g1.png", color=(2, 2, 2))
|
||||
entered, proceed = threading.Event(), threading.Event()
|
||||
self.tagger.block = (entered, proceed)
|
||||
job = await self.start_ok(self.auto_body(["g0.png", "g1.png"]))
|
||||
await self.wait_for(entered)
|
||||
await self.client.post(f"/prompt_manager/jobs/{job.id}/cancel")
|
||||
self.tagger.block = None
|
||||
proceed.set()
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
self.assertEqual(job.status_line, "Auto-tag cancelled · 1 file processed")
|
||||
self.assertNotIn("left", job.status_line)
|
||||
|
||||
def maximal_target(self, number):
|
||||
"""A distinct target whose path is 1024 non-ASCII code points long."""
|
||||
head = f"{number:03d}/" + ("界" * 200 + "/") * 5
|
||||
@@ -829,108 +802,3 @@ class TestFixRound(ImageTagJobTestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestFilterStart(ImageTagJobTestCase):
|
||||
"""#392: "Select all N matching" starts the job with the listing filter."""
|
||||
|
||||
async def test_filter_start_tags_every_matching_file_in_listing_order(self):
|
||||
for number in range(6):
|
||||
path = self.png(f"bulk/{number:02}.png", color=(number, 1, 1))
|
||||
os.utime(path, (100 + number, 100 + number))
|
||||
self.png("other/x.png")
|
||||
|
||||
job = await self.run_job(
|
||||
{
|
||||
"mode": "add",
|
||||
"filter": {"subfolder": "bulk", "sort": "oldest"},
|
||||
"tags": ["picked"],
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(job.scope, {"mode": "add", "images": 6})
|
||||
self.assertEqual(job.counters["tagged"], 6)
|
||||
self.assertEqual(
|
||||
[path for path, _detail in self.rows(job, "tagged")],
|
||||
[f"bulk/{number:02}.png" for number in range(6)],
|
||||
)
|
||||
self.assertEqual(self.stored("bulk/03.png")["tags"], ["picked"])
|
||||
self.assertEqual(self.stored("other/x.png")["tags"], [])
|
||||
|
||||
async def test_auto_filter_start_runs_the_model_on_the_matching_files(self):
|
||||
self.png("bulk/a.png")
|
||||
self.png("other/b.png", color=(1, 2, 3))
|
||||
|
||||
job = await self.run_job(
|
||||
{
|
||||
"mode": "auto",
|
||||
"filter": {"subfolder": "bulk"},
|
||||
"model_type": "wd14-vit",
|
||||
"options": dict(WD14),
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(job.scope["images"], 1)
|
||||
self.assertEqual(job.counters["tagged"], 1)
|
||||
self.assertEqual(self.stored("other/b.png")["tags"], [])
|
||||
|
||||
async def test_filter_with_no_match_is_refused_without_a_job(self):
|
||||
self.png("a.png")
|
||||
|
||||
status, data = await self.start(
|
||||
{"mode": "add", "filter": {"subfolder": "missing"}, "tags": ["x"]}
|
||||
)
|
||||
|
||||
self.assertEqual((status, data["code"]), (400, "no-images"))
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
|
||||
async def test_filter_over_the_cap_is_refused_without_a_job(self):
|
||||
from py.api import image_tag_jobs
|
||||
|
||||
for number in range(3):
|
||||
self.png(f"{number}.png", color=(number, 0, 0))
|
||||
with patch.object(image_tag_jobs, "MAX_FILTER_TARGETS", 2):
|
||||
status, data = await self.start(
|
||||
{"mode": "add", "filter": {}, "tags": ["x"]}
|
||||
)
|
||||
|
||||
self.assertEqual((status, data["code"]), (400, "too-many-images"))
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
self.assertEqual(image_tag_jobs.MAX_FILTER_TARGETS, 10_000)
|
||||
|
||||
async def test_filter_bodies_are_validated(self):
|
||||
self.png("a.png")
|
||||
target = self.target("a.png")
|
||||
for body in (
|
||||
{"mode": "add", "filter": {}, "targets": [target], "tags": ["x"]},
|
||||
{"mode": "add", "filter": "all", "tags": ["x"]},
|
||||
{"mode": "add", "filter": {"sort": "sideways"}, "tags": ["x"]},
|
||||
{"mode": "add", "filter": {"cursor": "x"}, "tags": ["x"]},
|
||||
{"mode": "add", "filter": {"root": "7"}, "tags": ["x"]},
|
||||
{"mode": "add", "filter": {}, "tags": []},
|
||||
):
|
||||
with self.subTest(body=body):
|
||||
status, data = await self.start(body)
|
||||
self.assertEqual(status, 400, data)
|
||||
self.assertEqual(self.api._jobs.list(), [])
|
||||
|
||||
async def test_a_filter_whose_source_was_renumbered_is_refused(self):
|
||||
self.png("a.png")
|
||||
scope = self.target("a.png")["root_scope"]
|
||||
|
||||
job = await self.run_job(
|
||||
{"mode": "add", "filter": {"root": "0", "root_scope": scope}, "tags": ["x"]}
|
||||
)
|
||||
self.assertEqual(job.counters["tagged"], 1)
|
||||
status, data = await self.start(
|
||||
{
|
||||
"mode": "add",
|
||||
"filter": {"root": "0", "root_scope": "f" * 64},
|
||||
"tags": ["x"],
|
||||
}
|
||||
)
|
||||
self.assertEqual((status, data["code"]), (409, "source-changed"))
|
||||
status, data = await self.start(
|
||||
{"mode": "add", "filter": {"root_scope": scope}, "tags": ["x"]}
|
||||
)
|
||||
self.assertEqual(status, 400, data)
|
||||
|
||||
@@ -59,7 +59,6 @@ def _minimal():
|
||||
{"class": "VAEEncode", "slot": 0, "strategy": "continue", "input": "pixels"}
|
||||
],
|
||||
"stage_inputs": {"shaped": ["positive"], "with_model": ["samples"]},
|
||||
"pass_through": {"skip_inputs": ["clip"]},
|
||||
"facts": {
|
||||
"configuration_outputs": {},
|
||||
"model_outputs": {},
|
||||
@@ -691,18 +690,6 @@ class ValidationTest(_TableCase):
|
||||
(lambda d: d["stage_inputs"].update(shaped="pipe"), "input name"),
|
||||
(lambda d: d["stage_inputs"].update(with_model=["positive"]), "both"),
|
||||
(lambda d: d["stage_inputs"].update(shaped=["model"]), "model"),
|
||||
(lambda d: d.pop("pass_through"), "category"),
|
||||
(lambda d: d["pass_through"].update(extra=["x"]), "pass_through"),
|
||||
(lambda d: d["pass_through"].update(skip_inputs="clip"), "input name"),
|
||||
(lambda d: d["pass_through"].update(skip_inputs=[]), "input name"),
|
||||
(
|
||||
lambda d: d["pass_through"].update(skip_inputs=["clip", "clip"]),
|
||||
"repeated",
|
||||
),
|
||||
(
|
||||
lambda d: d["pass_through"].update(skip_inputs=["conditioning"]),
|
||||
"carries text",
|
||||
),
|
||||
(
|
||||
lambda d: d["facts"]["configuration_outputs"].update(
|
||||
X={"kind": "sigmas", "fields": ["steps"], "type_fields": []}
|
||||
|
||||
@@ -535,9 +535,7 @@ class TestHeldJobStates(_LegacyHoldCase):
|
||||
self.assertIn("event: error", text)
|
||||
self.assertEqual(await self._listed(), [("thumbnails", "failed")])
|
||||
|
||||
async def test_legacy_autotag_refused_at_start_leaves_no_job(self):
|
||||
# #379: the legacy batch is the autotag job; a model that is not
|
||||
# downloaded refuses the start (one error frame) and registers nothing.
|
||||
async def test_legacy_autotag_that_ends_in_error_is_failed(self):
|
||||
service = MagicMock()
|
||||
service.get_models_status.return_value = {} # model not downloaded
|
||||
with patch("py.autotag.get_autotag_service", return_value=service):
|
||||
@@ -547,7 +545,7 @@ class TestHeldJobStates(_LegacyHoldCase):
|
||||
text = await resp.text()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertIn('"type": "error"', text)
|
||||
self.assertEqual(await self._listed(), [])
|
||||
self.assertEqual(await self._listed(), [("autotag", "failed")])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,187 +0,0 @@
|
||||
"""Legacy parity gate for the inspection resolver (#381, the PR #170 floor).
|
||||
|
||||
The legacy parser (``utils.generation_params``, the port of the browser's
|
||||
``extractGenerationParams``) is the floor: a graph whose positive prompt it
|
||||
found, and that the inspection resolver resolved to the same text, must keep
|
||||
resolving to that text. ``legacy-agreement.json`` records those graphs per
|
||||
corpus: the generation-params vectors, the inspection parity fixtures, the
|
||||
local resolver golden and selection cases, and the unknown-node shapes of
|
||||
#381. A graph may join the list, never leave it; regenerate with::
|
||||
|
||||
python3 -m tests.test_legacy_parity_gate --write
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from utils import prompt_pair_resolver as resolver
|
||||
from utils.generation_params import (
|
||||
GenerationParamsError,
|
||||
extract_generation_params,
|
||||
parse_js_json,
|
||||
)
|
||||
|
||||
try:
|
||||
from tests import inspection_parity as parity
|
||||
from tests import local_resolver_cases as cases
|
||||
from tests import local_resolver_selection_cases as selection
|
||||
from tests import unknown_node_graphs as unknown
|
||||
except ImportError: # discovered with ``-s tests``
|
||||
import inspection_parity as parity
|
||||
import local_resolver_cases as cases
|
||||
import local_resolver_selection_cases as selection
|
||||
import unknown_node_graphs as unknown
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
AGREEMENT_PATH = HERE / "fixtures" / "local-inspect" / "legacy-agreement.json"
|
||||
VECTORS_PATH = HERE / "fixtures" / "generation_params" / "vectors.json"
|
||||
NOT_FOUND = "No prompt found"
|
||||
|
||||
|
||||
def _parsed(value):
|
||||
return parse_js_json(value) if isinstance(value, str) else value
|
||||
|
||||
|
||||
def corpora() -> dict:
|
||||
"""Corpus name to ``{case name: (prompt graph, workflow or None[, file])}``."""
|
||||
vectors = json.loads(VECTORS_PATH.read_text(encoding="utf-8"))
|
||||
named = {**cases.golden_cases(), **selection.selection_cases()}
|
||||
return {
|
||||
"generation_params": {
|
||||
item["name"]: (_parsed(item["prompt"]), _parsed(item.get("workflow")))
|
||||
for item in vectors
|
||||
},
|
||||
"inspection_parity": {
|
||||
name: (parity.build(name)["prompt"], parity.build(name)["workflow"])
|
||||
for name in parity.COMMITTED
|
||||
},
|
||||
"local_resolver": {
|
||||
name: (
|
||||
_parsed(records["prompt"]),
|
||||
_parsed(records.get("workflow")),
|
||||
records.get(cases.FILE_NAME),
|
||||
)
|
||||
for name, records in named.items()
|
||||
if "prompt" in records
|
||||
},
|
||||
"unknown_nodes": {
|
||||
name: (graph, None) for name, graph in unknown.shapes().items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def legacy_positive(graph, workflow):
|
||||
"""The positive prompt the legacy parser found, or ``None``."""
|
||||
try:
|
||||
found = extract_generation_params({"prompt": graph, "workflow": workflow})
|
||||
except GenerationParamsError:
|
||||
return None
|
||||
text = found["positivePrompt"]
|
||||
return None if text == NOT_FOUND else text
|
||||
|
||||
|
||||
def inspected(graph, workflow=None, file_name=None) -> dict:
|
||||
"""The inspection resolver's answer, anchored by ``file_name`` when given."""
|
||||
metadata = {"prompt": graph}
|
||||
if workflow is not None:
|
||||
metadata["workflow"] = workflow
|
||||
return resolver.resolve_prompt_pair(
|
||||
metadata, profile=resolver.INSPECTION_PROFILE, file_name=file_name
|
||||
)
|
||||
|
||||
|
||||
# Graphs where the legacy parser's positive is wrong and the inspector's
|
||||
# differs on purpose: (corpus, case) -> (the inspector's positive, why).
|
||||
LEGACY_WRONG = {
|
||||
("local_resolver", "anchor-branch-3"): (
|
||||
"a red vintage car in the rain",
|
||||
"legacy shows the first sampler's prompt for every branch; the file "
|
||||
"name s45_branch_3_00002_ names branch 3's saver (gpu_unlock_bench)",
|
||||
),
|
||||
("local_resolver", "anchor-branch-linked"): (
|
||||
"an old fishing boat at dawn",
|
||||
"legacy shows branch 1; the traced prefix names branch 2's saver",
|
||||
),
|
||||
("local_resolver", "anchor-branch-unknown-saver"): (
|
||||
"a cozy library with tall shelves",
|
||||
"legacy shows branch 1; the file name names branch 4's saver",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def agreeing(graphs: dict) -> list:
|
||||
"""Names whose inspection positive equals the legacy parser's, sorted."""
|
||||
names = []
|
||||
for name, case in graphs.items():
|
||||
graph, workflow = case[0], case[1]
|
||||
if not isinstance(graph, dict):
|
||||
continue
|
||||
expected = legacy_positive(graph, workflow)
|
||||
if expected is None:
|
||||
continue
|
||||
result = inspected(*case)
|
||||
positive = result["positive"]
|
||||
if (
|
||||
result["state"] == "resolved"
|
||||
and isinstance(positive, dict)
|
||||
and positive["text"] == expected
|
||||
):
|
||||
names.append(name)
|
||||
return sorted(names)
|
||||
|
||||
|
||||
class LegacyParityGateTest(unittest.TestCase):
|
||||
def test_every_legacy_difference_is_stated(self):
|
||||
"""A graph may read differently from legacy only with a stated reason."""
|
||||
all_graphs = corpora()
|
||||
for (corpus, name), (expected, why) in LEGACY_WRONG.items():
|
||||
with self.subTest(case=name, why=why):
|
||||
case = all_graphs[corpus][name]
|
||||
legacy = legacy_positive(case[0], case[1])
|
||||
self.assertIsNotNone(legacy)
|
||||
result = inspected(*case)
|
||||
positive = result["positive"]
|
||||
text = positive["text"] if isinstance(positive, dict) else None
|
||||
if result["state"] != "resolved":
|
||||
text = None
|
||||
self.assertEqual(text, expected)
|
||||
self.assertNotEqual(text, legacy)
|
||||
|
||||
def test_no_graph_leaves_the_legacy_agreement(self):
|
||||
recorded = json.loads(AGREEMENT_PATH.read_text(encoding="utf-8"))
|
||||
current = {name: agreeing(graphs) for name, graphs in corpora().items()}
|
||||
self.assertEqual(set(recorded), set(current))
|
||||
for corpus, names in recorded.items():
|
||||
with self.subTest(corpus=corpus):
|
||||
lost = sorted(set(names) - set(current[corpus]))
|
||||
self.assertEqual(lost, [], "these graphs no longer match legacy")
|
||||
|
||||
def test_every_unknown_node_shape_legacy_reads_still_agrees(self):
|
||||
graphs = corpora()["unknown_nodes"]
|
||||
legacy = {name for name, pair in graphs.items() if legacy_positive(*pair)}
|
||||
self.assertLessEqual(legacy, set(agreeing(graphs)))
|
||||
|
||||
def test_the_reported_graph_reads_better_than_legacy(self):
|
||||
# Legacy found no positive here (cf0cef3 showed the positive text as
|
||||
# the negative); the floor is met, and the inspector now reads it.
|
||||
graph = unknown.any_switch_graph()
|
||||
self.assertIsNone(legacy_positive(graph, None))
|
||||
result = resolver.resolve_prompt_pair(
|
||||
{"prompt": graph}, profile=resolver.INSPECTION_PROFILE
|
||||
)
|
||||
self.assertEqual(result["positive"]["text"], unknown.POSITIVE)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--write", action="store_true", help="record the agreement")
|
||||
if parser.parse_args().write:
|
||||
current = {name: agreeing(graphs) for name, graphs in corpora().items()}
|
||||
text = json.dumps(current, indent=1, sort_keys=True, ensure_ascii=False)
|
||||
AGREEMENT_PATH.write_text(text + "\n", encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1118,17 +1118,8 @@ class LocalProjectionParityTests(unittest.TestCase):
|
||||
self.assertTrue({"parsed", "malformed", "limited"} <= seen)
|
||||
|
||||
|
||||
def _server_evidence(records, file_name=None):
|
||||
return inspect_prompt_evidence(
|
||||
records, None, profile=INSPECTION_PROFILE, file_name=file_name
|
||||
)
|
||||
|
||||
|
||||
FILE_NAME = resolver_cases.FILE_NAME
|
||||
|
||||
|
||||
def _records(case):
|
||||
return {key: value for key, value in case.items() if key != FILE_NAME}
|
||||
def _server_evidence(records):
|
||||
return inspect_prompt_evidence(records, None, profile=INSPECTION_PROFILE)
|
||||
|
||||
|
||||
class LocalResolverParityTests(unittest.TestCase):
|
||||
@@ -1152,12 +1143,7 @@ class LocalResolverParityTests(unittest.TestCase):
|
||||
observed = _run_oracle(
|
||||
self,
|
||||
[
|
||||
{
|
||||
"name": name,
|
||||
"mode": "resolve",
|
||||
"records": _records(records),
|
||||
"fileName": records.get(FILE_NAME),
|
||||
}
|
||||
{"name": name, "mode": "resolve", "records": records}
|
||||
for name, records in cases.items()
|
||||
],
|
||||
)
|
||||
@@ -1165,7 +1151,7 @@ class LocalResolverParityTests(unittest.TestCase):
|
||||
for name, records in cases.items():
|
||||
with self.subTest(case=name):
|
||||
local = observed[name]
|
||||
server = _server_evidence(_records(records), records.get(FILE_NAME))
|
||||
server = _server_evidence(records)
|
||||
answers[name] = server
|
||||
self.assertEqual(
|
||||
_canonical_bytes(local["evidence"]), _canonical_bytes(server)
|
||||
@@ -1204,7 +1190,7 @@ class LocalResolverParityTests(unittest.TestCase):
|
||||
# The single-chain subset (Task 8); the rest is Task 9's test below.
|
||||
if not resolver_cases.single_chain(records):
|
||||
continue
|
||||
server = _server_evidence(records, Path(path).name)
|
||||
server = _server_evidence(records)
|
||||
self.assertEqual(
|
||||
_canonical_bytes(local["evidence"]), _canonical_bytes(server)
|
||||
)
|
||||
@@ -1251,8 +1237,7 @@ class LocalResolverParityTests(unittest.TestCase):
|
||||
local, expect = observed[name], vector["expect"]
|
||||
try:
|
||||
server = _server_evidence(
|
||||
read_inspection_file(files[name]).metadata,
|
||||
Path(files[name]).name,
|
||||
read_inspection_file(files[name]).metadata
|
||||
)
|
||||
except MetadataReadError:
|
||||
server = unreadable_evidence()
|
||||
|
||||
+1
-158
@@ -115,12 +115,7 @@ class TestPairScan(AdminAPITestCase):
|
||||
self.assertEqual(second["pairs_added"], 0)
|
||||
|
||||
def test_legacy_single_prompt_second_pass_not_reported_as_new_link(self):
|
||||
# A sampler-less graph: one encoder wired into a guider (#252).
|
||||
single = {
|
||||
"1": graph()["1"],
|
||||
"2": {"class_type": "BasicGuider", "inputs": {"conditioning": ["1", 0]}},
|
||||
}
|
||||
meta = {"prompt": json.dumps(single)}
|
||||
meta = {"prompt": json.dumps({"1": graph()["1"]})}
|
||||
path = self.output_dir / "old.png"
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, meta), "added")
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, meta), "unchanged")
|
||||
@@ -297,155 +292,3 @@ class TestPairScan(AdminAPITestCase):
|
||||
self.api.db.get_image_prompt_info(str(path))["workflow_data"], workflow
|
||||
)
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, metadata), "unchanged")
|
||||
|
||||
|
||||
def flux_graph(text="a red fox", encoder="CLIPTextEncode"):
|
||||
"""A sampler-less Flux graph: the prompt reaches a guider's conditioning (#252)."""
|
||||
return {
|
||||
"1": {"class_type": "UNETLoader", "inputs": {"unet_name": "flux.safetensors"}},
|
||||
"2": {"class_type": "DualCLIPLoader", "inputs": {"clip_name1": "t5"}},
|
||||
"6": {"class_type": encoder, "inputs": {"text": text, "clip": ["2", 0]}},
|
||||
"22": {
|
||||
"class_type": "BasicGuider",
|
||||
"inputs": {"model": ["1", 0], "conditioning": ["6", 0]},
|
||||
},
|
||||
"13": {
|
||||
"class_type": "SamplerCustomAdvanced",
|
||||
"inputs": {"guider": ["22", 0], "sigmas": ["17", 0]},
|
||||
},
|
||||
"17": {"class_type": "BasicScheduler", "inputs": {"steps": 20}},
|
||||
"9": {"class_type": "SaveImage", "inputs": {"images": ["13", 0]}},
|
||||
}
|
||||
|
||||
|
||||
class TestSamplerLessScan(AdminAPITestCase):
|
||||
"""Graphs where no node takes a positive or negative input (#252)."""
|
||||
|
||||
def _scanned(self, name, prompt_graph, workflow=None):
|
||||
meta = {"prompt": json.dumps(prompt_graph)}
|
||||
if workflow is not None:
|
||||
meta["workflow"] = json.dumps(workflow)
|
||||
return self.api._ingest_scanned_file(self.output_dir / name, meta)
|
||||
|
||||
def _texts(self):
|
||||
return [p["text"] for p in self.api.db.get_recent_prompts()["prompts"]]
|
||||
|
||||
def test_the_one_wired_prompt_is_linked(self):
|
||||
self.assertEqual(self._scanned("flux.png", flux_graph()), "added")
|
||||
self.assertEqual(self._texts(), ["a red fox"])
|
||||
|
||||
def test_a_text_node_feeding_the_encoder_is_one_prompt(self):
|
||||
g = flux_graph(text=["30", 0])
|
||||
g["30"] = {"class_type": "TextInput", "inputs": {"text": "a red fox"}}
|
||||
self.assertEqual(self._scanned("textinput.png", g), "added")
|
||||
self.assertEqual(self._texts(), ["a red fox"])
|
||||
|
||||
def test_a_display_node_that_feeds_nothing_is_never_the_prompt(self):
|
||||
# A real scan stored a "=== GIT CLONE REPORT ===" display text this way.
|
||||
g = flux_graph()
|
||||
g["40"] = {
|
||||
"class_type": "easy showAnything",
|
||||
"inputs": {"text": "=== GIT CLONE REPORT ===\nTotal repositories: 1"},
|
||||
}
|
||||
self.assertEqual(self._scanned("report.png", g), "added")
|
||||
self.assertEqual(self._texts(), ["a red fox"])
|
||||
|
||||
def test_only_unconsumed_texts_store_nothing(self):
|
||||
g = {
|
||||
"40": {"class_type": "easy showAnything", "inputs": {"text": "report"}},
|
||||
"41": {"class_type": "Textbox", "inputs": {"text": "install note"}},
|
||||
}
|
||||
self.assertEqual(self._scanned("notes.png", g), "found")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_two_wired_texts_are_ambiguous_and_stay_unlinked(self):
|
||||
# A LoRA loader's text input is wired too: the scan stored "<lora:...>".
|
||||
g = flux_graph()
|
||||
g["50"] = {
|
||||
"class_type": "Lora Loader (LoraManager)",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"clip": ["2", 0],
|
||||
"text": "<lora:style:1.00>",
|
||||
},
|
||||
}
|
||||
g["22"]["inputs"]["model"] = ["50", 0]
|
||||
g["6"]["inputs"]["clip"] = ["50", 1]
|
||||
self.assertEqual(self._scanned("lora.png", g), "unresolved")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_an_untraceable_wired_text_makes_the_graph_ambiguous(self):
|
||||
# The encoder's text comes from a node no rule reads: never guess the other.
|
||||
g = flux_graph(text=["60", 0])
|
||||
g["60"] = {"class_type": "SomeLLMNode", "inputs": {"seed": 1}}
|
||||
g["50"] = {
|
||||
"class_type": "Lora Loader (LoraManager)",
|
||||
"inputs": {"model": ["1", 0], "text": "<lora:style:1.00>"},
|
||||
}
|
||||
g["22"]["inputs"]["model"] = ["50", 0]
|
||||
self.assertEqual(self._scanned("llm.png", g), "unresolved")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_this_repositorys_prompt_node_is_authoritative(self):
|
||||
g = flux_graph(text="a red fox", encoder="PromptManager")
|
||||
g["50"] = {
|
||||
"class_type": "Lora Loader (LoraManager)",
|
||||
"inputs": {"model": ["1", 0], "text": "<lora:style:1.00>"},
|
||||
}
|
||||
g["22"]["inputs"]["model"] = ["50", 0]
|
||||
self.assertEqual(self._scanned("own.png", g), "added")
|
||||
self.assertEqual(self._texts(), ["a red fox"])
|
||||
|
||||
def test_a_numeric_text_is_never_stored_as_a_prompt(self):
|
||||
self.assertEqual(self._scanned("number.png", flux_graph(text="2048")), "found")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_workflow_widget_values_are_never_read(self):
|
||||
# The API text is linked from a node no rule reads; the UI workflow
|
||||
# still holds a stale widget value, which the old scan stored.
|
||||
g = flux_graph(text=["70", 0])
|
||||
g["70"] = {"class_type": "PromptBuilder", "inputs": {"prompt": "x", "n": 2}}
|
||||
workflow = {
|
||||
"nodes": [
|
||||
{"id": 6, "type": "CLIPTextEncode", "widgets_values": ["stale text"]}
|
||||
]
|
||||
}
|
||||
self.assertEqual(self._scanned("stale.png", g, workflow), "unresolved")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_a_lora_loaders_trigger_word_output_is_not_its_text(self):
|
||||
# The encoder reads the loader's trigger-word output (slot 2), not its
|
||||
# text input: the prompt is unknown, never the "<lora:...>" syntax.
|
||||
g = flux_graph(text=["50", 2])
|
||||
g["50"] = {
|
||||
"class_type": "Lora Loader (LoraManager)",
|
||||
"inputs": {"model": ["1", 0], "text": "<lora:style:1.00>"},
|
||||
}
|
||||
g["22"]["inputs"]["model"] = ["50", 0]
|
||||
self.assertEqual(self._scanned("trigger.png", g), "unresolved")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_a_seeded_text_node_holds_a_template_not_the_prompt(self):
|
||||
g = flux_graph(text=["80", 0])
|
||||
g["80"] = {
|
||||
"class_type": "RandomPrompt",
|
||||
"inputs": {"text": "a {red|blue} fox", "seed": 5},
|
||||
}
|
||||
self.assertEqual(self._scanned("random.png", g), "unresolved")
|
||||
self.assertEqual(self._texts(), [])
|
||||
|
||||
def test_a1111_parameters_beside_an_undecided_graph_are_read(self):
|
||||
# Metadata savers write both; the old scan read the A1111 positive.
|
||||
g = flux_graph()
|
||||
g["50"] = {
|
||||
"class_type": "Lora Loader (LoraManager)",
|
||||
"inputs": {"model": ["1", 0], "text": "<lora:style:1.00>"},
|
||||
}
|
||||
g["22"]["inputs"]["model"] = ["50", 0]
|
||||
meta = {
|
||||
"prompt": json.dumps(g),
|
||||
"parameters": "a red fox\nNegative prompt: blur\nSteps: 20",
|
||||
}
|
||||
path = self.output_dir / "saver.png"
|
||||
self.assertEqual(self.api._ingest_scanned_file(path, meta), "added")
|
||||
self.assertEqual(self._texts(), ["a red fox"])
|
||||
|
||||
@@ -1019,8 +1019,7 @@ class TestResolverProfiles(unittest.TestCase):
|
||||
|
||||
def test_unsupported_node_names_the_blocking_node(self):
|
||||
source = graph()
|
||||
# No text and no links: nothing passes through it (#381).
|
||||
source["1"] = {"class_type": "UnknownEncoder", "inputs": {"seed": 1}}
|
||||
source["1"]["class_type"] = "UnknownEncoder"
|
||||
result = resolve_prompt_pair(source, profile=resolver.INSPECTION_PROFILE)
|
||||
self.assertEqual(result["reason_code"], "unsupported-node")
|
||||
self.assertEqual(result["node"], {"id": "1", "class_type": "UnknownEncoder"})
|
||||
@@ -1162,9 +1161,14 @@ class TestGuiderSamplers(unittest.TestCase):
|
||||
|
||||
def test_unsupported_node_names_first_blocking_node(self):
|
||||
source = _guided()
|
||||
# No links: nothing passes through them (#381).
|
||||
source["30"] = {"class_type": "FutureConditioner", "inputs": {"seed": 1}}
|
||||
source["31"] = {"class_type": "OtherConditioner", "inputs": {"seed": 2}}
|
||||
source["30"] = {
|
||||
"class_type": "FutureConditioner",
|
||||
"inputs": {"conditioning": ["6", 0]},
|
||||
}
|
||||
source["31"] = {
|
||||
"class_type": "OtherConditioner",
|
||||
"inputs": {"conditioning": ["7", 0]},
|
||||
}
|
||||
source["21"]["inputs"]["positive"] = ["30", 0]
|
||||
source["21"]["inputs"]["negative"] = ["31", 0]
|
||||
result = _inspect(source)
|
||||
@@ -1544,8 +1548,7 @@ class TestConditioningSemantics(unittest.TestCase):
|
||||
def test_diverging_branch_that_cannot_be_traced_is_never_guessed(self):
|
||||
extra = {
|
||||
"10": _encoded(parity.POS),
|
||||
# No text and no links: nothing passes through it (#381).
|
||||
"11": {"class_type": "FutureEncoder", "inputs": {"strength": 1.0}},
|
||||
"11": {"class_type": "FutureEncoder", "inputs": {"text": WOLF}},
|
||||
"12": _image_exists(),
|
||||
}
|
||||
source = _with_positive(_if_else(["12", 1], ["10", 0], ["11", 0]), extra)
|
||||
@@ -2045,17 +2048,12 @@ class TestEncoderAndStringSemantics(unittest.TestCase):
|
||||
result = _inspect(_with_positive(node))
|
||||
self.assertEqual(result["reason_code"], "malformed")
|
||||
|
||||
def test_unknown_text_nodes_pass_their_text_through(self):
|
||||
# #381: an unlisted node's ``text`` input is its text (the legacy rule);
|
||||
# without one, and with nothing linked, it is still named.
|
||||
def test_unverified_text_nodes_stay_unsupported_and_named(self):
|
||||
# TextBoxMira was verified in its source in Task 2.7b and is listed.
|
||||
for name in ("LoRAfromText", "RegexReplace", "XYZPrompt"):
|
||||
with self.subTest(node=name):
|
||||
node = {"class_type": name, "inputs": {"text": parity.POS}}
|
||||
result = _inspect(_string_positive(node))
|
||||
self.assertEqual(result["state"], "resolved")
|
||||
self.assertEqual(result["positive"]["text"], parity.POS)
|
||||
node = {"class_type": name, "inputs": {"pattern": parity.POS}}
|
||||
result = _inspect(_string_positive(node))
|
||||
self.assertEqual(result["reason_code"], "unsupported-node")
|
||||
self.assertEqual(result["node"], {"id": "10", "class_type": name})
|
||||
|
||||
@@ -2315,8 +2313,7 @@ class TestDetailerStages(unittest.TestCase):
|
||||
|
||||
def test_unresolved_stage_blocks_the_chain(self):
|
||||
source = _detailed(same_prompt=True)
|
||||
# No links: nothing passes through it (#381).
|
||||
source["30"] = {"class_type": "FutureConditioner", "inputs": {"c": 1.0}}
|
||||
source["30"] = {"class_type": "FutureConditioner", "inputs": {"c": ["6", 0]}}
|
||||
source["3"]["inputs"]["positive"] = ["30", 0]
|
||||
source["50"]["inputs"]["positive"] = ["10", 0]
|
||||
result = _inspect(source)
|
||||
@@ -2665,13 +2662,11 @@ class TestLatentAncestry(unittest.TestCase):
|
||||
},
|
||||
{
|
||||
"50": {"class_type": "LoadImage", "inputs": {"image": "a.png"}},
|
||||
# Its own encoder (#381: the same links would be one prompt).
|
||||
"73": parity._enc(parity.POS),
|
||||
"70": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["4", 0],
|
||||
"positive": ["73", 0],
|
||||
"positive": ["6", 0],
|
||||
"negative": ["7", 0],
|
||||
"latent_image": ["72", 0],
|
||||
},
|
||||
@@ -2974,8 +2969,7 @@ class TestLatentAncestry(unittest.TestCase):
|
||||
source["60"]["class_type"] = "UltimateSDUpscale"
|
||||
result = _inspect(source, output_node_id="9")
|
||||
self.assertEqual((result["state"], result["sampler_id"]), ("resolved", "3"))
|
||||
# Reached only through an unlisted image node, it stays unattributed:
|
||||
# a detailer is never a plain first stage, even on the same links (#381).
|
||||
# Reached only through an unlisted image node, it stays unattributed.
|
||||
source["61"] = {"class_type": "MysteryFilter", "inputs": {"image": ["60", 0]}}
|
||||
source["5"]["inputs"]["pixels"] = ["61", 0]
|
||||
result = _inspect(source, output_node_id="9")
|
||||
|
||||
@@ -1,391 +0,0 @@
|
||||
"""Unknown nodes between the prompt and the sampler pass through (#381).
|
||||
|
||||
The inspection resolver no longer needs a node class on an allow-list to read
|
||||
through it. An output the semantics table does not describe passes through: a
|
||||
``text`` input is the text, a ``positive``/``negative`` input routes by output
|
||||
slot, a ``conditioning`` input is followed alone, otherwise every linked input
|
||||
except core type inputs (``pass_through.skip_inputs``) is followed. One text,
|
||||
or several equal texts, resolve; PromptManager nodes are authoritative; several
|
||||
different texts are one candidate each. Ingestion (the default profile) is
|
||||
unchanged. Every graph here is synthetic, with neutral texts.
|
||||
"""
|
||||
|
||||
import json
|
||||
import unittest
|
||||
|
||||
from utils import prompt_pair_resolver as resolver
|
||||
from utils.prompt_inspector import inspect_prompt_evidence
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair
|
||||
|
||||
try:
|
||||
from tests import unknown_node_graphs as graphs
|
||||
except ImportError: # discovered with ``-s tests``
|
||||
import unknown_node_graphs as graphs
|
||||
|
||||
POSITIVE, OTHER, NEGATIVE = graphs.POSITIVE, graphs.OTHER, graphs.NEGATIVE
|
||||
any_switch_graph = graphs.any_switch_graph
|
||||
_encode, _pair, _with = graphs.encode, graphs.pair, graphs.with_nodes
|
||||
|
||||
|
||||
def _inspect(graph, **options):
|
||||
return resolve_prompt_pair(
|
||||
{"prompt": graph}, profile=resolver.INSPECTION_PROFILE, **options
|
||||
)
|
||||
|
||||
|
||||
def _texts(result):
|
||||
return tuple(
|
||||
result[role]["text"] if isinstance(result[role], dict) else None
|
||||
for role in ("positive", "negative")
|
||||
)
|
||||
|
||||
|
||||
class TestTheReportedGraph(unittest.TestCase):
|
||||
def test_prompt_manager_pair_through_any_switch_resolves(self):
|
||||
result = _inspect(any_switch_graph())
|
||||
self.assertEqual(result["state"], "resolved", result["reason"])
|
||||
self.assertEqual(_texts(result), (POSITIVE, ""))
|
||||
self.assertEqual(result["sampler_id"], "17")
|
||||
self.assertIsNone(result["reason_code"])
|
||||
path = [step["node_id"] for step in result["positive"]["provenance"]["path"]]
|
||||
self.assertEqual(path, ["15", "10"])
|
||||
|
||||
def test_the_inspector_shows_the_prompt(self):
|
||||
evidence = inspect_prompt_evidence({"prompt": json.dumps(any_switch_graph())})
|
||||
self.assertEqual(evidence["resolution"]["state"], "resolved")
|
||||
self.assertEqual(evidence["prompts"]["positive"]["status"], "resolved")
|
||||
self.assertEqual(evidence["prompts"]["positive"]["text"], POSITIVE)
|
||||
|
||||
def test_ingestion_is_unchanged(self):
|
||||
result = resolve_prompt_pair({"prompt": any_switch_graph()})
|
||||
self.assertEqual(result["state"], "unresolved")
|
||||
# Its first failure is the custom latent node, as before this change.
|
||||
self.assertEqual(result["reason"], "Unsupported latent ancestry")
|
||||
|
||||
def test_the_inspection_version_moves(self):
|
||||
self.assertEqual(resolver.INSPECTION_RESOLVER_VERSION, 2)
|
||||
self.assertEqual(resolver.RESOLVER_VERSION, 1)
|
||||
|
||||
|
||||
class TestSourcesThroughUnknownNodes(unittest.TestCase):
|
||||
def test_several_inputs_with_one_text_resolve(self):
|
||||
graph = any_switch_graph(any_01=["10", 0], any_02=["10", 0])
|
||||
self.assertEqual(_texts(_inspect(graph)), (POSITIVE, ""))
|
||||
|
||||
def test_equal_texts_from_different_nodes_resolve(self):
|
||||
graph = _with(
|
||||
any_switch_graph(any_01=["20", 0], any_02=["21", 0]),
|
||||
**{"20": _encode(OTHER), "21": _encode(OTHER)},
|
||||
)
|
||||
result = _inspect(graph)
|
||||
self.assertEqual(result["state"], "resolved")
|
||||
self.assertEqual(_texts(result)[0], OTHER)
|
||||
|
||||
def test_different_texts_are_candidates_not_a_refusal(self):
|
||||
graph = _with(
|
||||
any_switch_graph(any_01=["20", 0], any_02=["21", 0]),
|
||||
**{"20": _encode(POSITIVE), "21": _encode(OTHER)},
|
||||
)
|
||||
result = _inspect(graph)
|
||||
self.assertEqual(result["state"], "ambiguous")
|
||||
self.assertEqual(result["reason"], resolver.SEVERAL_SOURCES_REASON)
|
||||
self.assertEqual(
|
||||
[_texts(item) for item in result["candidates"]],
|
||||
[(POSITIVE, ""), (OTHER, "")],
|
||||
)
|
||||
for item in result["candidates"]:
|
||||
self.assertEqual(item["state"], "resolved")
|
||||
self.assertEqual(item["sampler_id"], "17")
|
||||
|
||||
def test_prompt_manager_is_authoritative(self):
|
||||
graph = _with(
|
||||
any_switch_graph(any_01=["20", 0], any_02=["10", 0]),
|
||||
**{"20": _encode(OTHER)},
|
||||
)
|
||||
result = _inspect(graph)
|
||||
self.assertEqual(result["state"], "resolved")
|
||||
self.assertEqual(_texts(result), (POSITIVE, ""))
|
||||
|
||||
def test_prompt_manager_wins_over_an_unreadable_branch(self):
|
||||
graph = _with(
|
||||
any_switch_graph(any_01=["20", 0], any_02=["10", 0]),
|
||||
**{
|
||||
"20": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": ["21", 0], "clip": ["7", 0]},
|
||||
},
|
||||
"21": {"class_type": "PromptSearchList", "inputs": {"search": "x"}},
|
||||
},
|
||||
)
|
||||
self.assertEqual(_texts(_inspect(graph)), (POSITIVE, ""))
|
||||
|
||||
def test_an_unreadable_branch_blocks_without_prompt_manager(self):
|
||||
graph = _with(
|
||||
any_switch_graph(any_01=["20", 0], any_02=["22", 0]),
|
||||
**{
|
||||
"20": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": ["21", 0], "clip": ["7", 0]},
|
||||
},
|
||||
"21": {"class_type": "PromptSearchList", "inputs": {"search": "x"}},
|
||||
"22": _encode(OTHER),
|
||||
},
|
||||
)
|
||||
result = _inspect(graph)
|
||||
self.assertEqual(result["state"], "unresolved")
|
||||
self.assertEqual(result["reason_code"], "runtime-value")
|
||||
|
||||
def test_chained_unknown_nodes(self):
|
||||
graph = _with(
|
||||
any_switch_graph(any_01=["20", 0]),
|
||||
**{
|
||||
"20": {
|
||||
"class_type": "Router Node (vendor)",
|
||||
"inputs": {"in": ["10", 0]},
|
||||
}
|
||||
},
|
||||
)
|
||||
self.assertEqual(_texts(_inspect(graph)), (POSITIVE, ""))
|
||||
|
||||
def test_no_text_names_the_unknown_node(self):
|
||||
graph = any_switch_graph(any_01=["7", 0])
|
||||
result = _inspect(graph)
|
||||
self.assertEqual(result["state"], "unresolved")
|
||||
self.assertEqual(result["reason_code"], "unsupported-node")
|
||||
self.assertEqual(
|
||||
result["node"], {"id": "15", "class_type": "Any Switch (rgthree)"}
|
||||
)
|
||||
|
||||
def test_another_sampler_is_never_entered(self):
|
||||
graph = any_switch_graph(any_01=["17", 0], any_02=["10", 0])
|
||||
graph["30"] = {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {**graph["17"]["inputs"], "positive": ["31", 0]},
|
||||
}
|
||||
graph["31"] = _encode(OTHER)
|
||||
graph["15"]["inputs"]["any_01"] = ["30", 0]
|
||||
result = _inspect(graph)
|
||||
found = {item["sampler_id"]: _texts(item) for item in result["candidates"]}
|
||||
self.assertEqual(found["17"], (POSITIVE, ""))
|
||||
|
||||
|
||||
class TestLegacyShapedRules(unittest.TestCase):
|
||||
def test_a_text_input_is_the_text(self):
|
||||
graph = _with(
|
||||
any_switch_graph(any_01=["20", 0]),
|
||||
**{
|
||||
"20": {
|
||||
"class_type": "VendorTextEncode",
|
||||
"inputs": {"text": OTHER, "clip": ["9", 1]},
|
||||
}
|
||||
},
|
||||
)
|
||||
self.assertEqual(_texts(_inspect(graph))[0], OTHER)
|
||||
|
||||
def test_role_inputs_route_by_output_slot(self):
|
||||
graph = any_switch_graph()
|
||||
graph["20"] = {
|
||||
"class_type": "VendorControlApply",
|
||||
"inputs": {
|
||||
"positive": ["21", 0],
|
||||
"negative": ["22", 0],
|
||||
"control_net": ["8", 0],
|
||||
"image": ["8", 0],
|
||||
},
|
||||
}
|
||||
graph["21"] = _encode(POSITIVE)
|
||||
graph["22"] = _encode(NEGATIVE)
|
||||
graph["17"]["inputs"]["positive"] = ["20", 0]
|
||||
graph["17"]["inputs"]["negative"] = ["20", 1]
|
||||
self.assertEqual(_texts(_inspect(graph)), (POSITIVE, NEGATIVE))
|
||||
|
||||
def test_a_conditioning_input_is_followed_alone(self):
|
||||
graph = _with(
|
||||
any_switch_graph(any_01=["20", 0]),
|
||||
**{
|
||||
"20": {
|
||||
"class_type": "VendorStyleApply",
|
||||
"inputs": {"conditioning": ["10", 0], "style": ["21", 0]},
|
||||
},
|
||||
"21": _encode(OTHER),
|
||||
},
|
||||
)
|
||||
self.assertEqual(_texts(_inspect(graph)), (POSITIVE, ""))
|
||||
|
||||
def test_core_type_inputs_are_skipped(self):
|
||||
# The vendor encoder's ``clip`` reaches a loader with its own ``text``:
|
||||
# that text is a lora tag, never the prompt.
|
||||
graph = _with(
|
||||
any_switch_graph(any_01=["20", 0]),
|
||||
**{
|
||||
"20": {
|
||||
"class_type": "VendorPromptEncode",
|
||||
"inputs": {"prompt": ["21", 0], "clip": ["22", 1]},
|
||||
},
|
||||
"21": {"class_type": "PrimitiveString", "inputs": {"value": OTHER}},
|
||||
"22": {
|
||||
"class_type": "VendorLoraLoader",
|
||||
"inputs": {"text": "<lora:x:1>", "clip": ["7", 0]},
|
||||
},
|
||||
},
|
||||
)
|
||||
result = _inspect(graph)
|
||||
self.assertEqual(result["state"], "resolved")
|
||||
self.assertEqual(_texts(result)[0], OTHER)
|
||||
|
||||
|
||||
class TestBounds(unittest.TestCase):
|
||||
def test_a_long_unknown_chain_reaches_the_depth_limit(self):
|
||||
graph = any_switch_graph(any_01=["100", 0])
|
||||
for index in range(100, 100 + resolver.INSPECTION_PROFILE.max_depth + 2):
|
||||
graph[str(index)] = {
|
||||
"class_type": "Router Node (vendor)",
|
||||
"inputs": {"in": [str(index + 1), 0]},
|
||||
}
|
||||
graph[str(index + 1)] = _pair()
|
||||
result = _inspect(graph)
|
||||
self.assertEqual(result["state"], "unresolved")
|
||||
self.assertEqual(result["reason_code"], "depth-limit")
|
||||
|
||||
def test_many_different_sources_hit_the_candidate_cap(self):
|
||||
inputs = {}
|
||||
graph = any_switch_graph()
|
||||
for index in range(resolver._MAX_BRANCH_CANDIDATES + 1):
|
||||
graph[str(200 + index)] = _encode(f"{OTHER} {index}")
|
||||
inputs[f"any_{index:02}"] = [str(200 + index), 0]
|
||||
graph["15"]["inputs"] = inputs
|
||||
result = _inspect(graph)
|
||||
self.assertEqual(result["state"], "unresolved")
|
||||
self.assertEqual(result["reason_code"], "work-limit")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestGeneratedAndTransformedText(unittest.TestCase):
|
||||
def test_a_seeded_text_node_is_generated_at_run_time(self):
|
||||
result = _inspect(graphs.generated_text_graph())
|
||||
self.assertEqual(result["state"], "unresolved")
|
||||
self.assertEqual(result["reason_code"], "runtime-value")
|
||||
self.assertEqual(
|
||||
result["node"], {"id": "30", "class_type": "VendorTextGenerate"}
|
||||
)
|
||||
|
||||
def test_the_recorded_text_is_authoritative(self):
|
||||
# The record is what ran: a generated or transformed text is never
|
||||
# checked against the generator's or transform's inputs.
|
||||
for name, records in graphs.recorded_shapes().items():
|
||||
with self.subTest(shape=name):
|
||||
metadata = {
|
||||
"prompt": json.loads(records["prompt"]),
|
||||
"comfyassets_prompt_pairs": records["comfyassets_prompt_pairs"],
|
||||
}
|
||||
result = resolve_prompt_pair(
|
||||
metadata, profile=resolver.INSPECTION_PROFILE
|
||||
)
|
||||
self.assertEqual(result["state"], "resolved", result["reason"])
|
||||
self.assertEqual(_texts(result), (graphs.GENERATED, ""))
|
||||
self.assertEqual(
|
||||
result["positive"]["provenance"]["text_source"], "execution"
|
||||
)
|
||||
|
||||
def test_a_seeded_conditioning_node_still_passes_through(self):
|
||||
graph = _with(
|
||||
any_switch_graph(any_01=["20", 0]),
|
||||
**{
|
||||
"20": {
|
||||
"class_type": "VendorConditioningNoise",
|
||||
"inputs": {"conditioning": ["10", 0], "seed": 3},
|
||||
}
|
||||
},
|
||||
)
|
||||
self.assertEqual(_texts(_inspect(graph)), (POSITIVE, ""))
|
||||
|
||||
|
||||
class TestReviewFindings(unittest.TestCase):
|
||||
"""Shapes from the independent review of the first #381 commit."""
|
||||
|
||||
def _recorded(self, text_source: dict) -> dict:
|
||||
graph = any_switch_graph()
|
||||
graph["10"]["inputs"]["text"] = ["30", 0]
|
||||
graph.update(text_source)
|
||||
metadata = {
|
||||
"prompt": graph,
|
||||
"comfyassets_prompt_pairs": graphs.execution("10", graphs.GENERATED),
|
||||
}
|
||||
return resolve_prompt_pair(metadata, profile=resolver.INSPECTION_PROFILE)
|
||||
|
||||
def test_a_failing_walk_never_overrides_the_recorded_text(self):
|
||||
sources = {
|
||||
"missing": {"30": {"class_type": "XLLM", "inputs": {"src": ["99", 0]}}},
|
||||
"composite": {
|
||||
"30": {"class_type": "XLLM", "inputs": {"src": ["31", 0]}},
|
||||
"31": {
|
||||
"class_type": "ConditioningCombine",
|
||||
"inputs": {
|
||||
"conditioning_1": ["32", 0],
|
||||
"conditioning_2": ["33", 0],
|
||||
},
|
||||
},
|
||||
"32": _encode(POSITIVE),
|
||||
"33": _encode(OTHER),
|
||||
},
|
||||
"several": {
|
||||
"30": {
|
||||
"class_type": "XLLM",
|
||||
"inputs": {"a": ["31", 0], "b": ["32", 0]},
|
||||
},
|
||||
"31": {"class_type": "PrimitiveString", "inputs": {"value": POSITIVE}},
|
||||
"32": {"class_type": "PrimitiveString", "inputs": {"value": OTHER}},
|
||||
},
|
||||
"list-text": {"30": {"class_type": "XLLM", "inputs": {"text": [1, 2, 3]}}},
|
||||
}
|
||||
for name, source in sources.items():
|
||||
with self.subTest(source=name):
|
||||
result = self._recorded(source)
|
||||
self.assertEqual(result["state"], "resolved", result["reason"])
|
||||
self.assertEqual(_texts(result)[0], graphs.GENERATED)
|
||||
|
||||
def test_the_seed_rule_reads_only_string_outputs(self):
|
||||
graph = _with(
|
||||
any_switch_graph(any_01=["20", 0]),
|
||||
**{
|
||||
"20": {"class_type": "XNoise", "inputs": {"conditioning": ["21", 0]}},
|
||||
"21": {
|
||||
"class_type": "XSeedEncode",
|
||||
"inputs": {"clip": ["7", 0], "text": OTHER, "seed": 3},
|
||||
},
|
||||
},
|
||||
)
|
||||
self.assertEqual(_texts(_inspect(graph))[0], OTHER)
|
||||
|
||||
def test_an_unlinked_role_input_adds_nothing(self):
|
||||
graph = any_switch_graph()
|
||||
graph["20"] = {
|
||||
"class_type": "XApply",
|
||||
"inputs": {"positive": ["21", 0], "negative_opt": ""},
|
||||
}
|
||||
graph["21"] = _encode(POSITIVE)
|
||||
graph["17"]["inputs"]["positive"] = ["20", 0]
|
||||
graph["17"]["inputs"]["negative"] = ["20", 1]
|
||||
result = _inspect(graph)
|
||||
self.assertEqual(result["state"], "unresolved")
|
||||
self.assertEqual(result["reason_code"], "unsupported-node")
|
||||
self.assertEqual(result["node"], {"id": "20", "class_type": "XApply"})
|
||||
|
||||
def test_one_text_offered_twice_is_one_candidate(self):
|
||||
graph = _with(
|
||||
any_switch_graph(any_01=["20", 0], any_02=["23", 0]),
|
||||
**{
|
||||
"20": {"class_type": "XB", "inputs": {"p": ["21", 0], "q": ["22", 0]}},
|
||||
"21": _encode(POSITIVE),
|
||||
"22": _encode(OTHER),
|
||||
"23": _encode(POSITIVE),
|
||||
},
|
||||
)
|
||||
result = _inspect(graph)
|
||||
self.assertEqual(result["state"], "ambiguous")
|
||||
self.assertEqual(
|
||||
[_texts(item)[0] for item in result["candidates"]], [POSITIVE, OTHER]
|
||||
)
|
||||
@@ -72,24 +72,16 @@ class TestHiresFixChain(unittest.TestCase):
|
||||
def test_an_unlisted_image_node_breaks_the_lineage(self):
|
||||
# The walk still finds the earlier sampler, but not through listed
|
||||
# pass-through nodes: the stages are never compared.
|
||||
graph = shapes.hires_fix(
|
||||
second_positive=shapes.POSITIVE, filter_class="MysteryImageEffect"
|
||||
)
|
||||
result = _inspect(graph)
|
||||
result = _inspect(shapes.hires_fix(filter_class="MysteryImageEffect"))
|
||||
self.assertEqual(result["state"], "ambiguous")
|
||||
later = next(
|
||||
item for item in result["candidates"] if item["sampler_id"] == "20"
|
||||
)
|
||||
self.assertEqual(later["reason"], resolver.MULTISTAGE_UNATTRIBUTED_REASON)
|
||||
# Stages on the very same conditioning links are one prompt (#381).
|
||||
result = _inspect(shapes.hires_fix(filter_class="MysteryImageEffect"))
|
||||
self.assertEqual(result["state"], "resolved")
|
||||
|
||||
def test_a_second_earlier_sampler_in_the_ancestry_stays_unattributed(self):
|
||||
graph = shapes.hires_fix()
|
||||
# Its own encoder: the same links would be one prompt (#381).
|
||||
graph["33"] = shapes.encode(shapes.POSITIVE)
|
||||
graph["30"] = shapes.ksampler(["33", 0], ["3", 0], ["4", 0])
|
||||
graph["30"] = shapes.ksampler(["2", 0], ["3", 0], ["4", 0])
|
||||
graph["31"] = {"class_type": "VAEDecode", "inputs": {"samples": ["30", 0]}}
|
||||
graph["32"] = {
|
||||
"class_type": "ImageBlend",
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
"""Saver anchoring by file name and shared-conditioning stages (#381).
|
||||
|
||||
An inspected file's name anchors the walk on the one saver whose
|
||||
``filename_prefix`` (literal or traced) makes ``<prefix>_<digits>[_]``; the
|
||||
samplers in that saver's image ancestry are the candidates. When no saver or
|
||||
more than one matches, nothing is guessed: the file stays "several possible
|
||||
prompts". A later stage reading the same positive and negative links as the
|
||||
earlier stage has the same prompt whatever its latent ancestry. Shapes come
|
||||
from the maintainer's runs (``tests/saver_anchor_graphs.py``).
|
||||
"""
|
||||
|
||||
import json
|
||||
import unittest
|
||||
|
||||
from utils import prompt_pair_resolver as resolver
|
||||
from utils.prompt_inspector import inspect_prompt_evidence
|
||||
|
||||
try:
|
||||
from tests import saver_anchor_graphs as shapes
|
||||
except ImportError: # discovered with ``-s tests``
|
||||
import saver_anchor_graphs as shapes
|
||||
|
||||
|
||||
def _inspect(graph, file_name=None):
|
||||
return resolver.resolve_prompt_pair(
|
||||
{"prompt": graph}, profile=resolver.INSPECTION_PROFILE, file_name=file_name
|
||||
)
|
||||
|
||||
|
||||
def _positive(result):
|
||||
side = result["positive"]
|
||||
return side["text"] if isinstance(side, dict) else None
|
||||
|
||||
|
||||
class TestSaverAnchor(unittest.TestCase):
|
||||
def test_the_file_name_picks_its_branch(self):
|
||||
for index, text in enumerate(shapes.BRANCH_TEXTS, start=1):
|
||||
for name in (
|
||||
f"s45_branch_{index}_00002_.png",
|
||||
f"s45_branch_{index}_00002.webp",
|
||||
):
|
||||
with self.subTest(file=name):
|
||||
result = _inspect(shapes.branch_bench(), name)
|
||||
self.assertEqual(result["state"], "resolved")
|
||||
self.assertEqual(_positive(result), text)
|
||||
self.assertEqual(result["sampler_id"], str(index * 10 + 1))
|
||||
|
||||
def test_unanchored_branches_stay_several(self):
|
||||
self.assertEqual(_inspect(shapes.branch_bench())["state"], "ambiguous")
|
||||
|
||||
def test_a_traced_prefix_anchors(self):
|
||||
result = _inspect(shapes.linked_prefix_bench(), "s45_branch_3_00007_.png")
|
||||
self.assertEqual(_positive(result), shapes.BRANCH_TEXTS[2])
|
||||
|
||||
def test_an_unknown_saver_class_anchors_too(self):
|
||||
result = _inspect(
|
||||
shapes.branch_bench(saver="VendorSaveImage"), "s45_branch_2_00001_.png"
|
||||
)
|
||||
self.assertEqual(_positive(result), shapes.BRANCH_TEXTS[1])
|
||||
|
||||
def test_no_guessing_when_the_name_matches_none_or_several(self):
|
||||
cases = {
|
||||
"no saver matches": (shapes.branch_bench(), "photo_00001_.png"),
|
||||
"not a counter": (shapes.branch_bench(), "s45_branch_3_final.png"),
|
||||
"prefix of another branch": (
|
||||
shapes.branch_bench(),
|
||||
"s45_branch_00001_.png",
|
||||
),
|
||||
"shared prefix": (shapes.shared_prefix(), "IMG_GEN_00371_.png"),
|
||||
}
|
||||
for label, (graph, name) in cases.items():
|
||||
with self.subTest(case=label):
|
||||
result = _inspect(graph, name)
|
||||
self.assertEqual(result["state"], "ambiguous")
|
||||
self.assertEqual(len(result["candidates"]), len(shapes.BRANCH_TEXTS))
|
||||
|
||||
def test_the_inspector_passes_the_file_name(self):
|
||||
records = {"prompt": json.dumps(shapes.branch_bench())}
|
||||
evidence = inspect_prompt_evidence(records, file_name="s45_branch_4_00009_.png")
|
||||
self.assertEqual(
|
||||
evidence["prompts"]["positive"]["text"], shapes.BRANCH_TEXTS[3]
|
||||
)
|
||||
self.assertEqual(
|
||||
inspect_prompt_evidence(records)["resolution"]["state"], "ambiguous"
|
||||
)
|
||||
|
||||
def test_ingestion_ignores_the_file_name(self):
|
||||
result = resolver.resolve_prompt_pair(
|
||||
{"prompt": shapes.branch_bench()}, file_name="s45_branch_1_00001_.png"
|
||||
)
|
||||
self.assertEqual(result["state"], "ambiguous")
|
||||
|
||||
|
||||
class TestSharedConditioningStages(unittest.TestCase):
|
||||
def test_a_later_stage_on_the_same_links_resolves(self):
|
||||
for name in (None, "final_00003_.png"):
|
||||
with self.subTest(file=name):
|
||||
result = _inspect(shapes.two_stage_same_conditioning(), name)
|
||||
self.assertEqual(result["state"], "resolved", result["reason"])
|
||||
self.assertEqual(_positive(result), shapes.BRANCH_TEXTS[0])
|
||||
|
||||
def test_different_links_stay_unattributed(self):
|
||||
graph = shapes.two_stage_same_conditioning(second_positive=["91", 0])
|
||||
result = _inspect(graph, "final_00003_.png")
|
||||
self.assertEqual(result["state"], "unresolved")
|
||||
self.assertEqual(result["reason"], resolver.MULTISTAGE_UNATTRIBUTED_REASON)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestReviewFindings(unittest.TestCase):
|
||||
"""Shapes from the independent review of the saver-anchor commit."""
|
||||
|
||||
def test_a_skipped_stage_never_hides_its_own_earlier_prompt(self):
|
||||
result = _inspect(shapes.hidden_earlier_prompt(), "out_00001_.png")
|
||||
self.assertNotEqual(result["state"], "resolved")
|
||||
self.assertEqual(result["reason"], resolver.MULTISTAGE_UNATTRIBUTED_REASON)
|
||||
|
||||
def test_a_skipped_stage_never_hides_its_override(self):
|
||||
result = _inspect(shapes.hidden_earlier_prompt(override=True), "out_00001_.png")
|
||||
self.assertNotEqual(result["state"], "resolved")
|
||||
|
||||
def test_an_unfinished_prefix_scan_anchors_nothing(self):
|
||||
graph = shapes.deep_prefixes(9)
|
||||
for visits in range(10, 30):
|
||||
with self.subTest(visits=visits):
|
||||
result = resolver.resolve_prompt_pair(
|
||||
{"prompt": graph},
|
||||
profile=resolver.INSPECTION_PROFILE,
|
||||
max_visits=visits,
|
||||
file_name="dup_00001_.png",
|
||||
)
|
||||
self.assertNotEqual(result["state"], "resolved")
|
||||
|
||||
|
||||
class TestPlainFirstStage(unittest.TestCase):
|
||||
def test_only_a_plain_first_stage_is_skipped(self):
|
||||
expected = {
|
||||
"no-latent": "resolved",
|
||||
"literal-latent": "unresolved",
|
||||
"missing-latent": "unresolved",
|
||||
"cyclic-latent": "unresolved",
|
||||
"stage-shaped": "unresolved",
|
||||
"detailer": "unresolved",
|
||||
}
|
||||
for kind, state in expected.items():
|
||||
with self.subTest(kind=kind):
|
||||
result = _inspect(shapes.first_stage_variant(kind), "final_00003_.png")
|
||||
self.assertEqual(result["state"], state, result["reason"])
|
||||
@@ -864,11 +864,7 @@ class TestInspectionResolverVersion(InspectionTestCase):
|
||||
}
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
review_capture,
|
||||
"INSPECTION_RESOLVER_VERSION",
|
||||
review_capture.INSPECTION_RESOLVER_VERSION + 1,
|
||||
),
|
||||
patch.object(review_capture, "INSPECTION_RESOLVER_VERSION", 2),
|
||||
patch.object(review_capture, "resolve_inspection", side_effect=newer),
|
||||
):
|
||||
self.assertNotEqual(_version(), version)
|
||||
|
||||
@@ -371,10 +371,7 @@ class ReviewSchemaTests(unittest.TestCase):
|
||||
self.require_module()
|
||||
for label, dropped in (
|
||||
("annotations only", TABLES),
|
||||
(
|
||||
"predates annotations",
|
||||
("media_image_autotags",) + ANNOTATION_TABLES + TABLES,
|
||||
),
|
||||
("predates annotations", ANNOTATION_TABLES + TABLES),
|
||||
):
|
||||
with self.subTest(label):
|
||||
path = str(Path(self.temp.name) / f"{label.replace(' ', '-')}.db")
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
"""``sampler_less_prompt``: the one wired text of a graph with no sampler (#252)."""
|
||||
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from utils.prompt_pair_resolver import sampler_less_prompt
|
||||
|
||||
|
||||
def _guided(**nodes):
|
||||
"""``nodes`` plus a guider consuming node ``"1"``."""
|
||||
graph = {"9": {"class_type": "BasicGuider", "inputs": {"conditioning": ["1", 0]}}}
|
||||
graph.update(nodes)
|
||||
return graph
|
||||
|
||||
|
||||
class TestSamplerLessPrompt(unittest.TestCase):
|
||||
def test_not_a_graph_or_too_large_is_absent(self):
|
||||
for graph in (None, [], {}, "text"):
|
||||
self.assertEqual(sampler_less_prompt(graph)["state"], "absent")
|
||||
big = {str(i): {"class_type": "X", "inputs": {}} for i in range(3)}
|
||||
self.assertEqual(sampler_less_prompt(big, max_visits=2)["state"], "absent")
|
||||
|
||||
def test_one_literal_text(self):
|
||||
graph = _guided(**{"1": {"class_type": "Enc", "inputs": {"text": " a fox "}}})
|
||||
self.assertEqual(
|
||||
sampler_less_prompt(graph),
|
||||
{"state": "resolved", "text": "a fox", "texts": ["a fox"]},
|
||||
)
|
||||
|
||||
def test_a_number_literal_is_never_a_prompt(self):
|
||||
for value in (2048, "2048", "1.5", " 7 "):
|
||||
graph = _guided(**{"1": {"class_type": "Enc", "inputs": {"text": value}}})
|
||||
self.assertEqual(sampler_less_prompt(graph)["state"], "absent", value)
|
||||
|
||||
def test_a_long_digit_run_with_a_letter_is_text_and_fast(self):
|
||||
text = "9" * 50_000 + "x"
|
||||
graph = _guided(**{"1": {"class_type": "Enc", "inputs": {"text": text}}})
|
||||
started = time.monotonic()
|
||||
self.assertEqual(sampler_less_prompt(graph)["text"], text)
|
||||
self.assertLess(time.monotonic() - started, 1.0)
|
||||
|
||||
def test_number_like_words_are_prompts(self):
|
||||
for value in ("nan", "inf", "Infinity", "1_000"):
|
||||
graph = _guided(**{"1": {"class_type": "Enc", "inputs": {"text": value}}})
|
||||
self.assertEqual(sampler_less_prompt(graph)["text"], value)
|
||||
|
||||
def test_an_own_node_with_unreadable_text_beside_another_text_is_ambiguous(self):
|
||||
graph = _guided(
|
||||
**{
|
||||
"1": {"class_type": "PromptManager", "inputs": {"text": ["5", 0]}},
|
||||
"5": {"class_type": "LLM", "inputs": {"seed": 3}},
|
||||
"2": {"class_type": "Enc", "inputs": {"text": "<lora:x:1>"}},
|
||||
"3": {"class_type": "Sink", "inputs": {"c": ["2", 0]}},
|
||||
}
|
||||
)
|
||||
self.assertEqual(sampler_less_prompt(graph)["state"], "ambiguous")
|
||||
|
||||
def test_unhashable_class_type_is_an_unknown_node_not_a_crash(self):
|
||||
graph = _guided(**{"1": {"class_type": ["x"], "inputs": {"text": "a fox"}}})
|
||||
self.assertEqual(sampler_less_prompt(graph)["text"], "a fox")
|
||||
|
||||
def test_own_negative_node_is_not_authoritative(self):
|
||||
graph = _guided(
|
||||
**{
|
||||
"1": {"class_type": "Enc", "inputs": {"text": "a fox"}},
|
||||
"2": {
|
||||
"class_type": "PromptManagerNegative",
|
||||
"inputs": {"text": "blur"},
|
||||
},
|
||||
"3": {"class_type": "Sink", "inputs": {"c": ["2", 0]}},
|
||||
}
|
||||
)
|
||||
found = sampler_less_prompt(graph)
|
||||
self.assertEqual(found["state"], "ambiguous")
|
||||
self.assertEqual(found["texts"], ["a fox", "blur"])
|
||||
|
||||
def test_own_node_text_wins_over_an_untraceable_text(self):
|
||||
graph = _guided(
|
||||
**{
|
||||
"1": {"class_type": "PromptManagerText", "inputs": {"text": "a fox"}},
|
||||
"2": {"class_type": "Enc", "inputs": {"text": ["5", 0]}},
|
||||
"5": {"class_type": "LLM", "inputs": {"seed": 3}},
|
||||
"6": {"class_type": "Sink", "inputs": {"c": ["2", 0]}},
|
||||
}
|
||||
)
|
||||
self.assertEqual(sampler_less_prompt(graph)["text"], "a fox")
|
||||
|
||||
def test_two_own_texts_are_ambiguous(self):
|
||||
graph = _guided(
|
||||
**{
|
||||
"1": {"class_type": "PromptManager", "inputs": {"text": "a fox"}},
|
||||
"2": {"class_type": "PromptManager", "inputs": {"text": "a cat"}},
|
||||
"3": {"class_type": "Sink", "inputs": {"c": ["2", 0]}},
|
||||
}
|
||||
)
|
||||
self.assertEqual(sampler_less_prompt(graph)["state"], "ambiguous")
|
||||
|
||||
def test_only_an_untraceable_text_is_ambiguous_with_nothing_to_show(self):
|
||||
graph = _guided(
|
||||
**{
|
||||
"1": {"class_type": "Enc", "inputs": {"text": ["5", 0]}},
|
||||
"5": {"class_type": "LLM", "inputs": {"seed": 3}},
|
||||
}
|
||||
)
|
||||
self.assertEqual(
|
||||
sampler_less_prompt(graph),
|
||||
{"state": "ambiguous", "text": None, "texts": []},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,321 +0,0 @@
|
||||
"""Synthetic graphs with unknown nodes between the prompt and the sampler (#381).
|
||||
|
||||
``any_switch_graph`` reproduces the shape of the maintainer's krea2 run
|
||||
(PromptManagerPair -> Any Switch (rgthree) -> KSampler, a LoraManager loader
|
||||
with its own lora ``text``, custom latent, filename and saver nodes) with
|
||||
neutral texts. ``shapes()`` names one graph per pass-through rule; the resolver
|
||||
tests and the browser-port parity vectors share them. No user file is involved.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
POSITIVE = "a lighthouse on a cliff, golden hour"
|
||||
OTHER = "a red bicycle in the rain"
|
||||
NEGATIVE = "blurry, lowres"
|
||||
|
||||
|
||||
def encode(text):
|
||||
return {"class_type": "CLIPTextEncode", "inputs": {"text": text, "clip": ["7", 0]}}
|
||||
|
||||
|
||||
def pair(text=POSITIVE, negative=""):
|
||||
return {
|
||||
"class_type": "PromptManagerPair",
|
||||
"inputs": {
|
||||
"text": text,
|
||||
"negative_text": negative,
|
||||
"category": "",
|
||||
"tags": "",
|
||||
"search_text": "",
|
||||
"prepend_text": "",
|
||||
"append_text": "",
|
||||
"clip": ["9", 1],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def any_switch_graph(**switch_inputs):
|
||||
"""The shape of #381: PromptManagerPair -> Any Switch (rgthree) -> KSampler.
|
||||
|
||||
Classes and links reproduce the maintainer's krea2 run (a LoraManager
|
||||
loader with its own lora ``text``, a project-path filename node, a custom
|
||||
latent and saver); the texts are neutral.
|
||||
"""
|
||||
inputs = switch_inputs or {"any_02": ["10", 0]}
|
||||
return {
|
||||
"3": {"class_type": "SeedHistory", "inputs": {"seed": 1538595059}},
|
||||
"5": {
|
||||
"class_type": "KikoProjectFilePath",
|
||||
"inputs": {"project_name": "demo", "filename": "final"},
|
||||
},
|
||||
"6": {"class_type": "UNETLoader", "inputs": {"unet_name": "model.safetensors"}},
|
||||
"7": {"class_type": "CLIPLoader", "inputs": {"clip_name": "clip.safetensors"}},
|
||||
"8": {"class_type": "VAELoader", "inputs": {"vae_name": "vae.safetensors"}},
|
||||
"9": {
|
||||
"class_type": "Lora Loader (LoraManager)",
|
||||
"inputs": {
|
||||
"text": "<lora:style-demo:1>",
|
||||
"loras": [{"name": "style-demo", "strength": 1, "active": True}],
|
||||
"model": ["6", 0],
|
||||
"clip": ["7", 0],
|
||||
},
|
||||
},
|
||||
"10": pair(),
|
||||
"15": {"class_type": "Any Switch (rgthree)", "inputs": inputs},
|
||||
"16": {
|
||||
"class_type": "EmptyLatentBatch",
|
||||
"inputs": {"width": 1024, "height": 1536, "batch_size": 1},
|
||||
},
|
||||
"17": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"seed": ["3", 0],
|
||||
"steps": 8,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "er_sde",
|
||||
"scheduler": "beta",
|
||||
"denoise": 1.0,
|
||||
"model": ["9", 0],
|
||||
"positive": ["15", 0],
|
||||
"negative": ["10", 1],
|
||||
"latent_image": ["16", 0],
|
||||
},
|
||||
},
|
||||
"18": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["17", 0], "vae": ["8", 0]},
|
||||
},
|
||||
"19": {
|
||||
"class_type": "KikoSaveImage",
|
||||
"inputs": {"filename_prefix": ["5", 0], "images": ["18", 0]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def with_nodes(graph, **nodes):
|
||||
return {**graph, **nodes}
|
||||
|
||||
|
||||
GENERATED = "a generated caption of a lighthouse"
|
||||
|
||||
|
||||
def generated_text_graph():
|
||||
"""PromptManagerPair whose ``text`` is a language model's output.
|
||||
|
||||
The generator (``sampling_mode.seed``, ``prompt``, ``system_prompt``)
|
||||
writes new text at run time: its inputs are not the prompt.
|
||||
"""
|
||||
graph = any_switch_graph()
|
||||
graph["10"]["inputs"]["text"] = ["30", 0]
|
||||
graph["30"] = {
|
||||
"class_type": "VendorTextGenerate",
|
||||
"inputs": {
|
||||
"prompt": ["31", 0],
|
||||
"system_prompt": ["32", 0],
|
||||
"sampling_mode.seed": 0,
|
||||
"clip": ["7", 0],
|
||||
},
|
||||
}
|
||||
graph["31"] = {"class_type": "PrimitiveStringMultiline", "inputs": {"value": OTHER}}
|
||||
graph["32"] = {"class_type": "PrimitiveStringMultiline", "inputs": {"value": "You"}}
|
||||
return graph
|
||||
|
||||
|
||||
def transformed_text_graph():
|
||||
"""PromptManagerPair whose ``text`` passes an unknown string transform."""
|
||||
graph = any_switch_graph()
|
||||
graph["10"]["inputs"]["text"] = ["30", 0]
|
||||
graph["30"] = {
|
||||
"class_type": "VendorRegexReplace",
|
||||
"inputs": {"text": OTHER, "pattern": "rain", "replace": "snow"},
|
||||
}
|
||||
return graph
|
||||
|
||||
|
||||
def execution(node_id, text):
|
||||
"""A ``comfyassets_prompt_pairs`` record of what the pair node ran."""
|
||||
record = {
|
||||
"text": text,
|
||||
"negative_text": "",
|
||||
"positive_text": text,
|
||||
"negative_effective_text": "",
|
||||
"prepend_text": "",
|
||||
"append_text": "",
|
||||
}
|
||||
return json.dumps({"version": 1, "nodes": {node_id: {"records": [record]}}})
|
||||
|
||||
|
||||
def recorded_shapes():
|
||||
"""Record sets whose PromptManager node recorded the text it ran."""
|
||||
return {
|
||||
"generated-recorded": {
|
||||
"prompt": json.dumps(generated_text_graph()),
|
||||
"comfyassets_prompt_pairs": execution("10", GENERATED),
|
||||
},
|
||||
"transformed-recorded": {
|
||||
"prompt": json.dumps(transformed_text_graph()),
|
||||
"comfyassets_prompt_pairs": execution("10", GENERATED),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _sampler_inputs(graph, positive, negative):
|
||||
graph["17"]["inputs"]["positive"] = positive
|
||||
graph["17"]["inputs"]["negative"] = negative
|
||||
return graph
|
||||
|
||||
|
||||
def shapes():
|
||||
"""One named graph per rule (resolved, candidates, PromptManager, refusals)."""
|
||||
vendor_encode = {
|
||||
"class_type": "VendorPromptEncode",
|
||||
"inputs": {"prompt": ["21", 0], "clip": ["22", 1]},
|
||||
}
|
||||
control = {
|
||||
"class_type": "VendorControlApply",
|
||||
"inputs": {
|
||||
"positive": ["21", 0],
|
||||
"negative": ["22", 0],
|
||||
"control_net": ["8", 0],
|
||||
"image": ["8", 0],
|
||||
},
|
||||
}
|
||||
return {
|
||||
"any-switch-pair": any_switch_graph(),
|
||||
"any-switch-same-pair": any_switch_graph(any_01=["10", 0], any_02=["10", 0]),
|
||||
"any-switch-equal-texts": with_nodes(
|
||||
any_switch_graph(any_01=["20", 0], any_02=["21", 0]),
|
||||
**{"20": encode(OTHER), "21": encode(OTHER)},
|
||||
),
|
||||
"any-switch-candidates": with_nodes(
|
||||
any_switch_graph(any_01=["20", 0], any_02=["21", 0]),
|
||||
**{"20": encode(POSITIVE), "21": encode(OTHER)},
|
||||
),
|
||||
"any-switch-prompt-manager-wins": with_nodes(
|
||||
any_switch_graph(any_01=["20", 0], any_02=["10", 0]),
|
||||
**{"20": encode(OTHER)},
|
||||
),
|
||||
"any-switch-runtime-blocks": with_nodes(
|
||||
any_switch_graph(any_01=["20", 0], any_02=["22", 0]),
|
||||
**{
|
||||
"20": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": ["21", 0], "clip": ["7", 0]},
|
||||
},
|
||||
"21": {"class_type": "PromptSearchList", "inputs": {"search": "x"}},
|
||||
"22": encode(OTHER),
|
||||
},
|
||||
),
|
||||
"any-switch-no-text": any_switch_graph(any_01=["7", 0]),
|
||||
"unknown-chain": with_nodes(
|
||||
any_switch_graph(any_01=["20", 0]),
|
||||
**{
|
||||
"20": {
|
||||
"class_type": "Router Node (vendor)",
|
||||
"inputs": {"in": ["10", 0]},
|
||||
}
|
||||
},
|
||||
),
|
||||
"unknown-text-input": with_nodes(
|
||||
any_switch_graph(any_01=["20", 0]),
|
||||
**{
|
||||
"20": {
|
||||
"class_type": "VendorTextEncode",
|
||||
"inputs": {"text": OTHER, "clip": ["9", 1]},
|
||||
}
|
||||
},
|
||||
),
|
||||
"unknown-role-inputs": _sampler_inputs(
|
||||
with_nodes(
|
||||
any_switch_graph(),
|
||||
**{"20": control, "21": encode(POSITIVE), "22": encode(NEGATIVE)},
|
||||
),
|
||||
["20", 0],
|
||||
["20", 1],
|
||||
),
|
||||
"unknown-conditioning-input": with_nodes(
|
||||
any_switch_graph(any_01=["20", 0]),
|
||||
**{
|
||||
"20": {
|
||||
"class_type": "VendorStyleApply",
|
||||
"inputs": {"conditioning": ["10", 0], "style": ["21", 0]},
|
||||
},
|
||||
"21": encode(OTHER),
|
||||
},
|
||||
),
|
||||
"generated-text-runtime": generated_text_graph(),
|
||||
"transformed-text": transformed_text_graph(),
|
||||
"unknown-skips-clip": with_nodes(
|
||||
any_switch_graph(any_01=["20", 0]),
|
||||
**{
|
||||
"20": vendor_encode,
|
||||
"21": {"class_type": "PrimitiveString", "inputs": {"value": OTHER}},
|
||||
"22": {
|
||||
"class_type": "VendorLoraLoader",
|
||||
"inputs": {"text": "<lora:x:1>", "clip": ["7", 0]},
|
||||
},
|
||||
},
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def review_shapes():
|
||||
"""Record sets for the review findings on the first #381 commit."""
|
||||
|
||||
def recorded(source):
|
||||
graph = any_switch_graph()
|
||||
graph["10"]["inputs"]["text"] = ["30", 0]
|
||||
graph.update(source)
|
||||
return {
|
||||
"prompt": json.dumps(graph),
|
||||
"comfyassets_prompt_pairs": execution("10", GENERATED),
|
||||
}
|
||||
|
||||
seeded = with_nodes(
|
||||
any_switch_graph(any_01=["20", 0]),
|
||||
**{
|
||||
"20": {"class_type": "XNoise", "inputs": {"conditioning": ["21", 0]}},
|
||||
"21": {
|
||||
"class_type": "XSeedEncode",
|
||||
"inputs": {"clip": ["7", 0], "text": OTHER, "seed": 3},
|
||||
},
|
||||
},
|
||||
)
|
||||
unlinked = any_switch_graph()
|
||||
unlinked["20"] = {
|
||||
"class_type": "XApply",
|
||||
"inputs": {"positive": ["21", 0], "negative_opt": ""},
|
||||
}
|
||||
unlinked["21"] = encode(POSITIVE)
|
||||
unlinked["17"]["inputs"]["positive"] = ["20", 0]
|
||||
unlinked["17"]["inputs"]["negative"] = ["20", 1]
|
||||
twice = with_nodes(
|
||||
any_switch_graph(any_01=["20", 0], any_02=["23", 0]),
|
||||
**{
|
||||
"20": {"class_type": "XB", "inputs": {"p": ["21", 0], "q": ["22", 0]}},
|
||||
"21": encode(POSITIVE),
|
||||
"22": encode(OTHER),
|
||||
"23": encode(POSITIVE),
|
||||
},
|
||||
)
|
||||
return {
|
||||
"recorded-missing": recorded(
|
||||
{"30": {"class_type": "XLLM", "inputs": {"src": ["99", 0]}}}
|
||||
),
|
||||
"recorded-several": recorded(
|
||||
{
|
||||
"30": {
|
||||
"class_type": "XLLM",
|
||||
"inputs": {"a": ["31", 0], "b": ["32", 0]},
|
||||
},
|
||||
"31": {"class_type": "PrimitiveString", "inputs": {"value": POSITIVE}},
|
||||
"32": {"class_type": "PrimitiveString", "inputs": {"value": OTHER}},
|
||||
}
|
||||
),
|
||||
"recorded-list-text": recorded(
|
||||
{"30": {"class_type": "XLLM", "inputs": {"text": [1, 2, 3]}}}
|
||||
),
|
||||
"seeded-encoder-behind-noise": {"prompt": json.dumps(seeded)},
|
||||
"unlinked-role-input": {"prompt": json.dumps(unlinked)},
|
||||
"one-text-twice": {"prompt": json.dumps(twice)},
|
||||
}
|
||||
Vendored
-21
@@ -28,24 +28,3 @@ for (const name of ['frontend-coverage', 'browser-workflows']) {
|
||||
assert.match(body, /node --version \| grep -q '\^v24\\\.'/, 'the image Node is the Node 24 the project requires');
|
||||
});
|
||||
}
|
||||
|
||||
// The real-host job drives Chromium against a managed ComfyUI. `playwright install --with-deps` pulled the browser's
|
||||
// system packages from Ubuntu's apt mirror on every run, and three runs (#349, #378, #382) crawled at ~26 kB/s until
|
||||
// the job hit its limit after every test had passed. The pinned image ships Chromium and its libraries, so the job runs
|
||||
// there; the image Python must be the reviewed host Python from tools/vendor/comfyui-runtime.json.
|
||||
test('real ComfyUI compatibility runs in the pinned Playwright image on the reviewed host Python', () => {
|
||||
const testWorkflow = readFileSync(new URL('../../.github/workflows/test.yml', import.meta.url), 'utf8');
|
||||
const runtime = JSON.parse(readFileSync(new URL('../../tools/vendor/comfyui-runtime.json', import.meta.url), 'utf8'));
|
||||
const match = testWorkflow.match(/^ {2}comfyui:\n([\s\S]*?)(?=^ {2}[a-z][a-z-]*:\n|(?![\s\S]))/m);
|
||||
assert.ok(match, 'comfyui job exists');
|
||||
const body = match[1];
|
||||
assert.match(body, new RegExp(`^ container:\\n image: mcr\\.microsoft\\.com/playwright:v${version.replaceAll('.', '\\.')}-noble$`, 'm'));
|
||||
assert.doesNotMatch(body, /playwright install/, 'Chromium and its system libraries come from the image, not apt');
|
||||
assert.doesNotMatch(body, /apt-get|actions\/setup-(python|node)@/);
|
||||
assert.match(body, /python3 -m venv --without-pip /, 'a user-space venv on the image Python');
|
||||
assert.match(body, /sha256sum -c/, 'the pip wheel that bootstraps the venv is checksum-pinned');
|
||||
assert.match(body, /node --version \| grep -q '\^v24\\\.'/, 'the image Node is Node 24');
|
||||
// The image Python is checked against the pin at run time, so a pin or image bump that disagrees fails loudly.
|
||||
assert.match(body, /steps\.runtime\.outputs\.pythonVersion/, 'the job compares the image Python with the reviewed pin');
|
||||
assert.match(runtime.pythonVersion, /^3\.12$/, 'the reviewed host Python is the image Python (3.12 on noble)');
|
||||
});
|
||||
|
||||
-116
@@ -1,116 +0,0 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { COLUMNS, CONTROL_EXEMPTIONS, STATUSES, declaredTitles, parseControl, parseCoverageTable, readCoverageTable,
|
||||
rowProblems } from '../../tools/vendor/legacy-coverage.mjs';
|
||||
|
||||
// TESTING.md "Legacy System capability coverage" (#249 System S12, #329): every legacy System capability names an
|
||||
// owner, a known status, a well-formed native control and a test its file really declares.
|
||||
|
||||
const ROW = Object.freeze({ Capability: 'Clear/truncate logs', 'Legacy anchor': '`admin.html:915`', Owner: 'S12 Logs',
|
||||
Status: 'covered', 'Native control': '`diagnostics` · button · "Clear all logs…"', Test: 'clears the logs (clear.spec.mjs)' });
|
||||
const declaring = source => ({ locate: file => `tests/browser/${file}`, read: () => source });
|
||||
|
||||
test('every coverage row has an owner, a known status and a declared test', () => {
|
||||
const rows = readCoverageTable();
|
||||
assert.ok(rows.length >= 54, `the table lists ${rows.length} capabilities`);
|
||||
for (const row of rows) {
|
||||
assert.deepEqual(rowProblems(row), [], `TESTING.md:${row.line} ${row.Capability}`);
|
||||
assert.ok(row.Owner.trim());
|
||||
assert.ok(STATUSES.includes(row.Status));
|
||||
}
|
||||
assert.ok(rows.some(row => row.Status === 'kept in legacy') && rows.some(row => row.Status === 'approved removal'));
|
||||
// A broken row fails: an empty owner, an unknown status, a test its file does not declare.
|
||||
const [first] = rows;
|
||||
assert.deepEqual(rowProblems({ ...first, Owner: '' }), ['no owner']);
|
||||
assert.ok(rowProblems({ ...first, Status: 'done' }).includes('unknown status "done"'));
|
||||
assert.match(rowProblems({ ...first, Test: first.Test.replace(/^(.+) \(/, '$1 again (') }).join(), /declares no test/);
|
||||
assert.match(rowProblems({ ...first, Test: 'some title (missing.spec.mjs)' }).join(), /no test file missing\.spec\.mjs/);
|
||||
});
|
||||
|
||||
test('a title that appears only in a comment does not count', () => {
|
||||
const commented = [
|
||||
"// test('clears the logs', async () => {});",
|
||||
"/* test('clears the logs', async () => {}); */",
|
||||
'/**',
|
||||
" * it('clears the logs')",
|
||||
' */',
|
||||
"const title = 'clears the logs';",
|
||||
"test.describe('clears the logs', () => {});",
|
||||
].join('\n');
|
||||
assert.equal(declaredTitles(commented, 'clear.spec.mjs').has('clears the logs'), false);
|
||||
assert.deepEqual(rowProblems(ROW, declaring(commented)), ['clear.spec.mjs declares no test "clears the logs"']);
|
||||
const declared = `${commented}\nfor (const theme of THEMES) {\n test('clears the logs', async ({ page }) => {});\n}\n`;
|
||||
assert.deepEqual(rowProblems(ROW, declaring(declared)), []);
|
||||
// Escaped quotes, template literals, modifiers and a title on the next line are declarations too.
|
||||
const titles = declaredTitles("test('a gate\\'s refusal', () => {});\ntest.only(`b`, () => {});\nit(\n 'c', () => {});",
|
||||
'x.test.js');
|
||||
assert.deepEqual([...titles], ["a gate's refusal", 'b', 'c']);
|
||||
// A block comment that starts after code on a line, and a `//` or `/*` inside a string or a regular expression.
|
||||
const trailing = [
|
||||
"const url = 'http://host/*'; /* test('clears the logs', () => {});",
|
||||
"test('clears the logs', () => {}); */",
|
||||
"const pattern = /['\"`]\\/\\*/;",
|
||||
"test('after the pattern', () => {});",
|
||||
].join('\n');
|
||||
assert.deepEqual([...declaredTitles(trailing, 'x.spec.mjs')], ['after the pattern']);
|
||||
// Python: only a real `def test_…` counts, never a commented one.
|
||||
const python = ['# def test_report_lists_config_only_keys(self):', 'class T:', ' \"\"\"Example:', '',
|
||||
' def test_in_a_docstring(self):', ' \"\"\"', ' def test_other(self):', ' def test_nested():',
|
||||
' pass', 'def test_module():', ' pass', ''].join('\n');
|
||||
assert.deepEqual([...declaredTitles(python, 'test_x.py')], ['test_other', 'test_module']);
|
||||
});
|
||||
|
||||
test('a malformed native control cell fails', () => {
|
||||
assert.deepEqual(parseControl('`diagnostics` · button · "Clear all logs…"'),
|
||||
{ section: 'diagnostics', role: 'button', name: 'Clear all logs…', row: null });
|
||||
assert.deepEqual(parseControl('`maintenance` · button · "Review plan" · in "Duplicate prompt text"'),
|
||||
{ section: 'maintenance', role: 'button', name: 'Review plan', row: 'Duplicate prompt text' });
|
||||
assert.equal(parseControl('—'), null);
|
||||
for (const [cell, message] of [
|
||||
['diagnostics · button · "Clear all logs…"', /malformed/],
|
||||
['`diagnostics` · button · Clear all logs…', /malformed/],
|
||||
['`diagnostics` - button - "Clear all logs…"', /malformed/],
|
||||
['', /malformed/],
|
||||
['`logs` · button · "Clear all logs…"', /unknown System section "logs"/],
|
||||
['`diagnostics` · link · "Clear all logs…"', /unknown role "link"/],
|
||||
]) {
|
||||
assert.throws(() => parseControl(cell), message, cell);
|
||||
assert.match(rowProblems({ ...ROW, 'Native control': cell }, declaring("test('clears the logs', () => {});")).join(), message);
|
||||
}
|
||||
// Only a covered row names a control.
|
||||
assert.deepEqual(rowProblems({ ...ROW, Status: 'kept in legacy' }, declaring("test('clears the logs', () => {});")),
|
||||
['a kept in legacy row names a native control']);
|
||||
// The table itself must keep its six columns.
|
||||
assert.throws(() => parseCoverageTable(`### Legacy System capability coverage\n\n| Capability | Owner |\n| --- | --- |\n`), /columns/);
|
||||
assert.deepEqual(COLUMNS, ['Capability', 'Legacy anchor', 'Owner', 'Status', 'Native control', 'Test']);
|
||||
});
|
||||
|
||||
test('a skipped test does not prove a capability', () => {
|
||||
for (const modifier of ['skip', 'fixme', 'fail']) {
|
||||
assert.deepEqual(rowProblems(ROW, declaring(`test.${modifier}('clears the logs', async () => {});`)),
|
||||
['clear.spec.mjs only skips "clears the logs"'], modifier);
|
||||
}
|
||||
for (const modifier of ['only', 'slow']) {
|
||||
assert.deepEqual(rowProblems(ROW, declaring(`test.${modifier}('clears the logs', async () => {});`)), [], modifier);
|
||||
}
|
||||
// A running declaration beside a skipped one counts.
|
||||
assert.deepEqual(rowProblems(ROW, declaring("test.skip('clears the logs', () => {});\ntest('clears the logs', () => {});")), []);
|
||||
});
|
||||
|
||||
test('a covered row names its native control unless it is exempt', () => {
|
||||
const source = declaring("test('clears the logs', () => {});");
|
||||
assert.deepEqual(rowProblems({ ...ROW, 'Native control': '—' }, source), ['a covered row names no native control']);
|
||||
for (const status of ['kept in legacy', 'approved removal']) {
|
||||
assert.deepEqual(rowProblems({ ...ROW, Status: status, 'Native control': '—' }, source), [], status);
|
||||
}
|
||||
const [exempt] = Object.keys(CONTROL_EXEMPTIONS);
|
||||
assert.deepEqual(rowProblems({ ...ROW, Capability: exempt, 'Native control': '—' }, source), []);
|
||||
// Every exemption has a reason and names a covered row of the table that has no control.
|
||||
const rows = readCoverageTable();
|
||||
for (const [capability, reason] of Object.entries(CONTROL_EXEMPTIONS)) {
|
||||
assert.ok(reason.length > 20, capability);
|
||||
const row = rows.find(item => item.Capability === capability);
|
||||
assert.ok(row, `${capability} is in the table`);
|
||||
assert.deepEqual([row.Status, row['Native control']], ['covered', '—']);
|
||||
}
|
||||
});
|
||||
+23
-35
@@ -1,9 +1,9 @@
|
||||
{
|
||||
"admin.js": {
|
||||
"lines": 37,
|
||||
"lines": 32,
|
||||
"branches": 61,
|
||||
"functions": 32,
|
||||
"statements": 37
|
||||
"functions": 27,
|
||||
"statements": 32
|
||||
},
|
||||
"api-client.js": {
|
||||
"lines": 98,
|
||||
@@ -48,10 +48,10 @@
|
||||
"statements": 98
|
||||
},
|
||||
"gallery.js": {
|
||||
"lines": 49,
|
||||
"branches": 68,
|
||||
"functions": 50,
|
||||
"statements": 49
|
||||
"lines": 45,
|
||||
"branches": 66,
|
||||
"functions": 48,
|
||||
"statements": 45
|
||||
},
|
||||
"html-escape.js": {
|
||||
"lines": 98,
|
||||
@@ -133,7 +133,7 @@
|
||||
},
|
||||
"sse-stream.js": {
|
||||
"lines": 98,
|
||||
"branches": 92,
|
||||
"branches": 90,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
@@ -181,7 +181,7 @@
|
||||
},
|
||||
"workspace/activity-scan.mjs": {
|
||||
"lines": 96,
|
||||
"branches": 90,
|
||||
"branches": 87,
|
||||
"functions": 98,
|
||||
"statements": 96
|
||||
},
|
||||
@@ -275,12 +275,6 @@
|
||||
"functions": 85,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/images-multiselect.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 98,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/images-selection.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 93,
|
||||
@@ -426,10 +420,10 @@
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/maintenance-review.mjs": {
|
||||
"lines": 95,
|
||||
"branches": 87,
|
||||
"functions": 91,
|
||||
"statements": 95
|
||||
"lines": 94,
|
||||
"branches": 86,
|
||||
"functions": 90,
|
||||
"statements": 94
|
||||
},
|
||||
"workspace/media-viewer.mjs": {
|
||||
"lines": 98,
|
||||
@@ -521,21 +515,15 @@
|
||||
"functions": 89,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/system-diagnostics.mjs": {
|
||||
"lines": 95,
|
||||
"branches": 86,
|
||||
"functions": 94,
|
||||
"statements": 95
|
||||
},
|
||||
"workspace/system-integrations.mjs": {
|
||||
"lines": 97,
|
||||
"branches": 90,
|
||||
"functions": 92,
|
||||
"statements": 97
|
||||
"lines": 96,
|
||||
"branches": 88,
|
||||
"functions": 91,
|
||||
"statements": 96
|
||||
},
|
||||
"workspace/system-maintenance.mjs": {
|
||||
"lines": 93,
|
||||
"branches": 81,
|
||||
"branches": 80,
|
||||
"functions": 92,
|
||||
"statements": 93
|
||||
},
|
||||
@@ -558,10 +546,10 @@
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/system-tagging.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 94,
|
||||
"functions": 95,
|
||||
"statements": 98
|
||||
"lines": 80,
|
||||
"branches": 70,
|
||||
"functions": 75,
|
||||
"statements": 80
|
||||
},
|
||||
"workspace/system-thumbnails.mjs": {
|
||||
"lines": 98,
|
||||
@@ -572,7 +560,7 @@
|
||||
"workspace/system.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 92,
|
||||
"functions": 98,
|
||||
"functions": 86,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/theme.mjs": {
|
||||
|
||||
Vendored
-215
@@ -1,215 +0,0 @@
|
||||
// The legacy System capability coverage table (TESTING.md "Legacy System capability coverage", #249 System S12 #329):
|
||||
// every capability of the legacy admin and gallery pages names its owner, a status, the native control that replaces it
|
||||
// and a declared test. tests/vendor/legacy-coverage.test.mjs checks each row against the test files; the browser spec
|
||||
// `every covered legacy System capability has its native control` finds each covered row's control on its System page.
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
export const ROOT = join(dirname(fileURLToPath(import.meta.url)), '../..');
|
||||
export const HEADING = '### Legacy System capability coverage';
|
||||
export const COLUMNS = Object.freeze(['Capability', 'Legacy anchor', 'Owner', 'Status', 'Native control', 'Test']);
|
||||
export const STATUSES = Object.freeze(['covered', 'kept in legacy', 'approved removal']);
|
||||
export const SECTIONS = Object.freeze(['overview', 'appearance', 'sources', 'integrations', 'tagging', 'maintenance', 'backups',
|
||||
'diagnostics']);
|
||||
export const ROLES = Object.freeze(['button', 'combobox', 'switch', 'slider', 'spinbutton', 'textbox', 'group', 'status', 'term']);
|
||||
// Where a Test cell's file is looked up, in order.
|
||||
export const TEST_FOLDERS = Object.freeze(['tests/browser', 'tests/js', 'tests/vendor', 'tests']);
|
||||
const NONE = '—';
|
||||
/**
|
||||
* Covered rows whose replacement is not on a System section, so they name no native control (`—`), each with why. Every
|
||||
* other covered row must name one.
|
||||
*/
|
||||
export const CONTROL_EXEMPTIONS = Object.freeze({
|
||||
'Gallery per-image auto-tag / review': 'owned by the Images selection bar (Auto-tag selected), not a System section',
|
||||
});
|
||||
|
||||
/** Split a Markdown table row into its cells (a `|` inside backticks is not a separator). */
|
||||
function cells(line) {
|
||||
const out = [];
|
||||
let cell = '';
|
||||
let code = false;
|
||||
for (const char of line.trim().replace(/^\|/, '').replace(/\|$/, '')) {
|
||||
if (char === '`') code = !code;
|
||||
if (char === '|' && !code) {
|
||||
out.push(cell.trim());
|
||||
cell = '';
|
||||
} else {
|
||||
cell += char;
|
||||
}
|
||||
}
|
||||
out.push(cell.trim());
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The table's rows as objects keyed by COLUMNS, from the section that starts at HEADING. */
|
||||
export function parseCoverageTable(markdown) {
|
||||
const lines = markdown.split('\n');
|
||||
const start = lines.findIndex(line => line.trim() === HEADING);
|
||||
if (start === -1) throw new Error(`TESTING.md has no "${HEADING}" section`);
|
||||
const header = lines.findIndex((line, index) => index > start && line.trim().startsWith('|'));
|
||||
if (header === -1 || lines.slice(start + 1, header).some(line => /^#{1,3} /.test(line))) {
|
||||
throw new Error('The coverage section has no table');
|
||||
}
|
||||
const columns = cells(lines[header]);
|
||||
if (columns.join('|') !== COLUMNS.join('|')) throw new Error(`The coverage table's columns are ${columns.join(', ')}`);
|
||||
const rows = [];
|
||||
for (let index = header + 2; index < lines.length && lines[index].trim().startsWith('|'); index += 1) {
|
||||
const values = cells(lines[index]);
|
||||
rows.push({ line: index + 1, ...Object.fromEntries(COLUMNS.map((column, at) => [column, values[at] ?? ''])) });
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* A Native control cell: `` `<section>` · <role> · "<name>" ``, optionally ` · in "<row title>"` when the name is shared
|
||||
* by several rows of the section → {section, role, name, row}; `—` → null; else it throws.
|
||||
*/
|
||||
export function parseControl(cell) {
|
||||
if (cell === NONE) return null;
|
||||
const match = /^`([a-z]+)` · ([a-z]+) · "([^"]+)"(?: · in "([^"]+)")?$/.exec(cell);
|
||||
if (!match) throw new Error(`malformed native control "${cell}"`);
|
||||
const [, section, role, name, row = null] = match;
|
||||
if (!SECTIONS.includes(section)) throw new Error(`unknown System section "${section}"`);
|
||||
if (!ROLES.includes(role)) throw new Error(`unknown role "${role}"`);
|
||||
return { section, role, name, row };
|
||||
}
|
||||
|
||||
/** A Test cell: `<title> (<file>)` → {title, file}. */
|
||||
export function parseTest(cell) {
|
||||
const match = /^(.+) \(([\w.-]+\.(?:mjs|js|py))\)$/.exec(cell);
|
||||
if (!match) throw new Error(`malformed test cell "${cell}"`);
|
||||
return { title: match[1], file: match[2] };
|
||||
}
|
||||
|
||||
// Modifiers that still run the test; `.skip`, `.fixme` and `.fail` do not prove the capability.
|
||||
const RUNS = new Set(['', 'only', 'slow']);
|
||||
// After these characters (or at the start) a `/` opens a regular expression, not a division.
|
||||
const REGEX_BEFORE = /[(,=:[!&|?{};+\-*%<>~^]$/;
|
||||
|
||||
/** Where the string, template literal or regular expression opened by `source[start]` ends (its closing index), or -1. */
|
||||
function literalEnd(source, start) {
|
||||
const quote = source[start];
|
||||
let inClass = false;
|
||||
for (let end = start + 1; end < source.length; end += 1) {
|
||||
const at = source[end];
|
||||
if (at === '\\') end += 1;
|
||||
else if (quote === '/' && at === '[') inClass = true;
|
||||
else if (quote === '/' && at === ']') inClass = false;
|
||||
else if (at === quote && !inClass) return end;
|
||||
// A quoted string or a regular expression never spans lines.
|
||||
else if (at === '\n' && quote !== '`') return -1;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* JavaScript `source` with every comment blanked (newlines kept), wherever it starts on a line. Strings, template
|
||||
* literals and regular expressions are skipped whole, so a `//` or `/*` inside them is not a comment.
|
||||
*/
|
||||
export function stripJsComments(source) {
|
||||
let out = '';
|
||||
let index = 0;
|
||||
while (index < source.length) {
|
||||
const char = source[index];
|
||||
const next = source[index + 1];
|
||||
if (char === '/' && next === '/') {
|
||||
while (index < source.length && source[index] !== '\n') index += 1;
|
||||
continue;
|
||||
}
|
||||
if (char === '/' && next === '*') {
|
||||
const close = source.indexOf('*/', index + 2);
|
||||
const stop = close === -1 ? source.length : close + 2;
|
||||
out += source.slice(index, stop).replace(/[^\n]/g, ' ');
|
||||
index = stop;
|
||||
continue;
|
||||
}
|
||||
const opens = char === "'" || char === '"' || char === '`' || (char === '/' && REGEX_BEFORE.test(out.trimEnd() || '('));
|
||||
const end = opens ? literalEnd(source, index) : -1;
|
||||
if (end === -1) {
|
||||
out += char;
|
||||
index += 1;
|
||||
} else {
|
||||
out += source.slice(index, end + 1);
|
||||
index = end + 1;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Python `source` with its comments and triple-quoted strings (docstrings) blanked, newlines kept. */
|
||||
export function stripPythonComments(source) {
|
||||
return source.replace(/("""|''')[\s\S]*?\1|#[^\n]*/g, text => text.replace(/[^\n]/g, ' '));
|
||||
}
|
||||
|
||||
/**
|
||||
* The tests a source declares, title → whether it runs: a `test('…'` or `it('…'` call at a line's start (`.only` and
|
||||
* `.slow` run; `.skip`, `.fixme` and `.fail` do not) in JavaScript, a `def test_…` at function or method indentation in
|
||||
* Python. Comments, and Python docstrings, never count.
|
||||
*/
|
||||
export function declarations(source, file) {
|
||||
const found = new Map();
|
||||
const add = (title, runs) => found.set(title, found.get(title) || runs);
|
||||
if (file.endsWith('.py')) {
|
||||
for (const match of stripPythonComments(source).matchAll(/^(?: {4})?(?:async[ \t]+)?def[ \t]+(test_\w+)[ \t]*\(/gm)) {
|
||||
add(match[1], true);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
const declaration = /^[ \t]*(?:test|it)(?:\.(only|skip|fixme|fail|slow))?\(\s*(['"`])((?:\\.|(?!\2)[^\\])*)\2/gm;
|
||||
for (const match of stripJsComments(source).matchAll(declaration)) add(match[3].replace(/\\(.)/g, '$1'), RUNS.has(match[1] ?? ''));
|
||||
return found;
|
||||
}
|
||||
|
||||
/** The titles of the tests a source declares and runs. */
|
||||
export function declaredTitles(source, file) {
|
||||
return new Set([...declarations(source, file)].filter(([, runs]) => runs).map(([title]) => title));
|
||||
}
|
||||
|
||||
/** The repository path of a Test cell's file, or null. */
|
||||
export function testFile(file, root = ROOT) {
|
||||
for (const folder of TEST_FOLDERS) {
|
||||
const path = join(root, folder, file);
|
||||
if (existsSync(path)) return path;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Every problem of one row (empty: the row is sound). `locate(file)` finds a test file's path, `read(path)` reads it. */
|
||||
export function rowProblems(row, { root = ROOT, locate = file => testFile(file, root), read = path => readFileSync(path, 'utf8') } = {}) {
|
||||
const problems = [];
|
||||
if (!row.Capability) problems.push('no capability');
|
||||
if (!row['Legacy anchor']) problems.push('no legacy anchor');
|
||||
if (!row.Owner) problems.push('no owner');
|
||||
if (!STATUSES.includes(row.Status)) problems.push(`unknown status "${row.Status}"`);
|
||||
let control = null;
|
||||
try {
|
||||
control = parseControl(row['Native control']);
|
||||
} catch (error) {
|
||||
problems.push(error.message);
|
||||
}
|
||||
if (control && row.Status !== 'covered') problems.push(`a ${row.Status} row names a native control`);
|
||||
if (row.Status === 'covered' && row['Native control'] === NONE && !Object.hasOwn(CONTROL_EXEMPTIONS, row.Capability)) {
|
||||
problems.push('a covered row names no native control');
|
||||
}
|
||||
let test;
|
||||
try {
|
||||
test = parseTest(row.Test);
|
||||
} catch (error) {
|
||||
problems.push(error.message);
|
||||
return problems;
|
||||
}
|
||||
const path = locate(test.file);
|
||||
if (!path) problems.push(`no test file ${test.file}`);
|
||||
else {
|
||||
const runs = declarations(read(path), test.file).get(test.title);
|
||||
if (runs === undefined) problems.push(`${test.file} declares no test "${test.title}"`);
|
||||
else if (!runs) problems.push(`${test.file} only skips "${test.title}"`);
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
/** The table read from TESTING.md. */
|
||||
export function readCoverageTable(root = ROOT) {
|
||||
return parseCoverageTable(readFileSync(join(root, 'TESTING.md'), 'utf8'));
|
||||
}
|
||||
@@ -13,7 +13,7 @@ profile never imports it, so a missing or bad table can never break ingestion.
|
||||
An invalid table raises ``SemanticsError`` so a bad edit fails loudly instead of
|
||||
silently changing what the inspector resolves. The file is bounded in size and
|
||||
nesting before it is validated. Every class appears in exactly one prompt
|
||||
category; an unlisted output of a listed class stays ``unsupported-node``. ``lineage``
|
||||
category; classes that are not listed stay ``unsupported-node``. ``lineage``
|
||||
(image and latent pass-through outputs for multi-stage chains) describes no
|
||||
prompt, so its classes are not claimed by a category.
|
||||
|
||||
@@ -23,13 +23,9 @@ keys, no whitespace): any change of content, never of layout, moves it.
|
||||
folds into the version of the items it records from the inspection profile
|
||||
(``utils.review_capture``): the SHA-256 of ``class_projection``, the table
|
||||
restricted to what it says about ``classes`` (the node classes of one graph)
|
||||
plus the class-independent ``schema``, ``stage_inputs`` and ``pass_through``.
|
||||
The resolver looks the table up only by the class of a graph node, so an edit
|
||||
about other classes cannot change how that graph resolves and leaves the
|
||||
version alone. A class the table does not list at all passes text through
|
||||
(#381); that rule lives in code, so ``INSPECTION_RESOLVER_VERSION`` moves for it.
|
||||
``pass_through.skip_inputs`` names the core type inputs (model, clip, image,
|
||||
...) such a node's text is never read through.
|
||||
plus the class-independent ``schema`` and ``stage_inputs``. The resolver looks
|
||||
the table up only by the class of a graph node, so an edit about other
|
||||
classes cannot change how that graph resolves and leaves the version alone.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
@@ -90,7 +86,7 @@ _CATEGORIES = frozenset(
|
||||
+ _CLASS_CATEGORIES
|
||||
+ _SLOT_CATEGORIES
|
||||
+ _SET_CATEGORIES
|
||||
+ ("lineage", "stage_inputs", "pass_through", "facts")
|
||||
+ ("lineage", "stage_inputs", "facts")
|
||||
)
|
||||
# Input names that make an unlisted node a possible sampler stage in an ancestry
|
||||
# walk (``shaped``: pipes, video embeds and custom-sampler parts carry a prompt
|
||||
@@ -99,10 +95,6 @@ _CATEGORIES = frozenset(
|
||||
# ``model`` alone (ModelSampling*) or with ``clip`` (LoraLoader) is a model patch.
|
||||
# The resolver and comfy-metadata.js both read these lists.
|
||||
_STAGE_INPUT_LISTS = ("shaped", "with_model")
|
||||
# Inputs of core ComfyUI types (model, clip, image, ...) that an unknown node's
|
||||
# output is never read through when the resolver passes text through it (#381):
|
||||
# a loader's lora tag reached through ``clip`` is not the prompt.
|
||||
_PASS_THROUGH_KEYS = ("skip_inputs",)
|
||||
_STRING_PARAMS = MappingProxyType(
|
||||
{
|
||||
"literal": frozenset(),
|
||||
@@ -135,7 +127,6 @@ class Semantics:
|
||||
non_file_outputs: FrozenSet[str]
|
||||
lineage: Mapping[Tuple[str, int], Mapping[str, Any]]
|
||||
stage_inputs: Mapping[str, Tuple[str, ...]]
|
||||
pass_through: Mapping[str, Tuple[str, ...]]
|
||||
facts: Mapping[str, Any]
|
||||
all_classes: FrozenSet[str]
|
||||
# SHA-256 of the canonical table JSON (set by ``load_semantics``).
|
||||
@@ -456,19 +447,6 @@ def _stage_inputs(doc: dict) -> Mapping[str, Tuple[str, ...]]:
|
||||
return _freeze(table)
|
||||
|
||||
|
||||
def _pass_through(doc: dict) -> Mapping[str, Tuple[str, ...]]:
|
||||
table = doc["pass_through"]
|
||||
if not isinstance(table, dict) or set(table) != set(_PASS_THROUGH_KEYS):
|
||||
raise SemanticsError(
|
||||
f"pass_through must have exactly {list(_PASS_THROUGH_KEYS)}"
|
||||
)
|
||||
_names(table["skip_inputs"], "pass_through.skip_inputs")
|
||||
for name in ("text", "positive", "negative", "conditioning"):
|
||||
if name in table["skip_inputs"]:
|
||||
raise SemanticsError(f"pass_through.skip_inputs: {name} carries text")
|
||||
return _freeze(table)
|
||||
|
||||
|
||||
def _facts(doc: dict) -> Mapping[str, Any]:
|
||||
facts = doc["facts"]
|
||||
if not isinstance(facts, dict) or set(facts) != set(_FACTS_TABLES):
|
||||
@@ -528,7 +506,6 @@ def _validate(doc: Any) -> Semantics:
|
||||
return Semantics(
|
||||
lineage=_lineage(doc),
|
||||
stage_inputs=_stage_inputs(doc),
|
||||
pass_through=_pass_through(doc),
|
||||
facts=_facts(doc),
|
||||
all_classes=owners.classes(),
|
||||
**tables,
|
||||
@@ -565,7 +542,7 @@ def load_semantics(path: Optional[Union[str, Path]] = None) -> Semantics:
|
||||
|
||||
|
||||
# Table parts that hold no class names: they apply to every graph.
|
||||
_CLASS_INDEPENDENT = frozenset({"schema", "stage_inputs", "pass_through"})
|
||||
_CLASS_INDEPENDENT = frozenset({"schema", "stage_inputs"})
|
||||
|
||||
|
||||
def _canonical_json(doc: Any) -> str:
|
||||
|
||||
@@ -534,7 +534,6 @@ def inspect_prompt_evidence(
|
||||
saved: dict | None = None,
|
||||
*,
|
||||
profile: ResolverProfile = INSPECTION_PROFILE,
|
||||
file_name: str | None = None,
|
||||
) -> dict:
|
||||
"""Keep uncertainty visible; a saved association never replaces file evidence.
|
||||
|
||||
@@ -598,7 +597,7 @@ def inspect_prompt_evidence(
|
||||
if workflow is not None and "prompt" in canonical:
|
||||
# Only to recognize overwritten PromptManager text (R1); never a role.
|
||||
canonical["workflow"] = workflow
|
||||
resolved = resolve_prompt_pair(canonical, profile=profile, file_name=file_name)
|
||||
resolved = resolve_prompt_pair(canonical, profile=profile)
|
||||
state = resolved["state"]
|
||||
result = {}
|
||||
for role in ("positive", "negative"):
|
||||
|
||||
+21
-470
@@ -6,9 +6,7 @@ contains effective ``text``, deduplicated-recipe ``storage_text`` and
|
||||
``provenance``; ComfyUI provenance records the exact output
|
||||
slot and complete followed link path. States distinguish a resolved pair (an
|
||||
explicit empty negative is valid), absent data, unsupported/unresolved data and
|
||||
ambiguous sampler selection. Listed composites are never flattened; in the
|
||||
inspection profile, nodes the semantics table does not name pass their text
|
||||
through (#381, ``_GraphResolver.passed``).
|
||||
ambiguous sampler selection. Unknown conditioning semantics are never flattened.
|
||||
|
||||
Future nodes can register declarative ``OutputAdapter`` entries by
|
||||
``(class_type, output_slot)``. This does not prescribe a future paired-node schema.
|
||||
@@ -50,9 +48,7 @@ RESOLVER_VERSION = 1
|
||||
# review items recorded from that profile (with the profile's budgets and
|
||||
# the semantics table). Bump it when a change would make the inspection
|
||||
# profile resolve the same metadata otherwise, even if ingestion is unchanged.
|
||||
# 2: unknown nodes pass through generically (#381); unknown classes are not in
|
||||
# the table's class projection, so only this number can move those items.
|
||||
INSPECTION_RESOLVER_VERSION = 2
|
||||
INSPECTION_RESOLVER_VERSION = 1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -117,15 +113,6 @@ _HISTORICAL_ADAPTERS = {
|
||||
for _name in ("PromptManagerPositive", "PromptManagerDetailer"):
|
||||
_HISTORICAL_ADAPTERS[(_name, 0)] = _COMMA
|
||||
_HISTORICAL_ADAPTERS[(_name, 1)] = replace(_COMMA, kind="string")
|
||||
# This repository's own node classes, current and historical: their text is the
|
||||
# prompt whatever unknown nodes sit between them and the sampler (#381).
|
||||
_OWN_CLASSES = frozenset(
|
||||
name
|
||||
for name, _ in list(_DEFAULT_ADAPTERS) + list(_HISTORICAL_ADAPTERS)
|
||||
if name.startswith("PromptManager")
|
||||
)
|
||||
# Nodes with these exact names are traced by code, not by the table.
|
||||
_CODED_CLASSES = frozenset({"Reroute", "StringConcatenate", "Text Concatenate"})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -210,17 +197,6 @@ _ZEROED_SOURCE = "zeroed-conditioning"
|
||||
_MAX_BRANCH_CANDIDATES = 32
|
||||
# Statically undecidable text (a search, a list output): never pick an item.
|
||||
_RUNTIME_REASON = "Text is decided at run time"
|
||||
# Different texts reach one sampler through unknown nodes (#381): one candidate
|
||||
# per text, never a refusal.
|
||||
SEVERAL_SOURCES_REASON = "Several prompts reach the sampler"
|
||||
SELECTOR_BRANCHES_REASON = "Selector branches have different prompts"
|
||||
# Failures that bound the work: never skipped by a pass-through walk.
|
||||
_LIMIT_CODES = frozenset({"work-limit", "depth-limit", "text-limit"})
|
||||
_PASSED_UNVERIFIABLE = "Text passed through an unknown node is not verifiable"
|
||||
# Inputs that make an unknown node a conditioning transform, not a generator.
|
||||
_CONDITIONING_INPUTS = ("conditioning", "positive", "negative")
|
||||
# The first item of a pass-through divergence's 3-tuple choice key.
|
||||
_PASS_THROUGH = "pass-through"
|
||||
# Uncoded outcomes the inspector explains by their exact reason.
|
||||
EMPTY_POSITIVE_REASON = "Positive prompt is empty"
|
||||
# Several sampler branches, every one resolved to the same pair (#249).
|
||||
@@ -265,23 +241,17 @@ class _Unresolved(Exception):
|
||||
|
||||
|
||||
class _Diverging(Exception):
|
||||
"""A runtime choice whose options trace to different texts.
|
||||
"""A runtime selector whose branches trace to different texts.
|
||||
|
||||
Raised from inside tracing; ``_GraphResolver.branches`` resolves the sampler
|
||||
again once per choice, so each candidate is internally consistent. A
|
||||
selector's choices are ``(True, False)``; an unknown node passing several
|
||||
texts through (#381) has one choice per distinct text, ``0..n-1``.
|
||||
again once per choice, so each candidate is internally consistent.
|
||||
"""
|
||||
|
||||
def __init__(self, key: tuple, choices: tuple, sources: bool = False) -> None:
|
||||
def __init__(self, key: Tuple[str, int]) -> None:
|
||||
super().__init__(key)
|
||||
# A selector's condition output (node ID, slot): one runtime value, so
|
||||
# every selector it drives is forced together. A pass-through key is
|
||||
# ``(_PASS_THROUGH, node ID, slot)``.
|
||||
# The condition output (node ID, slot): one runtime value, so every
|
||||
# selector it drives is forced together.
|
||||
self.key = key
|
||||
self.choices = choices
|
||||
# Raised by an unknown node passing several texts (the outcome's reason).
|
||||
self.sources = sources
|
||||
|
||||
|
||||
def _node_ref(node_id: str, class_type: Any) -> dict:
|
||||
@@ -477,8 +447,6 @@ class _GraphResolver:
|
||||
self.latent_charged: set = set()
|
||||
# ``semantics`` (inspection) names the samplers and guiders it follows.
|
||||
self.semantics = semantics
|
||||
# Nesting depth of execution-record checks (``passed``).
|
||||
self.verifying = 0
|
||||
if semantics is None:
|
||||
self.sampler_classes = self.stage_classes = _SAMPLERS
|
||||
else:
|
||||
@@ -489,19 +457,6 @@ class _GraphResolver:
|
||||
)
|
||||
# Any listed sampler upstream in latent ancestry is an earlier stage.
|
||||
self.stage_classes = frozenset(semantics.samplers)
|
||||
# Classes the resolver knows something about: an unlisted output of
|
||||
# one stays ``unsupported-node``; any other class passes through.
|
||||
facts = semantics.facts
|
||||
self.described = (
|
||||
semantics.all_classes
|
||||
| {name for name, _ in semantics.lineage}
|
||||
| {name for name, _ in self.adapters}
|
||||
| _CODED_CLASSES
|
||||
| frozenset(facts["configuration_outputs"])
|
||||
| frozenset(facts["model_outputs"])
|
||||
| frozenset(facts["model_passthroughs"])
|
||||
| frozenset(facts["scalar_outputs"])
|
||||
)
|
||||
|
||||
def node(self, node_id: str) -> Tuple[str, dict]:
|
||||
if self.legacy:
|
||||
@@ -602,9 +557,7 @@ class _GraphResolver:
|
||||
self.inherit_source(node_id, slot, [path])
|
||||
elif (name, slot) in self.adapters:
|
||||
adapter = self.adapters[(name, slot)]
|
||||
if not isinstance(adapter, OutputAdapter) or (
|
||||
kind != "any" and adapter.kind != kind
|
||||
):
|
||||
if not isinstance(adapter, OutputAdapter) or adapter.kind != kind:
|
||||
raise _Unresolved(_INCOMPATIBLE_SLOT, "malformed")
|
||||
evidence = (
|
||||
self.execution_record(node_id, inputs)
|
||||
@@ -660,7 +613,7 @@ class _GraphResolver:
|
||||
elif (
|
||||
name in ("StringConcatenate", "Text Concatenate")
|
||||
and slot == 0
|
||||
and kind in ("string", "any")
|
||||
and kind == "string"
|
||||
):
|
||||
if name == "StringConcatenate":
|
||||
keys = ["string_a", "string_b"]
|
||||
@@ -699,10 +652,6 @@ class _GraphResolver:
|
||||
text, path, storage_text = self.semantic(
|
||||
node_id, name, slot, kind, inputs, depth
|
||||
)
|
||||
elif self.semantics is not None and name not in self.described:
|
||||
text, path, storage_text = self.passed(
|
||||
node_id, slot, kind, inputs, depth
|
||||
)
|
||||
else:
|
||||
raise self.unsupported(
|
||||
"Unsupported node semantics or output slot", node_id
|
||||
@@ -798,7 +747,7 @@ class _GraphResolver:
|
||||
return True
|
||||
if slot == 0 and name in semantics.selectors:
|
||||
return True
|
||||
return kind in ("conditioning", "any") and (
|
||||
return kind == "conditioning" and (
|
||||
(name, slot) in semantics.conditioning
|
||||
or (slot == 0 and name in semantics.composites)
|
||||
)
|
||||
@@ -820,13 +769,11 @@ class _GraphResolver:
|
||||
):
|
||||
listed = table.get((name, slot))
|
||||
if listed is not None:
|
||||
if kind not in (expected, "any"):
|
||||
if kind != expected:
|
||||
raise _Unresolved(_INCOMPATIBLE_SLOT, "malformed")
|
||||
return build(node_id, listed, inputs, depth)
|
||||
entry = (
|
||||
semantics.conditioning.get((name, slot))
|
||||
if kind in ("conditioning", "any")
|
||||
else None
|
||||
semantics.conditioning.get((name, slot)) if kind == "conditioning" else None
|
||||
)
|
||||
if entry is not None and entry["strategy"] == "zeroed":
|
||||
# The upstream text is discarded, so it is not traced.
|
||||
@@ -878,120 +825,9 @@ class _GraphResolver:
|
||||
for label in ("true", "false")
|
||||
]
|
||||
if parts[0][0] != parts[1][0]:
|
||||
raise _Diverging((str(condition[0]), condition[1]), (True, False))
|
||||
raise _Diverging((str(condition[0]), condition[1]))
|
||||
return self.merged(node_id, 0, parts)
|
||||
|
||||
def passed(
|
||||
self, node_id: str, slot: int, kind: str, inputs: dict, depth: int
|
||||
) -> Tuple[str, list, str]:
|
||||
"""``read_through``, except while checking an execution record.
|
||||
|
||||
A pair node's recorded text is what ran. While its inputs are checked
|
||||
against the record (``execution_record``), any outcome of a walk through
|
||||
an unknown node but running out of visits only means "cannot be read
|
||||
statically" (``unsupported-node``, unverifiable), as before #381.
|
||||
"""
|
||||
if not self.verifying:
|
||||
return self.read_through(node_id, slot, kind, inputs, depth)
|
||||
try:
|
||||
return self.read_through(node_id, slot, kind, inputs, depth)
|
||||
except _Diverging:
|
||||
raise self.unsupported(_PASSED_UNVERIFIABLE, node_id) from None
|
||||
except _Unresolved as error:
|
||||
if error.code == "work-limit":
|
||||
raise
|
||||
raise self.unsupported(_PASSED_UNVERIFIABLE, node_id) from None
|
||||
|
||||
def read_through(
|
||||
self, node_id: str, slot: int, kind: str, inputs: dict, depth: int
|
||||
) -> Tuple[str, list, str]:
|
||||
"""An output no rule describes passes its text through (#381).
|
||||
|
||||
A string output (kind ``string``) of a node with a seed input
|
||||
(``seed``, ``noise_seed``, ``sampling_mode.seed``) and no linked
|
||||
conditioning input is generated at run time (a language model, a
|
||||
wildcard or random line): ``runtime-value``, never its inputs. With a
|
||||
linked conditioning input it is a conditioning transform (noise). A
|
||||
``text`` input is the text (the legacy encoder rule). Otherwise the
|
||||
followed inputs (``pass_inputs``) are traced with kind ``any``: an
|
||||
unknown node may turn a string into conditioning. A followed input that
|
||||
finds no text (``unsupported-node``) adds nothing; limits and runtime
|
||||
choices propagate; any other failure blocks, unless a PromptManager
|
||||
node was reached: this repository's own nodes are authoritative, so
|
||||
only their texts count then. One distinct text is the result; several
|
||||
are a runtime choice with one candidate per text, in input order.
|
||||
"""
|
||||
if (
|
||||
kind == "string"
|
||||
and any(name.endswith("seed") for name in inputs)
|
||||
and not any(_link(inputs.get(name)) for name in _CONDITIONING_INPUTS)
|
||||
):
|
||||
raise self.named(_RUNTIME_REASON, "runtime-value", node_id)
|
||||
if "text" in inputs:
|
||||
traced = self.text(inputs["text"], "string", depth + 1)
|
||||
self.inherit_source(node_id, slot, [traced[1]])
|
||||
return traced
|
||||
found: list = []
|
||||
blocking: Optional[_Unresolved] = None
|
||||
for value in self.pass_inputs(slot, inputs):
|
||||
try:
|
||||
found.append(self.text(value, "any", depth + 1))
|
||||
except _Unresolved as error:
|
||||
if error.code in _LIMIT_CODES:
|
||||
raise
|
||||
if error.code != "unsupported-node" and blocking is None:
|
||||
blocking = error
|
||||
owned = [traced for traced in found if self.owned(traced[1])]
|
||||
if owned:
|
||||
found = owned
|
||||
elif blocking is not None:
|
||||
raise blocking
|
||||
if not found:
|
||||
raise self.unsupported("Unsupported node semantics or output slot", node_id)
|
||||
texts = list(dict.fromkeys(traced[0] for traced in found))
|
||||
if len(texts) == 1:
|
||||
return self.merged(node_id, slot, found)
|
||||
if len(texts) > _MAX_BRANCH_CANDIDATES:
|
||||
raise _Unresolved("Too many selector branches", "work-limit")
|
||||
key = (_PASS_THROUGH, node_id, slot)
|
||||
if key not in self.choices:
|
||||
raise _Diverging(key, tuple(range(len(texts))), sources=True)
|
||||
self.tainted = True
|
||||
chosen = texts[self.choices[key]]
|
||||
return self.merged(node_id, slot, [t for t in found if t[0] == chosen])
|
||||
|
||||
def pass_inputs(self, slot: int, inputs: dict) -> list:
|
||||
"""The linked inputs an unknown node's output is read through.
|
||||
|
||||
Output slot 0 of a node with a ``positive`` or ``negative`` input reads
|
||||
only ``positive`` (slot 1, ``negative``), nothing when that one is not
|
||||
linked: the paired-conditioning convention, which the legacy parser
|
||||
followed by role. A linked ``conditioning`` is read alone (the
|
||||
legacy order). Otherwise every linked input but the core type inputs
|
||||
(``pass_through.skip_inputs``: model, clip, image, ...) is read.
|
||||
"""
|
||||
if slot in (0, 1) and ("positive" in inputs or "negative" in inputs):
|
||||
value = inputs.get(("positive", "negative")[slot])
|
||||
return [value] if _link(value) else []
|
||||
if _link(inputs.get("conditioning")):
|
||||
return [inputs["conditioning"]]
|
||||
skipped = self.semantics.pass_through["skip_inputs"]
|
||||
return [
|
||||
value
|
||||
for name, value in inputs.items()
|
||||
if _link(value) and name not in skipped
|
||||
]
|
||||
|
||||
def through_unknown(self, path: list) -> bool:
|
||||
"""Whether a followed path passes through a node no rule describes."""
|
||||
self.scan(len(path))
|
||||
return any(self.node(step["node_id"])[0] not in self.described for step in path)
|
||||
|
||||
def owned(self, path: list) -> bool:
|
||||
"""Whether a followed path passes through one of this repository's nodes."""
|
||||
self.scan(len(path))
|
||||
return any(self.node(step["node_id"])[0] in _OWN_CLASSES for step in path)
|
||||
|
||||
def merged(self, node_id: str, slot: int, parts: list) -> Tuple[str, list, str]:
|
||||
"""One text reached through several followed paths (deduplicated)."""
|
||||
text, merged, storage_text = self.joined_paths(parts, parts)
|
||||
@@ -1185,9 +1021,8 @@ class _GraphResolver:
|
||||
if record is None:
|
||||
return None
|
||||
for name in ("text", "negative_text", "prepend_text", "append_text"):
|
||||
self.verifying += 1
|
||||
try:
|
||||
expected, followed, _ = self.text(inputs.get(name, ""), "string")
|
||||
expected = self.text(inputs.get(name, ""), "string")[0]
|
||||
except _Unresolved as error:
|
||||
if self.unverifiable(error):
|
||||
value = inputs.get(name)
|
||||
@@ -1208,11 +1043,6 @@ class _GraphResolver:
|
||||
raise
|
||||
continue
|
||||
raise
|
||||
finally:
|
||||
self.verifying -= 1
|
||||
if self.semantics is not None and self.through_unknown(followed):
|
||||
# A text passed through an unknown node may be transformed there.
|
||||
continue
|
||||
if name in ("prepend_text", "append_text"):
|
||||
expected = expected.strip()
|
||||
if expected != record[name]:
|
||||
@@ -1532,121 +1362,6 @@ class _GraphResolver:
|
||||
_link(inputs.get(key)) for key in stage_inputs["with_model"]
|
||||
)
|
||||
|
||||
def same_conditioning(self, first: str, second: str) -> bool:
|
||||
"""Whether two listed samplers read the very same conditioning links.
|
||||
|
||||
Compared by the sampler table's conditioning inputs (positive and
|
||||
negative, or a custom sampler's guider): each a link, equal by node ID
|
||||
text and slot. Then their prompt is one, whatever their stage order.
|
||||
"""
|
||||
keys = []
|
||||
for node_id in (first, second):
|
||||
name, inputs = self.node(node_id)
|
||||
conditioning = self.semantics.samplers[name]["conditioning"]
|
||||
values = []
|
||||
for role in sorted(conditioning):
|
||||
value = inputs.get(conditioning[role])
|
||||
if not _link(value):
|
||||
return False
|
||||
values.append((role, str(value[0]), str(value[1])))
|
||||
keys.append(tuple(values))
|
||||
return keys[0] == keys[1]
|
||||
|
||||
def alone(self, node_id: str) -> bool:
|
||||
"""Whether a sampler's prompt is its own links' text alone.
|
||||
|
||||
True when it sets no prompt override and its latent ancestry reaches no
|
||||
listed sampler (a plain first stage). Any failure but running out of
|
||||
visits is ``False``; the walk never recurses into another stage.
|
||||
"""
|
||||
name, inputs = self.node(node_id)
|
||||
entry = self.semantics.samplers[name]
|
||||
if entry["strategy"] == _DETAILER:
|
||||
return False
|
||||
if any(
|
||||
inputs.get(override) not in (None, "")
|
||||
for override in entry.get("prompt_overrides", ())
|
||||
):
|
||||
return False
|
||||
value = inputs.get(entry.get("latent")) if entry.get("latent") else None
|
||||
if value is None:
|
||||
return True
|
||||
if not _link(value):
|
||||
return False
|
||||
queue, queued = self.walk_from(value)
|
||||
try:
|
||||
while queue:
|
||||
current, depth = queue.popleft()
|
||||
if current == node_id or depth >= self.max_depth:
|
||||
return False
|
||||
self.budget.spend()
|
||||
name, inputs = self.node(current)
|
||||
if name in self.stage_classes or self.stage_shaped(name, inputs):
|
||||
return False
|
||||
self.enqueue(queue, queued, inputs, depth + 1)
|
||||
except _Unresolved as error:
|
||||
if error.code == "work-limit":
|
||||
raise
|
||||
return False
|
||||
return True
|
||||
|
||||
def saver_stages(self, file_name: Any) -> Optional[list]:
|
||||
"""The candidate samplers of the one saver that wrote ``file_name``.
|
||||
|
||||
A saver is any node with a ``filename_prefix`` (literal, or traced as a
|
||||
string) whose last path part makes the file's stem
|
||||
``<prefix>_<digits>`` with an optional trailing ``_`` (ComfyUI's and
|
||||
VideoHelperSuite's counters). Exactly one saver must match; its
|
||||
linked inputs are walked like a detailer's image ancestry. No match,
|
||||
several, an unreadable walk or no candidate sampler is ``None``: the
|
||||
file is then judged unanchored, never guessed (#381).
|
||||
"""
|
||||
stem = _file_stem(file_name)
|
||||
if stem is None:
|
||||
return None
|
||||
self.budget.reset(self.per_candidate_visits)
|
||||
try:
|
||||
matches = [
|
||||
node_id
|
||||
for node_id, node in self.graph.items()
|
||||
if self.writes(node, stem)
|
||||
]
|
||||
if len(matches) != 1:
|
||||
return None
|
||||
_, inputs = self.node(matches[0])
|
||||
values = [
|
||||
value
|
||||
for name, value in inputs.items()
|
||||
if name != "filename_prefix" and _link(value)
|
||||
]
|
||||
found = self.stages_from(matches[0], values)
|
||||
except (_Unresolved, _Diverging):
|
||||
return None
|
||||
candidates = [
|
||||
node_id
|
||||
for node_id in found
|
||||
if self.graph[node_id]["class_type"] in self.sampler_classes
|
||||
]
|
||||
return candidates or None
|
||||
|
||||
def writes(self, node: Any, stem: str) -> bool:
|
||||
"""Whether a node's ``filename_prefix`` names the file ``stem``."""
|
||||
if not isinstance(node, dict) or not isinstance(node.get("inputs"), dict):
|
||||
return False
|
||||
prefix = node["inputs"].get("filename_prefix")
|
||||
if _link(prefix):
|
||||
try:
|
||||
prefix = self.text(prefix, "string")[0]
|
||||
except _Diverging:
|
||||
return False
|
||||
except _Unresolved as error:
|
||||
if error.code == "work-limit":
|
||||
raise # an unfinished scan anchors nothing (never a guess)
|
||||
return False
|
||||
if not isinstance(prefix, str):
|
||||
return False
|
||||
return _counter_of(prefix.replace("\\", "/").split("/")[-1], stem)
|
||||
|
||||
def upstream_stages(self, origin: str, value: Any) -> list:
|
||||
"""Samplers reached from a detailer's image input, in breadth-first order.
|
||||
|
||||
@@ -1655,18 +1370,8 @@ class _GraphResolver:
|
||||
"""
|
||||
if not _link(value):
|
||||
raise _Unresolved("Expected an image link", "malformed")
|
||||
return self.stages_from(origin, [value])
|
||||
|
||||
def stages_from(self, origin: str, values: list) -> list:
|
||||
"""``upstream_stages`` from several start links, queued in order once."""
|
||||
found: list = []
|
||||
queue: deque = deque()
|
||||
queued: set = set()
|
||||
for value in values:
|
||||
start = str(value[0])
|
||||
if start not in queued:
|
||||
queued.add(start)
|
||||
queue.append((start, 0))
|
||||
queue, queued = self.walk_from(value)
|
||||
while queue:
|
||||
node_id, depth = queue.popleft()
|
||||
if node_id == origin:
|
||||
@@ -1858,20 +1563,17 @@ class _GraphResolver:
|
||||
"""
|
||||
pending: list = [{}]
|
||||
results: list = []
|
||||
several = False
|
||||
try:
|
||||
while pending:
|
||||
self.choices = pending.pop(0)
|
||||
try:
|
||||
results.append(self.stage(sampler_id, frozenset()))
|
||||
except _Diverging as diverging:
|
||||
count = len(diverging.choices)
|
||||
if len(results) + len(pending) + count > _MAX_BRANCH_CANDIDATES:
|
||||
if len(results) + len(pending) + 2 > _MAX_BRANCH_CANDIDATES:
|
||||
raise _Unresolved("Too many selector branches", "work-limit")
|
||||
several = several or diverging.sources
|
||||
pending.extend(
|
||||
{**self.choices, diverging.key: choice}
|
||||
for choice in diverging.choices
|
||||
for choice in (True, False)
|
||||
)
|
||||
except _Unresolved as error:
|
||||
return _result(
|
||||
@@ -1885,18 +1587,12 @@ class _GraphResolver:
|
||||
self.choices = {}
|
||||
if all(item["state"] == "unresolved" for item in results):
|
||||
return results[0]
|
||||
if several:
|
||||
# Chained unknown nodes can offer one text twice: list it once.
|
||||
unique: dict = {}
|
||||
for item in results:
|
||||
unique.setdefault(_outcome_key(item), item)
|
||||
results = list(unique.values())
|
||||
if len({_outcome_key(item) for item in results}) == 1:
|
||||
return results[0]
|
||||
return _result(
|
||||
"ambiguous",
|
||||
sampler_id=sampler_id,
|
||||
reason=SEVERAL_SOURCES_REASON if several else SELECTOR_BRANCHES_REASON,
|
||||
reason="Selector branches have different prompts",
|
||||
candidates=results,
|
||||
)
|
||||
|
||||
@@ -2054,10 +1750,6 @@ class _GraphResolver:
|
||||
if not traced:
|
||||
lineage, traced = self.lineage_predecessor(value), True
|
||||
if node_id != lineage:
|
||||
if self.same_conditioning(origin, node_id) and self.alone(node_id):
|
||||
# Same positive and negative links: the same prompt
|
||||
# whichever stage this is (#381).
|
||||
continue
|
||||
# Not the one continued stage: no stage's prompt is chosen.
|
||||
raise _Unresolved(MULTISTAGE_UNATTRIBUTED_REASON)
|
||||
earlier = node_id
|
||||
@@ -2191,26 +1883,6 @@ def _parameters(value: Any, max_text_length: int, inspection: bool = False) -> d
|
||||
return _result("resolved", positive_side, negative_side)
|
||||
|
||||
|
||||
def _file_stem(file_name: Any) -> Optional[str]:
|
||||
"""A file name's last path part without its extension, or ``None``."""
|
||||
if not isinstance(file_name, str) or not file_name:
|
||||
return None
|
||||
base = file_name.replace("\\", "/").split("/")[-1]
|
||||
dot = base.rfind(".")
|
||||
stem = base[:dot] if dot > 0 else base
|
||||
return stem or None
|
||||
|
||||
|
||||
def _counter_of(prefix: str, stem: str) -> bool:
|
||||
"""Whether ``stem`` is ``prefix`` + ``_`` + ASCII digits [+ ``_``]."""
|
||||
if not prefix or not stem.startswith(prefix + "_"):
|
||||
return False
|
||||
rest = stem[len(prefix) + 1 :]
|
||||
if rest.endswith("_"):
|
||||
rest = rest[:-1]
|
||||
return rest != "" and all("0" <= char <= "9" for char in rest)
|
||||
|
||||
|
||||
def _profile_with(
|
||||
profile: ResolverProfile,
|
||||
max_visits: Optional[int],
|
||||
@@ -2255,7 +1927,6 @@ def resolve_prompt_pair(
|
||||
max_visits: Optional[int] = None,
|
||||
max_depth: Optional[int] = None,
|
||||
max_text_length: Optional[int] = None,
|
||||
file_name: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Resolve parsed metadata; a present API graph is authoritative over text.
|
||||
|
||||
@@ -2266,11 +1937,9 @@ def resolve_prompt_pair(
|
||||
``sampler_id`` (the inspection profile first drops stages that only reach
|
||||
previews).
|
||||
``profile`` selects the bounds; explicit ``max_*`` keywords override it.
|
||||
``file_name`` (inspection only, unanchored) anchors on the one saver that
|
||||
wrote that file (``_GraphResolver.saver_stages``) when exactly one matches.
|
||||
"""
|
||||
profile = _profile_with(profile, max_visits, max_depth, max_text_length)
|
||||
result = _resolve(metadata, output_node_id, adapters, profile, file_name)
|
||||
result = _resolve(metadata, output_node_id, adapters, profile)
|
||||
return _legacy_result(result) if profile.legacy else result
|
||||
|
||||
|
||||
@@ -2323,7 +1992,6 @@ def _resolve(
|
||||
output_node_id: Optional[str],
|
||||
adapters: Optional[Mapping[Tuple[str, int], OutputAdapter]],
|
||||
profile: ResolverProfile,
|
||||
file_name: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
if not isinstance(metadata, dict) or not metadata:
|
||||
return _result("absent", reason="No parsed prompt metadata")
|
||||
@@ -2379,19 +2047,14 @@ def _resolve(
|
||||
"unresolved", reason=str(error), reason_code=error.code, node=error.node
|
||||
)
|
||||
else:
|
||||
anchored = (
|
||||
resolver.saver_stages(file_name)
|
||||
if semantics is not None and file_name is not None
|
||||
else None
|
||||
)
|
||||
sampler_ids = anchored or [
|
||||
sampler_ids = [
|
||||
node_id
|
||||
for node_id, node in normalized.items()
|
||||
if isinstance(node, dict)
|
||||
and isinstance(node.get("class_type"), str)
|
||||
and node["class_type"] in resolver.sampler_classes
|
||||
]
|
||||
if semantics is not None and anchored is None:
|
||||
if semantics is not None:
|
||||
sampler_ids = resolver.final_stages(sampler_ids)
|
||||
if not sampler_ids:
|
||||
return _no_sampler(normalized, profile)
|
||||
@@ -2578,115 +2241,3 @@ def resolve_text_input(
|
||||
return resolver.text(inputs.get(input_name), "string")[0]
|
||||
except _Unresolved:
|
||||
return None
|
||||
|
||||
|
||||
_NUMBER = re.compile(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?")
|
||||
|
||||
|
||||
def _numeric(text: str) -> bool:
|
||||
"""Whether ``text`` is a plain number (a widget value such as ``2048``)."""
|
||||
return _NUMBER.fullmatch(text) is not None
|
||||
|
||||
|
||||
def sampler_less_prompt(graph: Any, *, max_visits: int = 500) -> Dict[str, Any]:
|
||||
"""The positive prompt of an API graph with no sampler pair, if one (#252).
|
||||
|
||||
For graphs ``_no_sampler`` finds ``absent`` (no node takes a ``positive``
|
||||
or ``negative`` input, such as a Flux guider or a video wrapper). Only the
|
||||
executed API graph is read, never the UI workflow. A candidate is a node
|
||||
with a ``text`` input whose output another node consumes: a text that
|
||||
feeds nothing did not make the image. Its text is traced as a STRING
|
||||
(``resolve_text_input``); a linked text no rule reads is unknown unless
|
||||
it is built from a candidate's text (``_covered``). Blank and
|
||||
numeric texts are never prompts, and a node with a seed input holds a
|
||||
template, not the prompt (unknown). This repository's own positive-text
|
||||
nodes are authoritative: when they give a text, only they count.
|
||||
|
||||
Returns ``{"state", "text", "texts"}``: ``resolved`` with the one distinct
|
||||
text; ``ambiguous`` when several texts or an unknown one could be the
|
||||
prompt (``texts`` lists the readable ones); ``absent`` when none.
|
||||
"""
|
||||
absent = {"state": "absent", "text": None, "texts": []}
|
||||
if not isinstance(graph, dict) or not graph or len(graph) > max_visits:
|
||||
return absent
|
||||
nodes = {str(key): value for key, value in graph.items()}
|
||||
consumed = {
|
||||
str(value[0])
|
||||
for node in nodes.values()
|
||||
if isinstance(node, dict) and isinstance(node.get("inputs"), dict)
|
||||
for value in node["inputs"].values()
|
||||
if _link(value)
|
||||
}
|
||||
own: list = []
|
||||
other: list = []
|
||||
unknown = False
|
||||
for node_id, node in nodes.items():
|
||||
inputs = node.get("inputs") if isinstance(node, dict) else None
|
||||
if node_id not in consumed or not _has_text_input(node):
|
||||
continue
|
||||
if _seeded(inputs):
|
||||
# Generated at run time (a wildcard or a language model): its
|
||||
# text input is a template, not the prompt that ran.
|
||||
unknown = True
|
||||
continue
|
||||
value = inputs["text"]
|
||||
text = resolve_text_input(nodes, node_id, "text", max_visits=max_visits)
|
||||
if text is None:
|
||||
# A literal that is not a string (a number) is never a prompt.
|
||||
if _link(value) and not _covered(nodes, value):
|
||||
unknown = True
|
||||
continue
|
||||
text = text.strip()
|
||||
if not text or _numeric(text):
|
||||
continue
|
||||
kind = node.get("class_type")
|
||||
if isinstance(kind, str) and kind in _OWN_CLASSES and "Negative" not in kind:
|
||||
own.append(text)
|
||||
else:
|
||||
other.append(text)
|
||||
texts = list(dict.fromkeys(own or other))
|
||||
if len(texts) == 1 and (own or not unknown):
|
||||
return {"state": "resolved", "text": texts[0], "texts": texts}
|
||||
if texts or unknown:
|
||||
return {"state": "ambiguous", "text": None, "texts": texts}
|
||||
return absent
|
||||
|
||||
|
||||
def _has_text_input(node: Any) -> bool:
|
||||
inputs = node.get("inputs") if isinstance(node, dict) else None
|
||||
return isinstance(inputs, dict) and "text" in inputs
|
||||
|
||||
|
||||
def _seeded(inputs: dict) -> bool:
|
||||
"""A seed input: the node's string output is a run-time value (the
|
||||
resolver's ``runtime-value`` rule)."""
|
||||
return any(isinstance(name, str) and name.endswith("seed") for name in inputs)
|
||||
|
||||
|
||||
def _covered(nodes: dict, link: list) -> bool:
|
||||
"""Whether an unreadable text link is built from a candidate's text.
|
||||
|
||||
True when a node upstream of ``link`` is read through its first output
|
||||
(slot 0, the legacy encoder rule's text) and has an unseeded ``text``
|
||||
input: that candidate is read on its own, and whatever else is joined to
|
||||
it at run time (a caption, a language model) is no stored prompt to
|
||||
choose. Another output of such a node (a loader's trigger words) is not
|
||||
its text. False when the text comes only from inputs no candidate holds
|
||||
(a builder's ``prompt`` field): the prompt is unknown then.
|
||||
"""
|
||||
seen, stack = set(), [(str(link[0]), link[1])]
|
||||
while stack:
|
||||
node_id, slot = stack.pop()
|
||||
if (node_id, slot) in seen:
|
||||
continue
|
||||
seen.add((node_id, slot))
|
||||
node = nodes.get(node_id)
|
||||
inputs = node.get("inputs") if isinstance(node, dict) else None
|
||||
if not isinstance(inputs, dict):
|
||||
continue
|
||||
if "text" in inputs and slot == 0 and not _seeded(inputs):
|
||||
return True
|
||||
stack.extend(
|
||||
(str(value[0]), value[1]) for value in inputs.values() if _link(value)
|
||||
)
|
||||
return False
|
||||
|
||||
+4
-4
@@ -1187,13 +1187,14 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr
|
||||
<div class="space-y-1.5">
|
||||
<label class="flex items-center p-2.5 bg-pm-input rounded-pm-md cursor-pointer hover:bg-pm-hover border border-pm">
|
||||
<input type="radio" name="autoTagMode" value="skip" checked class="w-3.5 h-3.5 accent-[var(--pm-accent)]">
|
||||
<span class="ml-2.5 text-pm text-xs">Skip images already auto-tagged</span>
|
||||
<span class="ml-2.5 text-pm text-xs">Skip images with existing tags</span>
|
||||
</label>
|
||||
<label class="flex items-center p-2.5 bg-pm-input rounded-pm-md cursor-pointer hover:bg-pm-hover border border-pm">
|
||||
<input type="radio" name="autoTagMode" value="retag" class="w-3.5 h-3.5 accent-[var(--pm-accent)]">
|
||||
<span class="ml-2.5 text-pm text-xs">Re-tag all images</span>
|
||||
</label>
|
||||
</div>
|
||||
<p class="text-pm-muted text-[11px] mt-1">Note: "auto-scanned" is not counted as a real tag</p>
|
||||
</div>
|
||||
|
||||
<!-- Action Buttons -->
|
||||
@@ -1292,7 +1293,6 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr
|
||||
<div class="w-full bg-pm-input rounded-full h-1.5">
|
||||
<div id="autoTagProgressBar" class="bg-pm-accent h-1.5 rounded-full transition-all duration-300" style="width: 0%"></div>
|
||||
</div>
|
||||
<p id="autoTagCloseNote" class="text-pm-muted text-[11px] mt-1.5">Closing this window stops tagging. Tags already added are kept. Prompt Manager's Activity page shows the time left.</p>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-3 gap-3 mb-4">
|
||||
@@ -1332,7 +1332,7 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr
|
||||
<!-- Re-tag Confirmation Modal -->
|
||||
<div id="retagConfirmModal" class="fixed inset-0 bg-black/60 backdrop-blur-[2px] hidden items-center justify-center z-50">
|
||||
<div class="bg-pm-surface rounded-pm-lg p-4 max-w-lg w-full mx-4 border border-pm shadow-pm">
|
||||
<h3 class="text-sm font-semibold text-pm mb-3">Image Already Auto-tagged</h3>
|
||||
<h3 class="text-sm font-semibold text-pm mb-3">Image Already Tagged</h3>
|
||||
|
||||
<div class="mb-3">
|
||||
<img id="retagPreviewImage" src="" alt="Preview" class="w-full h-40 object-contain bg-pm-primary rounded-pm-md">
|
||||
@@ -1345,7 +1345,7 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p class="text-pm-secondary text-xs mb-4">A tagging model already tagged this image. Generate new tags?</p>
|
||||
<p class="text-pm-secondary text-xs mb-4">This image already has tags. Would you like to generate new tags?</p>
|
||||
|
||||
<div class="flex gap-2 justify-end">
|
||||
<button id="retagSkipBtn"
|
||||
|
||||
@@ -682,7 +682,6 @@ No sentences, no commentary, no captions. Only tags. Keep length short but descr
|
||||
<div class="w-full bg-pm-input rounded-full h-2">
|
||||
<div id="autoTagProgressBar" class="bg-pm-accent h-2 rounded-full transition-all duration-300" style="width: 0%"></div>
|
||||
</div>
|
||||
<p id="autoTagCloseNote" class="text-pm-muted text-xs mt-1.5">Closing this window stops tagging. Tags already added are kept. Prompt Manager's Activity page shows the time left.</p>
|
||||
</div>
|
||||
|
||||
<!-- Stats -->
|
||||
|
||||
+65
-52
@@ -3801,7 +3801,7 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
} else if (data.type === 'complete') {
|
||||
this.autoTagState.eventSource.close();
|
||||
this.hideModal("autoTagProgressModal");
|
||||
this.showNotification(ImageHelpers.autotagCompleteMessage(data.tagged), 'success');
|
||||
this.showNotification(`Auto tagging complete! Applied tags to ${data.tagged || 0} prompts.`, 'success');
|
||||
this.refreshList(); // Refresh the prompt list
|
||||
} else if (data.type === 'error') {
|
||||
this.autoTagState.eventSource.close();
|
||||
@@ -3819,7 +3819,7 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
console.error('EventSource readyState:', this.autoTagState.eventSource.readyState);
|
||||
this.autoTagState.eventSource.close();
|
||||
this.hideModal("autoTagProgressModal");
|
||||
this.showNotification((error && error.serverMessage) || 'Connection lost during auto tagging', 'error');
|
||||
this.showNotification('Connection lost during auto tagging', 'error');
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Auto tag error:', error);
|
||||
@@ -3894,14 +3894,10 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
}
|
||||
}
|
||||
|
||||
/** POST /autotag/single for `image` (#379: `existing` "skip" answers already_tagged for an auto-tagged image). */
|
||||
async requestReviewTags(image, existing) {
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/autotag/single', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(ImageHelpers.autotagSingleBody(image, { ...this.autoTagState, existing }))
|
||||
});
|
||||
return response.json();
|
||||
// Check if tags array has "real" tags (excluding auto-scanned and prepend:* metadata)
|
||||
getRealTags(tags) {
|
||||
if (!tags || !Array.isArray(tags)) return [];
|
||||
return tags.filter(tag => tag !== 'auto-scanned' && !tag.startsWith('prepend:') && !tag.startsWith('append:'));
|
||||
}
|
||||
|
||||
// Show the re-tag confirmation modal
|
||||
@@ -3953,48 +3949,26 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
// always provides a servable url; never rebuild it from the path.
|
||||
const imageUrl = this.getImageUrl(image);
|
||||
|
||||
this.autoTagState.currentImage = image;
|
||||
this.autoTagState.currentSha256 = null;
|
||||
this.autoTagState.tagsExpanded = false; // Reset accordion for new image
|
||||
// Check if image already has real tags (excluding auto-scanned)
|
||||
const realTags = this.getRealTags(image.prompt_tags);
|
||||
|
||||
let data;
|
||||
try {
|
||||
// Each image is first asked with "skip": an image a tagging model already tagged answers
|
||||
// already_tagged (its image tags, never its prompt's) without running the model (#379).
|
||||
data = await this.requestReviewTags(image, 'skip');
|
||||
if (data.success && data.already_tagged) {
|
||||
if (this.autoTagState.skipAllTagged) {
|
||||
this.autoTagState.reviewIndex++;
|
||||
return this.loadNextReviewImage();
|
||||
}
|
||||
const choice = await this.showRetagConfirmation(image, imageUrl, data.image_tags || []);
|
||||
if (choice === 'skip' || choice === 'skipAll') {
|
||||
this.autoTagState.reviewIndex++;
|
||||
return this.loadNextReviewImage();
|
||||
}
|
||||
// choice === 'retag': generate new tags for it
|
||||
this.showReviewImage(image, imageUrl);
|
||||
data = await this.requestReviewTags(image, 'merge');
|
||||
} else {
|
||||
this.showReviewImage(image, imageUrl);
|
||||
if (realTags.length > 0) {
|
||||
// Skip if "Skip All Tagged" was selected
|
||||
if (this.autoTagState.skipAllTagged) {
|
||||
this.autoTagState.reviewIndex++;
|
||||
return this.loadNextReviewImage();
|
||||
}
|
||||
if (data.success) {
|
||||
this.autoTagState.currentTags = data.tags;
|
||||
this.autoTagState.currentSha256 = data.sha256 || null;
|
||||
this.renderReviewTags();
|
||||
} else {
|
||||
document.getElementById('reviewTagsVisible').innerHTML =
|
||||
`<div class="text-pm-error">Error: ${escapeHtml(data.error)}</div>`;
|
||||
|
||||
// Show confirmation modal
|
||||
const choice = await this.showRetagConfirmation(image, imageUrl, realTags);
|
||||
|
||||
if (choice === 'skip' || choice === 'skipAll') {
|
||||
this.autoTagState.reviewIndex++;
|
||||
return this.loadNextReviewImage();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error generating tags:', error);
|
||||
this.showReviewImage(image, imageUrl);
|
||||
document.getElementById('reviewTagsVisible').innerHTML =
|
||||
'<div class="text-pm-error">Failed to generate tags</div>';
|
||||
// choice === 'retag' - continue with tagging
|
||||
}
|
||||
}
|
||||
|
||||
showReviewImage(image, imageUrl) {
|
||||
document.getElementById('reviewImage').src = imageUrl;
|
||||
// The record's file name, else the last segment of its path (#328: `filename` was never declared).
|
||||
const filename = image.filename || String(image.image_path || image.path || '').split(/[\\/]/).pop();
|
||||
@@ -4003,6 +3977,37 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
document.getElementById('reviewTagsHidden').innerHTML = '';
|
||||
document.getElementById('reviewTagsAccordion').classList.add('hidden');
|
||||
document.getElementById('reviewTagsToggle').classList.add('hidden');
|
||||
|
||||
// Store prompt_id from database for applying tags later
|
||||
this.autoTagState.currentPromptId = image.prompt_id;
|
||||
this.autoTagState.tagsExpanded = false; // Reset accordion for new image
|
||||
|
||||
try {
|
||||
// Review rows come from /images/all (database ids), so this resolves to image_id
|
||||
const requestBody = ImageHelpers.autotagSingleBody(image, this.autoTagState);
|
||||
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/autotag/single', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(requestBody)
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
this.autoTagState.currentTags = data.tags;
|
||||
// Use prompt_id from database, fallback to API response
|
||||
this.autoTagState.currentPromptId = image.prompt_id || data.prompt_id;
|
||||
this.renderReviewTags();
|
||||
} else {
|
||||
document.getElementById('reviewTagsVisible').innerHTML =
|
||||
`<div class="text-pm-error">Error: ${escapeHtml(data.error)}</div>`;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error generating tags:', error);
|
||||
document.getElementById('reviewTagsVisible').innerHTML =
|
||||
'<div class="text-pm-error">Failed to generate tags</div>';
|
||||
}
|
||||
}
|
||||
|
||||
/** /prompt_manager/images/all is capped per call; collect every page. */
|
||||
@@ -4121,9 +4126,15 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
}
|
||||
|
||||
async applyReviewTags() {
|
||||
// #379: the reviewed tags go to this image (linked or not), never its prompt; an empty
|
||||
// list still marks the image as reviewed by a tagging model.
|
||||
if (!this.autoTagState.currentSha256) {
|
||||
if (this.autoTagState.currentTags.length === 0) {
|
||||
this.skipReviewImage();
|
||||
return;
|
||||
}
|
||||
|
||||
const promptId = this.autoTagState.currentPromptId;
|
||||
|
||||
if (!promptId) {
|
||||
this.showNotification('No linked prompt found for this image', 'warning');
|
||||
this.skipReviewImage();
|
||||
return;
|
||||
}
|
||||
@@ -4132,8 +4143,10 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/autotag/apply', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(ImageHelpers.autotagApplyBody(
|
||||
this.autoTagState.currentImage, this.autoTagState.currentSha256, this.autoTagState.currentTags))
|
||||
body: JSON.stringify({
|
||||
prompt_id: promptId,
|
||||
tags: this.autoTagState.currentTags
|
||||
})
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
+16
-20
@@ -3560,24 +3560,19 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
} else if (data.type === 'complete') {
|
||||
this.autoTagState.eventSource.close();
|
||||
this.hideModal("autoTagProgressModal");
|
||||
this.showNotification(ImageHelpers.autotagCompleteMessage(data.tagged), 'success');
|
||||
this.showNotification(`Auto tagging complete! Applied tags to ${data.tagged || 0} prompts.`, 'success');
|
||||
this.loadImages(); // Refresh the gallery
|
||||
} else if (data.type === 'error') {
|
||||
this.autoTagState.eventSource.close();
|
||||
this.hideModal("autoTagProgressModal");
|
||||
this.showNotification(`Auto tag error: ${data.message}`, 'error');
|
||||
} else if (data.type === 'cancelled') {
|
||||
// Cancelled in Prompt Manager's Activity (#379): as its own cancel.
|
||||
this.autoTagState.eventSource.close();
|
||||
this.hideModal("autoTagProgressModal");
|
||||
this.showNotification('Auto tagging cancelled', 'info');
|
||||
}
|
||||
};
|
||||
|
||||
this.autoTagState.eventSource.onerror = (error) => {
|
||||
this.autoTagState.eventSource.onerror = () => {
|
||||
this.autoTagState.eventSource.close();
|
||||
this.hideModal("autoTagProgressModal");
|
||||
this.showNotification((error && error.serverMessage) || 'Connection lost during auto tagging', 'error');
|
||||
this.showNotification('Connection lost during auto tagging', 'error');
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Auto tag error:', error);
|
||||
@@ -3634,8 +3629,6 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
this.reviewRequest = request;
|
||||
this.autoTagState.currentTags = [];
|
||||
this.autoTagState.reviewPromptId = null;
|
||||
this.autoTagState.reviewImage = null;
|
||||
this.autoTagState.reviewSha256 = null;
|
||||
if (this.autoTagState.reviewIndex >= this.autoTagState.reviewImages.length) {
|
||||
this.hideModal("autoTagReviewModal");
|
||||
this.showNotification('Review complete!', 'success');
|
||||
@@ -3650,9 +3643,8 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
document.getElementById('reviewTagsContainer').innerHTML = '<div class="text-pm-secondary">Generating tags...</div>';
|
||||
|
||||
try {
|
||||
// Gallery entries come from /images/output: id is a path digest, so address by path.
|
||||
// The gallery review always suggests (it never skipped tagged images, #379).
|
||||
const requestBody = ImageHelpers.autotagSingleBody(image, { ...this.autoTagState, existing: 'merge' });
|
||||
// Gallery entries come from /images/output: id is a path digest, so address by path
|
||||
const requestBody = ImageHelpers.autotagSingleBody(image, this.autoTagState);
|
||||
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/autotag/single', {
|
||||
method: 'POST',
|
||||
@@ -3665,8 +3657,6 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
if (this.reviewRequest !== request) return;
|
||||
if (data.success) {
|
||||
this.autoTagState.reviewPromptId = data.prompt_id ?? null;
|
||||
this.autoTagState.reviewImage = image;
|
||||
this.autoTagState.reviewSha256 = data.sha256 || null;
|
||||
this.autoTagState.currentTags = data.tags;
|
||||
this.renderReviewTags();
|
||||
} else {
|
||||
@@ -3707,12 +3697,16 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
}
|
||||
|
||||
async applyReviewTags() {
|
||||
// #379: the reviewed tags go to this image (linked or not), never its prompt; an empty
|
||||
// list still marks the image as reviewed by a tagging model.
|
||||
if (!this.autoTagState.reviewSha256) {
|
||||
if (this.autoTagState.currentTags.length === 0) {
|
||||
this.skipReviewImage();
|
||||
return;
|
||||
}
|
||||
|
||||
const promptId = this.autoTagState.reviewPromptId;
|
||||
if (!promptId) {
|
||||
this.showNotification('This image has no linked prompt. Scan the output folder first.', 'warning');
|
||||
return;
|
||||
}
|
||||
const request = this.reviewRequest;
|
||||
if (this.reviewApplying === request) return;
|
||||
this.reviewApplying = request;
|
||||
@@ -3721,8 +3715,10 @@ Seed: ${this.currentMetadata.seed || 'Unknown'}`;
|
||||
const response = await ApiClient.fetchUrl('/prompt_manager/autotag/apply', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(ImageHelpers.autotagApplyBody(
|
||||
this.autoTagState.reviewImage, this.autoTagState.reviewSha256, this.autoTagState.currentTags))
|
||||
body: JSON.stringify({
|
||||
prompt_id: promptId,
|
||||
tags: this.autoTagState.currentTags
|
||||
})
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
+10
-44
@@ -70,28 +70,10 @@
|
||||
* path digest as id, which would 404, so those are addressed by path.
|
||||
*
|
||||
* @param {{id?: unknown, path?: unknown}} image
|
||||
* `existing` (#379): "skip" answers `already_tagged` for an image a model
|
||||
* already tagged; anything else is "merge" (always suggest).
|
||||
*
|
||||
* @param {{id?: unknown, path?: unknown}} image
|
||||
* @param {{modelType: string, generalThreshold?: number, characterThreshold?: number, prompt?: string, existing?: string}} settings
|
||||
* @param {{modelType: string, generalThreshold?: number, characterThreshold?: number, prompt?: string}} settings
|
||||
* @returns {object} JSON-serialisable request body
|
||||
*/
|
||||
function autotagSingleBody(image, settings) {
|
||||
const target = autotagTarget(image);
|
||||
const existing = settings.existing === "skip" ? "skip" : "merge";
|
||||
const modelType = String(settings.modelType || "");
|
||||
const params = modelType.startsWith("wd14")
|
||||
? {
|
||||
general_threshold: settings.generalThreshold,
|
||||
character_threshold: settings.characterThreshold,
|
||||
}
|
||||
: { prompt: settings.prompt };
|
||||
return { ...target, model_type: modelType, existing, ...params };
|
||||
}
|
||||
|
||||
/** The image a legacy single/apply body names: its row id, or its path (and root). */
|
||||
function autotagTarget(image) {
|
||||
const entry = image || {};
|
||||
const target =
|
||||
typeof entry.path === "string" && entry.path && !Number.isInteger(entry.id)
|
||||
@@ -100,22 +82,14 @@
|
||||
if ("path" in target && Number.isInteger(entry.root_index)) {
|
||||
target.root_index = entry.root_index;
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
/**
|
||||
* Body for POST /prompt_manager/autotag/apply (#379): the image, the
|
||||
* `sha256` /autotag/single answered and the reviewed tags. The tags go to
|
||||
* that image only, never to its prompt; an empty list marks it reviewed.
|
||||
*/
|
||||
function autotagApplyBody(image, sha256, tags) {
|
||||
return { ...autotagTarget(image), sha256: String(sha256 || ""), tags: Array.isArray(tags) ? [...tags] : [] };
|
||||
}
|
||||
|
||||
/** The legacy batch's completion toast (#379): images, never prompts. */
|
||||
function autotagCompleteMessage(tagged) {
|
||||
const count = Number.isSafeInteger(tagged) && tagged > 0 ? tagged : 0;
|
||||
return `Auto tagging complete! Tagged ${count} ${count === 1 ? "image" : "images"}.`;
|
||||
const modelType = String(settings.modelType || "");
|
||||
const params = modelType.startsWith("wd14")
|
||||
? {
|
||||
general_threshold: settings.generalThreshold,
|
||||
character_threshold: settings.characterThreshold,
|
||||
}
|
||||
: { prompt: settings.prompt };
|
||||
return { ...target, model_type: modelType, ...params };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -148,15 +122,7 @@
|
||||
}
|
||||
}
|
||||
|
||||
const api = {
|
||||
formatImageCaption,
|
||||
workflowDocumentHtml,
|
||||
autotagSingleBody,
|
||||
autotagApplyBody,
|
||||
autotagCompleteMessage,
|
||||
fallbackImageSource,
|
||||
metadataLookupUrl,
|
||||
};
|
||||
const api = { formatImageCaption, workflowDocumentHtml, autotagSingleBody, fallbackImageSource, metadataLookupUrl };
|
||||
if (typeof module !== "undefined" && module.exports) module.exports = api;
|
||||
else root.ImageHelpers = api;
|
||||
})(typeof window !== "undefined" ? window : globalThis);
|
||||
|
||||
@@ -246,29 +246,6 @@
|
||||
],
|
||||
"with_model": ["conditioning", "neg_conditioning", "samples", "latent"]
|
||||
},
|
||||
"pass_through": {
|
||||
"skip_inputs": [
|
||||
"model",
|
||||
"clip",
|
||||
"vae",
|
||||
"clip_vision",
|
||||
"clip_vision_output",
|
||||
"control_net",
|
||||
"style_model",
|
||||
"upscale_model",
|
||||
"image",
|
||||
"images",
|
||||
"pixels",
|
||||
"mask",
|
||||
"latent",
|
||||
"latent_image",
|
||||
"samples",
|
||||
"noise",
|
||||
"sampler",
|
||||
"sigmas",
|
||||
"guider"
|
||||
]
|
||||
},
|
||||
"facts": {
|
||||
"configuration_outputs": {
|
||||
"RandomNoise": { "kind": "noise", "fields": ["noise_seed"] },
|
||||
|
||||
+1
-15
@@ -59,20 +59,6 @@
|
||||
* @returns {{onmessage: Function|null, onerror: Function|null, onclose: Function|null,
|
||||
* readyState: number, close: Function, CONNECTING: number, OPEN: number, CLOSED: number}}
|
||||
*/
|
||||
// A non-2xx answer as an Error carrying its status and, when the body is
|
||||
// the API's JSON {error}, that sentence as serverMessage for the page.
|
||||
async function httpError(response) {
|
||||
const error = new Error(`HTTP ${response.status}`);
|
||||
error.status = response.status;
|
||||
try {
|
||||
const data = await response.json();
|
||||
if (data && typeof data.error === "string" && data.error) error.serverMessage = data.error;
|
||||
} catch (_) {
|
||||
// Not JSON: the status alone.
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
function connect(url, { method = "GET", body, fetchImpl } = {}) {
|
||||
const doFetch = fetchImpl || ((...args) => globalThis.fetch(...args));
|
||||
const controller = new AbortController();
|
||||
@@ -117,7 +103,7 @@
|
||||
if (body !== undefined) init.body = body;
|
||||
const requestUrl = root.ApiClient ? root.ApiClient.resolveUrl(url) : url;
|
||||
const response = await doFetch(requestUrl, init);
|
||||
if (!response.ok) throw await httpError(response);
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
if (source.readyState === CLOSED) return;
|
||||
source.readyState = OPEN;
|
||||
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
/**
|
||||
* File-manager selection on the Images page (#392, D135): pure helpers for the selection bar (images-selection.mjs).
|
||||
* Keys are image keys (`root_index:path`) in the listing's display order, so a range spans every loaded batch
|
||||
* whether or not its cards are mounted.
|
||||
*/
|
||||
|
||||
/** The press that turns on touch selection mode, and how far it may drift before it counts as a scroll. */
|
||||
export const LONG_PRESS_MS = 500;
|
||||
export const LONG_PRESS_SLOP = 10;
|
||||
|
||||
/**
|
||||
* The keys from `from` to `to`, inclusive, in `keys` order (either direction). Without a listed `from` (no anchor
|
||||
* yet, or it left the listing) the range is `to` alone; an unlisted `to` gives none.
|
||||
*/
|
||||
export function rangeKeys(keys, from, to) {
|
||||
const end = keys.indexOf(to);
|
||||
if (end < 0) return [];
|
||||
const start = from === null || from === undefined ? -1 : keys.indexOf(from);
|
||||
if (start < 0) return [to];
|
||||
return keys.slice(Math.min(start, end), Math.max(start, end) + 1);
|
||||
}
|
||||
|
||||
/** `selected` with `keys` added (`on`) or removed, frozen; the same set when nothing changes. */
|
||||
export function applyKeys(selected, keys, on) {
|
||||
if (keys.every(key => selected.has(key) === on)) return selected;
|
||||
const next = new Set(selected);
|
||||
for (const key of keys) {
|
||||
if (on) next.add(key); else next.delete(key);
|
||||
}
|
||||
return Object.freeze(next);
|
||||
}
|
||||
|
||||
/** Whether `keys` is non-empty and every one of them is in `selected`. */
|
||||
export function everyChecked(selected, keys) {
|
||||
return keys.length > 0 && selected.size >= keys.length && keys.every(key => selected.has(key));
|
||||
}
|
||||
|
||||
/** "Select all N matching" shows once every loaded image is checked and the listing holds more than is loaded. */
|
||||
export function offersAllMatching(selected, keys, { total, hasMore }) {
|
||||
return Boolean(hasMore) && total > keys.length && everyChecked(selected, keys);
|
||||
}
|
||||
|
||||
const grouped = count => count.toLocaleString('en-US');
|
||||
export const matchingLabel = total => `All ${grouped(total)} matching selected`;
|
||||
export const matchingOffer = total => `Select all ${grouped(total)} matching`;
|
||||
|
||||
/**
|
||||
* A touch long-press: `down` starts the timer, a `move` past the slop, `up` or `cancel` stops it, and
|
||||
* `onLongPress(target)` runs once it fires. `consumeClick()` answers whether the click the press ends in belongs to
|
||||
* the long press (so the page swallows it); it answers once. A new `down` forgets an unclaimed long press.
|
||||
*/
|
||||
export function createLongPress({ onLongPress, delay = LONG_PRESS_MS, slop = LONG_PRESS_SLOP,
|
||||
setTimer = setTimeout, clearTimer = clearTimeout }) {
|
||||
let timer = null, origin = null, fired = false;
|
||||
const stop = () => {
|
||||
if (timer !== null) clearTimer(timer);
|
||||
timer = null; origin = null;
|
||||
};
|
||||
return {
|
||||
down(x, y, target) {
|
||||
stop();
|
||||
fired = false;
|
||||
origin = { x, y };
|
||||
timer = setTimer(() => { timer = null; origin = null; fired = true; onLongPress(target); }, delay);
|
||||
},
|
||||
move(x, y) {
|
||||
if (origin && Math.hypot(x - origin.x, y - origin.y) > slop) stop();
|
||||
},
|
||||
up: stop,
|
||||
/** A press that ends without its click (a mouse press next, a scroll): its long press is forgotten too. */
|
||||
cancel() { stop(); fired = false; },
|
||||
/** Whether a touch press is held and has not yet fired (its context menu belongs to the long press). */
|
||||
pending: () => timer !== null,
|
||||
consumeClick() {
|
||||
const claimed = fired;
|
||||
fired = false;
|
||||
return claimed;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -4,16 +4,11 @@ import { inspectPath, toInspectionRecord } from './inspection-state.mjs';
|
||||
import { BUNDLE_FILENAME, metadataBundle, collectImagesMetadata, exportOutcome } from './images-export.mjs';
|
||||
import { CATEGORY_DIALOG, CATEGORY_FAILED, CATEGORY_MAX_LENGTH, CATEGORY_NONE_LINKED, categoryOutcome, categoryTargets, updatedCount } from './images-category.mjs';
|
||||
import { openInputDialog } from './workspace-dialogs.mjs';
|
||||
import { MAX_FILTER_IMAGES, TOO_MANY_MATCHING, openAddTags, openAutoTag } from './images-tagging-bulk.mjs';
|
||||
import { openAddTags, openAutoTag } from './images-tagging-bulk.mjs';
|
||||
import { showWorkspaceToast } from './toast.mjs';
|
||||
import { applyKeys, createLongPress, matchingLabel, matchingOffer, offersAllMatching, rangeKeys } from './images-multiselect.mjs';
|
||||
|
||||
const CHECKBOX = '.image-overlays input[type="checkbox"]';
|
||||
const CHECKBOX_LABEL = '.image-overlays > label';
|
||||
const CARD = 'article[data-image-key]';
|
||||
const CATEGORY_URL = '/prompt_manager/bulk/category';
|
||||
// D135: Export metadata reads every image's inspection in the browser, so it waits for loaded images.
|
||||
export const EXPORT_LOADED_ONLY = 'Export metadata reads each image in the browser, so it works on loaded images only.';
|
||||
|
||||
function el(tag, text, className) {
|
||||
const element = document.createElement(tag);
|
||||
@@ -34,196 +29,60 @@ function el(tag, text, className) {
|
||||
* - `retain(kept)` drops the checked files a refreshed listing no longer holds (a tag save under a filter, I7).
|
||||
* - With the Activity store `activity`, Add tags and Auto-tag selected (D85) follow the count, as the concept's Add tags
|
||||
* leads its actions; `onTagged(keys)` re-reads the files a finished tagging job tagged. The selection stays.
|
||||
*
|
||||
* File-manager selection (#392, D135, beyond the concept's checkbox): the anchor is the last image toggled on its own;
|
||||
* Shift+click (a checkbox or anywhere on a card) applies the anchor's range, Ctrl/Cmd+click toggles a card without
|
||||
* opening it, a touch long-press starts touch selection mode in which taps toggle. `selectAll`, `extend` and
|
||||
* `clear` serve the grid's keys (images.mjs). `getListing()` gives `{total, hasMore, positiveCount}` and
|
||||
* `getFilter()` the listing filter: once every loaded image is checked and more match, the bar offers "Select all N
|
||||
* matching", whose actions send that filter to the server.
|
||||
*/
|
||||
export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, onCategorySet, activity = null, onTagged,
|
||||
getListing, getFilter, checked: initial = Object.freeze(new Set()) }) {
|
||||
checked: initial = Object.freeze(new Set()) }) {
|
||||
let checked = initial;
|
||||
let exporting = false, categorizing = false;
|
||||
// D135: the range anchor, "all N matching" mode, touch selection mode and a pending Shift+click on a checkbox.
|
||||
let anchor = null, allMatching = false, touchMode = false, lastPointer = 'mouse', shiftBox = null;
|
||||
const element = el('div');
|
||||
element.id = 'bulkbar';
|
||||
const bulk = el('div', undefined, 'bulkbar'), count = el('strong');
|
||||
const selectMatching = createButton('');
|
||||
scope.listen(selectMatching, 'click', () => {
|
||||
allMatching = true;
|
||||
render();
|
||||
focusAfterOffer();
|
||||
});
|
||||
const setCategory = createButton('Set category');
|
||||
scope.listen(setCategory, 'click', () => openCategory(setCategory));
|
||||
const exportMetadata = createButton('Export metadata');
|
||||
scope.listen(exportMetadata, 'click', () => { if (!allMatching) exportChecked(exportMetadata); });
|
||||
scope.listen(exportMetadata, 'click', () => exportChecked(exportMetadata));
|
||||
const clearButton = createButton('Clear');
|
||||
// Clear hides the bar it sits in, so focus moves to the page heading instead of falling to the page.
|
||||
scope.listen(clearButton, 'click', () => { clear(); focusFallback.focus({ preventScroll: true }); });
|
||||
bulk.append(count);
|
||||
if (activity) {
|
||||
const addTags = createButton('Add tags');
|
||||
scope.listen(addTags, 'click', () => openAddTags({ scope, store: activity, images: checkedImages(), trigger: addTags, onTagged,
|
||||
...matchingTarget() }));
|
||||
scope.listen(addTags, 'click', () => openAddTags({ scope, store: activity, images: checkedImages(), trigger: addTags, onTagged }));
|
||||
const autoTag = createButton('Auto-tag selected'); // D85
|
||||
scope.listen(autoTag, 'click', () => openAutoTag({ api, scope, store: activity, images: checkedImages(), trigger: autoTag, onTagged,
|
||||
...matchingTarget() }));
|
||||
scope.listen(autoTag, 'click', () => openAutoTag({ api, scope, store: activity, images: checkedImages(), trigger: autoTag, onTagged }));
|
||||
bulk.append(addTags, autoTag);
|
||||
}
|
||||
bulk.append(setCategory, exportMetadata, clearButton);
|
||||
|
||||
/** The count updates in place: the same bar and `strong`, its text written only when it changes. */
|
||||
function render() {
|
||||
if (!checked.size && !allMatching) { touchMode = false; bulk.remove(); return; }
|
||||
const listing = getListing();
|
||||
const label = allMatching ? matchingLabel(listing.total) : selectionLabel(checked.size);
|
||||
if (!checked.size) { bulk.remove(); return; }
|
||||
const label = selectionLabel(checked.size);
|
||||
if (count.textContent !== label) count.textContent = label;
|
||||
// "Select all N matching" (D135) follows the count while every loaded image is checked and more match.
|
||||
if (!allMatching && offersAllMatching(checked, loadedKeys(), listing)) {
|
||||
const offer = matchingOffer(listing.total);
|
||||
if (selectMatching.textContent !== offer) selectMatching.textContent = offer;
|
||||
if (count.nextElementSibling !== selectMatching) count.after(selectMatching);
|
||||
} else selectMatching.remove();
|
||||
if (allMatching) {
|
||||
exportMetadata.setAttribute('aria-disabled', 'true');
|
||||
exportMetadata.title = EXPORT_LOADED_ONLY;
|
||||
} else if (exportMetadata.title) {
|
||||
exportMetadata.removeAttribute('title');
|
||||
if (!exporting) exportMetadata.removeAttribute('aria-disabled');
|
||||
}
|
||||
if (bulk.parentNode !== element) element.append(bulk);
|
||||
}
|
||||
/** Every rendered checkbox shows the selection (cards mounted later read `isChecked`). */
|
||||
function paint() {
|
||||
for (const box of grid.querySelectorAll(CHECKBOX)) {
|
||||
const on = isChecked(box.closest(CARD).dataset.imageKey);
|
||||
if (box.checked !== on) box.checked = on;
|
||||
}
|
||||
}
|
||||
function update(next) {
|
||||
checked = next;
|
||||
paint();
|
||||
render();
|
||||
}
|
||||
/** Leaving "all N matching": the loaded images become the explicit selection the next change applies to. */
|
||||
function settleMatching() {
|
||||
if (!allMatching) return;
|
||||
allMatching = false;
|
||||
checked = Object.freeze(new Set(loadedKeys()));
|
||||
}
|
||||
function clear() {
|
||||
anchor = null; touchMode = false;
|
||||
if (!checked.size && !allMatching) return;
|
||||
allMatching = false;
|
||||
update(Object.freeze(new Set()));
|
||||
if (!checked.size) return;
|
||||
checked = Object.freeze(new Set());
|
||||
for (const box of grid.querySelectorAll(CHECKBOX)) box.checked = false;
|
||||
render();
|
||||
}
|
||||
/** Keep only the checked files `kept(key)` still lists (a refreshed result), so the count is what the actions apply to. */
|
||||
function retain(kept) {
|
||||
if (allMatching) { render(); return; }
|
||||
const next = [...checked].filter(kept);
|
||||
if (next.length === checked.size) return;
|
||||
checked = Object.freeze(new Set(next));
|
||||
render();
|
||||
}
|
||||
function loadedKeys() {
|
||||
return getItems().map(imageKey);
|
||||
}
|
||||
function isChecked(key) {
|
||||
return allMatching || checked.has(key);
|
||||
}
|
||||
function checkedImages() {
|
||||
return allMatching ? getItems() : getItems().filter(image => checked.has(imageKey(image)));
|
||||
}
|
||||
/** The actions' images for "all N matching": the listing filter and its total, which the server resolves. */
|
||||
function matchingTarget() {
|
||||
return allMatching ? { filter: getFilter(), count: getListing().total } : {};
|
||||
}
|
||||
/** Toggle one image on its own; it becomes the anchor. */
|
||||
function toggle(key, on = !isChecked(key)) {
|
||||
settleMatching();
|
||||
anchor = key;
|
||||
update(toggleSelection(checked, key, on));
|
||||
}
|
||||
/** The anchor's range to `key` takes `on`; without an anchor `key` is toggled on and becomes it. */
|
||||
function range(key, on = true) {
|
||||
settleMatching();
|
||||
const keys = rangeKeys(loadedKeys(), anchor, key);
|
||||
if (anchor === null || keys.length === 1) anchor = key;
|
||||
update(applyKeys(checked, keys, on));
|
||||
}
|
||||
/** Shift+arrow keys: the range from the anchor (`from`, the card focus left, when there is none) to `to`. */
|
||||
function extend(from, to) {
|
||||
if (anchor === null || !loadedKeys().includes(anchor)) anchor = from;
|
||||
range(to, true);
|
||||
}
|
||||
function selectAll() {
|
||||
if (allMatching) return;
|
||||
update(applyKeys(checked, loadedKeys(), true));
|
||||
}
|
||||
function focusAfterOffer() {
|
||||
// The pressed offer left the bar: focus moves to the count's first action instead of the page.
|
||||
(bulk.querySelector('button:not([aria-disabled="true"])') ?? focusFallback).focus({ preventScroll: true });
|
||||
return getItems().filter(image => checked.has(imageKey(image)));
|
||||
}
|
||||
// Every card carries its image key; its only form control is the checkbox, so a toggle never opens or inspects.
|
||||
scope.listen(grid, 'change', event => {
|
||||
const box = event.target;
|
||||
if (!box.matches(CHECKBOX)) return;
|
||||
const key = box.closest(CARD).dataset.imageKey;
|
||||
if (shiftBox === key) range(key, box.checked);
|
||||
else toggle(key, box.checked);
|
||||
shiftBox = null;
|
||||
});
|
||||
// D135: modified clicks select instead of opening, inspecting or starring. Captured, so the page's own click
|
||||
// handler (images.mjs) never sees them; a plain click keeps its meaning.
|
||||
grid.addEventListener('click', event => {
|
||||
const card = event.target.closest(CARD);
|
||||
if (!card) return;
|
||||
const key = card.dataset.imageKey;
|
||||
if (longPress.consumeClick()) { swallow(event); return; }
|
||||
if (event.target.closest(CHECKBOX_LABEL)) {
|
||||
// The box toggles natively; its change event applies the range.
|
||||
if (event.shiftKey) shiftBox = key;
|
||||
return;
|
||||
}
|
||||
// A keyboard click (Enter, Space: `detail` 0) keeps its plain meaning in touch selection mode.
|
||||
const tapped = touchMode && lastPointer !== 'mouse' && event.detail !== 0;
|
||||
if (event.shiftKey) { swallow(event); range(key, true); }
|
||||
else if (event.ctrlKey || event.metaKey || tapped) { swallow(event); toggle(key); }
|
||||
}, { capture: true, signal: scope.signal });
|
||||
function swallow(event) {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}
|
||||
// A touch long-press selects the card and starts touch selection mode (D135).
|
||||
const longPress = createLongPress({ onLongPress: key => {
|
||||
touchMode = true;
|
||||
anchor = key;
|
||||
if (!isChecked(key)) toggle(key, true);
|
||||
} });
|
||||
scope.cleanup(() => longPress.cancel());
|
||||
scope.listen(grid, 'pointerdown', event => {
|
||||
lastPointer = event.pointerType || 'mouse';
|
||||
shiftBox = null;
|
||||
const card = event.target.closest(CARD);
|
||||
if (lastPointer === 'mouse' || !card || !event.isPrimary) { longPress.cancel(); return; }
|
||||
longPress.down(event.clientX, event.clientY, card.dataset.imageKey);
|
||||
});
|
||||
scope.listen(grid, 'pointermove', event => longPress.move(event.clientX, event.clientY));
|
||||
for (const type of ['pointerup', 'pointercancel', 'pointerleave']) scope.listen(grid, type, () => longPress.up());
|
||||
// A long-press's context menu (Android) would cover the selection it just started.
|
||||
scope.listen(grid, 'contextmenu', event => {
|
||||
if (lastPointer !== 'mouse' && (touchMode || longPress.pending())) event.preventDefault();
|
||||
});
|
||||
// Escape in the bar clears the selection; the bar hides, so focus moves to the page heading as after Clear.
|
||||
scope.listen(element, 'keydown', event => {
|
||||
if (event.key !== 'Escape' || (!checked.size && !allMatching)) return;
|
||||
event.preventDefault();
|
||||
clear();
|
||||
focusFallback.focus({ preventScroll: true });
|
||||
checked = toggleSelection(checked, box.closest('article[data-image-key]').dataset.imageKey, box.checked);
|
||||
render();
|
||||
});
|
||||
|
||||
/** Busy until it settles: `aria-disabled` now, released in a `finally` so a throw cannot leave it stuck. */
|
||||
@@ -267,11 +126,8 @@ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, o
|
||||
*/
|
||||
function openCategory(button) {
|
||||
if (categorizing) return;
|
||||
// All matching (D135): the listing's positive count says whether any match is linked; the server resolves.
|
||||
if (allMatching && getListing().total > MAX_FILTER_IMAGES) { showWorkspaceToast(TOO_MANY_MATCHING); return; }
|
||||
const targets = allMatching ? {filter: getFilter()} : categoryTargets(checkedImages());
|
||||
const linked = allMatching ? getListing().positiveCount !== 0 : targets.promptIds.length > 0;
|
||||
if (!linked) { showWorkspaceToast(CATEGORY_NONE_LINKED); return; }
|
||||
const targets = categoryTargets(checkedImages());
|
||||
if (!targets.promptIds.length) { showWorkspaceToast(CATEGORY_NONE_LINKED); return; }
|
||||
openInputDialog({ scope, trigger: button, title: CATEGORY_DIALOG.title, label: CATEGORY_DIALOG.label,
|
||||
action: CATEGORY_DIALOG.action, maxLength: CATEGORY_MAX_LENGTH, onApply: category => applyCategory(button, targets, category) });
|
||||
}
|
||||
@@ -279,12 +135,12 @@ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, o
|
||||
* The write carries no page signal: once sent it finishes even if the page is left (`writing` then tells the page
|
||||
* not to save a visit that predates it). A failure keeps the selection; a success reloads through `onCategorySet`.
|
||||
*/
|
||||
async function applyCategory(button, { promptIds, skipped, filter }, category) {
|
||||
async function applyCategory(button, { promptIds, skipped }, category) {
|
||||
categorizing = true;
|
||||
const release = busy(button);
|
||||
let result = null;
|
||||
try {
|
||||
result = await api.post(CATEGORY_URL, filter ? { images_filter: filter, category } : { prompt_ids: promptIds, category });
|
||||
result = await api.post(CATEGORY_URL, { prompt_ids: promptIds, category });
|
||||
} catch {
|
||||
// Reported below; the selection stays as it was.
|
||||
} finally {
|
||||
@@ -293,9 +149,7 @@ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, o
|
||||
}
|
||||
if (scope.signal.aborted) return;
|
||||
if (!result) { showWorkspaceToast(CATEGORY_FAILED); return; }
|
||||
if (filter && !result.prompt_count) { showWorkspaceToast(CATEGORY_NONE_LINKED); return; }
|
||||
showWorkspaceToast(filter ? categoryOutcome(updatedCount(result, result.prompt_count), Number(result.skipped) || 0)
|
||||
: categoryOutcome(updatedCount(result, promptIds.length), skipped));
|
||||
showWorkspaceToast(categoryOutcome(updatedCount(result, promptIds.length), skipped));
|
||||
const focused = document.activeElement === button;
|
||||
onCategorySet();
|
||||
// The reload hides the bar with the focused action in it, as Clear does: focus moves to the page heading.
|
||||
@@ -305,16 +159,10 @@ export function mountSelectionBar({ scope, api, grid, getItems, focusFallback, o
|
||||
render();
|
||||
return {
|
||||
element,
|
||||
isChecked,
|
||||
// A saved visit keeps "all N matching" as its loaded images; the offer then shows again.
|
||||
checked: () => (allMatching ? Object.freeze(new Set(loadedKeys())) : checked),
|
||||
hasSelection: () => allMatching || checked.size > 0,
|
||||
isChecked: key => checked.has(key),
|
||||
checked: () => checked,
|
||||
clear,
|
||||
retain,
|
||||
selectAll,
|
||||
extend,
|
||||
/** A loaded batch or a refreshed total: the count and the offer follow it. */
|
||||
listingChanged: () => { if (checked.size || allMatching) render(); },
|
||||
/** Whether a Set category write is still in flight. */
|
||||
writing: () => categorizing,
|
||||
};
|
||||
|
||||
@@ -56,16 +56,6 @@ function queryFor(filters, batchSize) {
|
||||
return query.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* The listing's filter as the bulk routes take it (#392 "Select all N matching"): the same parameters the listing
|
||||
* query carries, as an object of strings, without `mode`, `limit` and `cursor`.
|
||||
*/
|
||||
export function listingFilter(filters) {
|
||||
const query = new URLSearchParams(queryFor(filters, 1));
|
||||
query.delete('mode'); query.delete('limit');
|
||||
return Object.fromEntries(query);
|
||||
}
|
||||
|
||||
function validatePage(page, cursor) {
|
||||
if (!page || !Array.isArray(page.images) || !Number.isSafeInteger(page.total) || page.total < 0 ||
|
||||
typeof page.has_more !== 'boolean' ||
|
||||
|
||||
@@ -9,10 +9,6 @@
|
||||
* apply** (D90, ruling T3-bulk): **Apply automatically** starts the job, **Review each image** starts the browser
|
||||
* review session (images-tagging-review.mjs). A model that is not downloaded keeps Start disabled and says why.
|
||||
*
|
||||
* - **All N matching** (#392, D135): with `filter` (the listing's, images-state.mjs `listingFilter`) and `count` (its
|
||||
* total), both send the filter instead of the targets and the server resolves it, up to 10,000 images. Review each
|
||||
* image walks images in the browser, so it is not offered.
|
||||
*
|
||||
* The toast says the job started (the Activity store's "Tagging started." / "Auto-tagging started."), or why it did
|
||||
* not: a running record-changing job (named, with its link), a held model, a refused body. Activity shows progress.
|
||||
* The selection stays either way. When a started job ends having tagged files, `onTagged(keys)` re-reads them.
|
||||
@@ -30,9 +26,6 @@ import { showWorkspaceToast } from './toast.mjs';
|
||||
export const IMAGE_TAG_JOB = 'image-tags';
|
||||
export const MAX_BULK_IMAGES = 500;
|
||||
export const TOO_MANY = 'Select at most 500 images.';
|
||||
// #392: a filter start resolves on the server (py/api/images.py MAX_LISTING_SELECTION).
|
||||
export const MAX_FILTER_IMAGES = 10000;
|
||||
export const TOO_MANY_MATCHING = 'Narrow the view to at most 10,000 images.';
|
||||
export const NO_TAGS = 'Enter at least one tag.'; // D118
|
||||
export const ADD_DIALOG = Object.freeze({title: 'Tag selected images', label: 'Tags, separated by commas', action: 'Apply'});
|
||||
export const AUTO_TITLE = 'Auto-tag selected images';
|
||||
@@ -60,16 +53,6 @@ export function bulkTargets(images) {
|
||||
return images.map(imageTarget).filter(Boolean);
|
||||
}
|
||||
|
||||
/**
|
||||
* The job body's images: `{filter}` for all matching (null over MAX_FILTER_IMAGES), else `{targets}` (null over 500).
|
||||
* Returns `[images, refusal]`, the refusal being the toast to show.
|
||||
*/
|
||||
function jobImages(images, filter, count) {
|
||||
if (filter) return count > MAX_FILTER_IMAGES ? [null, TOO_MANY_MATCHING] : [{filter}, null];
|
||||
const targets = bulkTargets(images);
|
||||
return targets ? [{targets}, null] : [null, TOO_MANY];
|
||||
}
|
||||
|
||||
/** The Add tags field's names, deduplicated as the Tag image dialog's manual field is. */
|
||||
export const parseBulkTags = parseManualTags;
|
||||
|
||||
@@ -124,13 +107,13 @@ function followTagged({store, scope, job, images, onTagged}) {
|
||||
* to no names keeps the dialog and its text, and reports NO_TAGS on the field (D118). `openInput` is the dialog seam.
|
||||
*/
|
||||
export function openAddTags({scope, store, images, trigger, onTagged = () => {}, notify = showWorkspaceToast,
|
||||
openInput = openInputDialog, filter = null, count = images.length}) {
|
||||
const [chosen, refusal] = jobImages(images, filter, count);
|
||||
if (refusal) { notify(refusal); return; }
|
||||
openInput = openInputDialog}) {
|
||||
const targets = bulkTargets(images);
|
||||
if (!targets) { notify(TOO_MANY); return; }
|
||||
openInput({scope, trigger, ...ADD_DIALOG, validate: text => (parseBulkTags(text).length ? null : NO_TAGS),
|
||||
onApply: async text => {
|
||||
const tags = parseBulkTags(text);
|
||||
const job = await startTagJob({store, scope, body: {mode: 'add', ...chosen, tags}, notify});
|
||||
const job = await startTagJob({store, scope, body: {mode: 'add', targets, tags}, notify});
|
||||
if (job) followTagged({store, scope, job, images, onTagged});
|
||||
}});
|
||||
}
|
||||
@@ -158,16 +141,15 @@ function applyChoice(onChange) {
|
||||
* Auto-tag selected (D85): the model panel, How to apply (D90), Cancel and Start. Start validates the fields as
|
||||
* Generate does, then starts the job or the review session over `images` (the checked files, taken now).
|
||||
*/
|
||||
export function openAutoTag({api, scope, store, images, trigger, onTagged = () => {}, notify = showWorkspaceToast,
|
||||
filter = null, count = images.length}) {
|
||||
const [chosen, refusal] = jobImages(images, filter, count);
|
||||
if (refusal) { notify(refusal); return; }
|
||||
export function openAutoTag({api, scope, store, images, trigger, onTagged = () => {}, notify = showWorkspaceToast}) {
|
||||
const targets = bulkTargets(images);
|
||||
if (!targets) { notify(TOO_MANY); return; }
|
||||
const listed = images.filter(imageTarget);
|
||||
const local = createScope();
|
||||
const options = createTaggingOptionsController({api});
|
||||
// D85: the Tag image dialog's model panel without Generate; its Suggestions field is the job's count (D91).
|
||||
const {element: panel, validate} = renderModelPanel({scope: local, controller: options,
|
||||
heading: `Auto-tag ${countOf(filter ? count : listed.length, 'image')}`, generate: false}); // D118
|
||||
heading: `Auto-tag ${countOf(listed.length, 'image')}`, generate: false}); // D118
|
||||
let choice = APPLY_CHOICES[0];
|
||||
const actions = el('div', undefined, 'dialog-actions');
|
||||
const cancel = createButton('Cancel');
|
||||
@@ -177,8 +159,7 @@ export function openAutoTag({api, scope, store, images, trigger, onTagged = () =
|
||||
const status = el('div', undefined, 'small');
|
||||
status.id = 'tag-generation-status';
|
||||
status.setAttribute('role', 'status');
|
||||
// D90; all matching (D135) has no Review each image, which walks images in the browser.
|
||||
const choices = filter ? null : applyChoice(next => { choice = next; start.textContent = next.action; });
|
||||
const choices = applyChoice(next => { choice = next; start.textContent = next.action; }); // D90
|
||||
const render = state => {
|
||||
if (status.textContent !== state.unavailable) status.textContent = state.unavailable;
|
||||
if (state.canStart) start.removeAttribute('aria-disabled');
|
||||
@@ -197,11 +178,11 @@ export function openAutoTag({api, scope, store, images, trigger, onTagged = () =
|
||||
return;
|
||||
}
|
||||
closeWorkspaceDialog();
|
||||
startTagJob({store, scope, body: {mode: 'auto', ...chosen, ...request}, notify}).then(job => {
|
||||
startTagJob({store, scope, body: {mode: 'auto', targets, ...request}, notify}).then(job => {
|
||||
if (job) followTagged({store, scope, job, images: listed, onTagged});
|
||||
});
|
||||
});
|
||||
openContentDialog({scope, title: AUTO_TITLE, nodes: [panel, choices, status, actions].filter(Boolean), trigger, onClose: () => {
|
||||
openContentDialog({scope, title: AUTO_TITLE, nodes: [panel, choices, status, actions], trigger, onClose: () => {
|
||||
document.getElementById('modal')?.classList.remove('tagging-modal');
|
||||
local.dispose();
|
||||
options.dispose();
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createScope, createButton } from '../shared/toolkit/core.js';
|
||||
import { createImageStream, withItem, imageKey, countLine, captionDate, scopeLabel, activeRefinements, initialImagesView, listingFilter } from './images-state.mjs';
|
||||
import { createImageStream, withItem, imageKey, countLine, captionDate, scopeLabel, activeRefinements, initialImagesView } from './images-state.mjs';
|
||||
import { createImageCard } from './image-card.mjs';
|
||||
import { layoutImages, visibleImages, gridMetrics, stacksCardInfo } from './image-layout.mjs';
|
||||
import { mountFileInspector } from './file-inspector.mjs';
|
||||
@@ -254,15 +254,8 @@ export function mountImages(main, api, parentScope, {embedded = false, fetchPage
|
||||
body.append(collection, details);
|
||||
// The checked cards and the concept's selection bar (#249 D8), between the filter row and the gallery layout. The
|
||||
// selection is kept across appended batches and cleared with the collection; the Library-embedded gallery has none.
|
||||
const bar = embedded ? null : mountSelectionBar({scope, api, grid, getItems:() => state?.items ?? [], focusFallback,
|
||||
checked:visit?.checked, onCategorySet:() => { loadCategories(); reset(); }, activity, onTagged:keys => refreshTagged(keys),
|
||||
// #392 "Select all N matching": the listing's total and its filter, as the server lists them.
|
||||
getListing:() => ({total:state?.total ?? 0, hasMore:Boolean(state?.hasMore), positiveCount:state?.positiveCount ?? null}),
|
||||
// A Source filter carries its root's scope, so a renumbered source is refused (source-changed), not acted on.
|
||||
getFilter:() => {
|
||||
const filter = listingFilter(filters), scope = state?.items[0]?.root_scope;
|
||||
return filter.root !== undefined && typeof scope === 'string' ? {...filter, root_scope:scope} : filter;
|
||||
}});
|
||||
const bar = embedded ? null : mountSelectionBar({scope, api, grid, getItems:() => state.items, focusFallback,
|
||||
checked:visit?.checked, onCategorySet:() => { loadCategories(); reset(); }, activity, onTagged:keys => refreshTagged(keys)});
|
||||
if (bar) main.append(bar.element);
|
||||
main.append(body);
|
||||
const mediaViewer = createMediaViewer({api,container:main,parentScope:scope,inspectionController:inspection,sender,
|
||||
@@ -393,19 +386,8 @@ export function mountImages(main, api, parentScope, {embedded = false, fetchPage
|
||||
const columns = geometry.columns;
|
||||
const target = { Home: 0, End: state.items.length - 1, ArrowRight: index + 1,
|
||||
ArrowLeft: index - 1, ArrowDown: index + columns, ArrowUp: index - columns }[event.key];
|
||||
if (target !== undefined) {
|
||||
event.preventDefault();
|
||||
const next = Math.max(0, Math.min(state.items.length - 1, target));
|
||||
focusImage(next);
|
||||
// D135: Shift extends the selection from the anchor (this card when there is none) to the new card.
|
||||
if (event.shiftKey && bar) bar.extend(card.dataset.imageKey, imageKey(state.items[next]));
|
||||
} else if (bar && (event.ctrlKey || event.metaKey) && !event.altKey && event.key.toLowerCase() === 'a') {
|
||||
event.preventDefault();
|
||||
bar.selectAll();
|
||||
} else if (bar && event.key === 'Escape' && bar.hasSelection()) {
|
||||
event.preventDefault();
|
||||
bar.clear();
|
||||
} else if (event.key === 'Tab') {
|
||||
if (target !== undefined) { event.preventDefault(); focusImage(Math.max(0, Math.min(state.items.length - 1, target))); }
|
||||
else if (event.key === 'Tab') {
|
||||
const backward = event.shiftKey && event.target.matches('.image-open');
|
||||
const forward = !event.shiftKey && event.target.closest('.image-card-actions');
|
||||
const adjacent = backward ? index - 1 : index + 1;
|
||||
@@ -480,7 +462,6 @@ export function mountImages(main, api, parentScope, {embedded = false, fetchPage
|
||||
if (replaced && selection !== null && !itemIndexes.has(selection)) dropSelection();
|
||||
if (replaced) bar?.retain(key => itemIndexes.has(key));
|
||||
state = next;
|
||||
bar?.listingChanged();
|
||||
if (!embedded) {
|
||||
setText(count, countLine(next));
|
||||
setText(captionDay, captionDate(next.items, filters.sort));
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user